claude-db 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +236 -0
- package/README.md +74 -113
- package/dist/capture/flush.d.ts +1 -0
- package/dist/capture/flush.js +4 -3
- package/dist/capture/git.js +2 -1
- package/dist/capture/index.d.ts +5 -1
- package/dist/capture/index.js +4 -1
- package/dist/capture/manual.js +1 -1
- package/dist/capture/progress.js +1 -0
- package/dist/capture/redact.d.ts +1 -0
- package/dist/capture/redact.js +17 -0
- package/dist/capture/reingest.d.ts +19 -0
- package/dist/capture/reingest.js +56 -0
- package/dist/capture/scrub.d.ts +11 -0
- package/dist/capture/scrub.js +48 -0
- package/dist/capture/summarize.js +3 -9
- package/dist/capture/transcript.d.ts +1 -0
- package/dist/capture/transcript.js +12 -3
- package/dist/capture/turn-extractor.d.ts +0 -1
- package/dist/capture/turn-extractor.js +10 -19
- package/dist/cli/commands/adoption.js +3 -3
- package/dist/cli/commands/distill.d.ts +1 -0
- package/dist/cli/commands/distill.js +67 -0
- package/dist/cli/commands/doctor.js +11 -3
- package/dist/cli/commands/insight.js +13 -4
- package/dist/cli/commands/maintain.d.ts +2 -1
- package/dist/cli/commands/maintain.js +53 -17
- package/dist/cli/commands/pick.d.ts +3 -0
- package/dist/cli/commands/pick.js +25 -0
- package/dist/cli/commands/setup.js +11 -2
- package/dist/cli/commands/sync.js +73 -12
- package/dist/cli/commands/transfer.d.ts +0 -5
- package/dist/cli/commands/transfer.js +1 -24
- package/dist/cli/commands/use.js +1 -1
- package/dist/cli/commands/view.js +30 -28
- package/dist/cli/index.js +13 -2
- package/dist/cli/install.d.ts +2 -0
- package/dist/cli/install.js +44 -21
- package/dist/cli/instructions.js +4 -2
- package/dist/cli/refresh.js +5 -1
- package/dist/cli/usage.js +4 -0
- package/dist/config/dir.d.ts +1 -0
- package/dist/config/dir.js +3 -0
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +1 -1
- package/dist/config/load.d.ts +4 -1
- package/dist/config/load.js +30 -13
- package/dist/config/schema.d.ts +19 -0
- package/dist/config/schema.js +17 -2
- package/dist/embed/index.d.ts +1 -1
- package/dist/embed/index.js +1 -1
- package/dist/embed/local.js +2 -1
- package/dist/facts/budget.d.ts +4 -0
- package/dist/facts/budget.js +12 -0
- package/dist/facts/claude-memory.d.ts +19 -0
- package/dist/facts/claude-memory.js +100 -0
- package/dist/facts/distill.d.ts +25 -0
- package/dist/facts/distill.js +141 -0
- package/dist/facts/model.d.ts +28 -0
- package/dist/facts/model.js +82 -0
- package/dist/facts/notice.d.ts +2 -0
- package/dist/facts/notice.js +22 -0
- package/dist/facts/ops.d.ts +23 -0
- package/dist/facts/ops.js +91 -0
- package/dist/facts/render.d.ts +4 -0
- package/dist/facts/render.js +13 -0
- package/dist/facts/start.d.ts +6 -0
- package/dist/facts/start.js +57 -0
- package/dist/graph/index.d.ts +2 -6
- package/dist/graph/index.js +2 -3
- package/dist/graph/languages/basic.js +1 -1
- package/dist/graph/languages/index.d.ts +1 -1
- package/dist/graph/parser.js +1 -1
- package/dist/graph/query/index.d.ts +0 -2
- package/dist/graph/query/index.js +0 -1
- package/dist/graph/scan/extract.js +1 -1
- package/dist/graph/scan/files.js +1 -1
- package/dist/graph/scan/index.d.ts +1 -2
- package/dist/graph/scan/index.js +1 -1
- package/dist/hooks/background.d.ts +3 -0
- package/dist/hooks/background.js +43 -0
- package/dist/hooks/compact.d.ts +8 -0
- package/dist/hooks/compact.js +42 -0
- package/dist/hooks/payload.d.ts +3 -0
- package/dist/hooks/payload.js +18 -1
- package/dist/hooks/pick-deliver.d.ts +2 -0
- package/dist/hooks/pick-deliver.js +16 -0
- package/dist/hooks/prompt-recall.d.ts +21 -0
- package/dist/hooks/prompt-recall.js +36 -0
- package/dist/hooks/relevance.d.ts +11 -3
- package/dist/hooks/relevance.js +56 -26
- package/dist/hooks/session-end.js +7 -1
- package/dist/hooks/session-start.js +31 -59
- package/dist/hooks/shown.d.ts +1 -0
- package/dist/hooks/shown.js +5 -2
- package/dist/hooks/start-legacy.d.ts +6 -0
- package/dist/hooks/start-legacy.js +47 -0
- package/dist/hooks/user-prompt.js +29 -25
- package/dist/mcp/server.js +13 -1
- package/dist/pick/pending.d.ts +17 -0
- package/dist/pick/pending.js +106 -0
- package/dist/pick/prompt.d.ts +14 -0
- package/dist/pick/prompt.js +80 -0
- package/dist/pick/run.d.ts +22 -0
- package/dist/pick/run.js +36 -0
- package/dist/pick/worker.d.ts +2 -0
- package/dist/pick/worker.js +32 -0
- package/dist/store/adapter.d.ts +1 -0
- package/dist/store/each.d.ts +5 -0
- package/dist/store/each.js +25 -0
- package/dist/store/mongo/docs.d.ts +2 -0
- package/dist/store/mongo/docs.js +4 -0
- package/dist/store/mongo/driver.js +3 -2
- package/dist/store/mongo/filters.d.ts +1 -0
- package/dist/store/mongo/filters.js +8 -0
- package/dist/store/mongo/index.d.ts +1 -0
- package/dist/store/mongo/index.js +5 -2
- package/dist/store/mongo/observations.d.ts +2 -1
- package/dist/store/mongo/observations.js +11 -1
- package/dist/store/mongo/search.js +12 -3
- package/dist/store/mongo/sessions.js +7 -1
- package/dist/store/mongo/vectorSearch.js +9 -11
- package/dist/store/postgres/driver.js +3 -2
- package/dist/store/postgres/filters.js +5 -0
- package/dist/store/postgres/index.d.ts +1 -0
- package/dist/store/postgres/index.js +3 -0
- package/dist/store/postgres/meta.js +2 -1
- package/dist/store/postgres/observations.d.ts +1 -0
- package/dist/store/postgres/observations.js +14 -3
- package/dist/store/postgres/rows.js +4 -0
- package/dist/store/postgres/schema.d.ts +2 -2
- package/dist/store/postgres/schema.js +4 -1
- package/dist/store/postgres/sessions.js +14 -7
- package/dist/store/session-time.d.ts +2 -0
- package/dist/store/session-time.js +5 -0
- package/dist/store/sqlite/filters.js +5 -0
- package/dist/store/sqlite/index.d.ts +1 -0
- package/dist/store/sqlite/index.js +23 -4
- package/dist/store/sqlite/observations.d.ts +1 -0
- package/dist/store/sqlite/observations.js +14 -1
- package/dist/store/sqlite/rows.js +4 -0
- package/dist/store/sqlite/schema.sql +3 -1
- package/dist/store/sqlite/search.js +7 -3
- package/dist/store/sqlite/sessions.js +13 -8
- package/dist/types.d.ts +5 -1
- package/dist/usages/find.js +5 -3
- package/dist/usages/index.d.ts +0 -1
- package/dist/util/claude-cli.d.ts +6 -0
- package/dist/util/claude-cli.js +38 -0
- package/dist/util/daily-budget.d.ts +7 -0
- package/dist/util/daily-budget.js +38 -0
- package/dist/util/day.d.ts +1 -0
- package/dist/util/day.js +11 -0
- package/dist/util/job-lock.d.ts +4 -0
- package/dist/util/job-lock.js +56 -0
- package/dist/util/prompt.d.ts +5 -0
- package/dist/util/prompt.js +31 -0
- package/docs/how-it-works.md +69 -3
- package/docs/setup-guide.md +339 -0
- package/docs/with-and-without.md +43 -36
- package/package.json +23 -7
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { basename } from 'node:path';
|
|
2
|
+
import { claimJob, finishJob, jobMark, releaseJob } from '../util/job-lock.js';
|
|
3
|
+
import { flushSession, resetCursor } from './flush.js';
|
|
4
|
+
import { readTranscript, transcriptsFor } from './transcript.js';
|
|
5
|
+
import { observationsFromTurns } from './turn-extractor.js';
|
|
6
|
+
export const REINGEST_VERSION = 1;
|
|
7
|
+
const SESSION_ROWS = 10_000;
|
|
8
|
+
export async function reingestSession(ctx, project, path) {
|
|
9
|
+
const sessionId = basename(path, '.jsonl');
|
|
10
|
+
const { turns, lastTimestamp } = readTranscript(path, 0);
|
|
11
|
+
resetCursor(sessionId);
|
|
12
|
+
const { observations: saved } = await flushSession(ctx, sessionId, project, path, true);
|
|
13
|
+
if (lastTimestamp === 0)
|
|
14
|
+
return { sessionId, saved, replaced: 0 };
|
|
15
|
+
const current = new Set(observationsFromTurns(turns, sessionId, project, ctx.config).map((obs) => obs.id));
|
|
16
|
+
const stored = await ctx.store.list({ project, sessionId, limit: SESSION_ROWS });
|
|
17
|
+
const stale = stored
|
|
18
|
+
.filter((obs) => obs.status !== 'replaced' && obs.createdAt <= lastTimestamp)
|
|
19
|
+
.filter((obs) => !current.has(obs.id))
|
|
20
|
+
.map((obs) => obs.id);
|
|
21
|
+
return { sessionId, saved, replaced: await ctx.store.markReplaced(stale) };
|
|
22
|
+
}
|
|
23
|
+
export async function rememberedTranscripts(ctx, project, paths = transcriptsFor(project)) {
|
|
24
|
+
const kept = [];
|
|
25
|
+
for (const path of paths) {
|
|
26
|
+
const sessionId = basename(path, '.jsonl');
|
|
27
|
+
if ((await ctx.store.list({ project, sessionId, limit: 1 })).length > 0)
|
|
28
|
+
kept.push(path);
|
|
29
|
+
}
|
|
30
|
+
return kept;
|
|
31
|
+
}
|
|
32
|
+
export async function reingestProject(ctx, project, paths = transcriptsFor(project), onSession = () => { }) {
|
|
33
|
+
const total = { transcripts: paths.length, saved: 0, replaced: 0 };
|
|
34
|
+
for (const path of paths) {
|
|
35
|
+
const result = await reingestSession(ctx, project, path);
|
|
36
|
+
total.saved += result.saved;
|
|
37
|
+
total.replaced += result.replaced;
|
|
38
|
+
onSession(result);
|
|
39
|
+
}
|
|
40
|
+
return total;
|
|
41
|
+
}
|
|
42
|
+
const JOB = 'reingest';
|
|
43
|
+
export function reingestDone(project) {
|
|
44
|
+
return jobMark(JOB, project) >= REINGEST_VERSION;
|
|
45
|
+
}
|
|
46
|
+
export function claimReingest(project, now = Date.now()) {
|
|
47
|
+
if (reingestDone(project))
|
|
48
|
+
return false;
|
|
49
|
+
return claimJob(JOB, project, now);
|
|
50
|
+
}
|
|
51
|
+
export function finishReingest(project) {
|
|
52
|
+
finishJob(JOB, project, REINGEST_VERSION);
|
|
53
|
+
}
|
|
54
|
+
export function releaseReingest(project) {
|
|
55
|
+
releaseJob(JOB, project);
|
|
56
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { RecallContext } from '../context.js';
|
|
2
|
+
export declare const REDACT_VERSION = 1;
|
|
3
|
+
export interface ScrubResult {
|
|
4
|
+
observations: number;
|
|
5
|
+
sessions: number;
|
|
6
|
+
}
|
|
7
|
+
export declare function scrubSecrets(ctx: RecallContext, now?: number): Promise<ScrubResult>;
|
|
8
|
+
export declare function scrubDone(database: string): boolean;
|
|
9
|
+
export declare function claimScrub(database: string, now?: number): boolean;
|
|
10
|
+
export declare function finishScrub(database: string): void;
|
|
11
|
+
export declare function releaseScrub(database: string): void;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { eachObservation } from '../store/each.js';
|
|
2
|
+
import { claimJob, finishJob, jobMark, releaseJob } from '../util/job-lock.js';
|
|
3
|
+
import { embedObservations } from './flush.js';
|
|
4
|
+
import { redact } from './redact.js';
|
|
5
|
+
export const REDACT_VERSION = 1;
|
|
6
|
+
const JOB = 'redact';
|
|
7
|
+
const SESSION_LIMIT = 100_000;
|
|
8
|
+
function redacted(obs) {
|
|
9
|
+
const title = redact(obs.title);
|
|
10
|
+
const body = redact(obs.body);
|
|
11
|
+
return title === obs.title && body === obs.body ? null : { ...obs, title, body };
|
|
12
|
+
}
|
|
13
|
+
export async function scrubSecrets(ctx, now = Date.now()) {
|
|
14
|
+
let observations = 0;
|
|
15
|
+
await eachObservation(ctx, {}, async (batch) => {
|
|
16
|
+
const changed = batch.map(redacted).filter((obs) => obs !== null);
|
|
17
|
+
if (changed.length === 0)
|
|
18
|
+
return;
|
|
19
|
+
await embedObservations(ctx, changed);
|
|
20
|
+
await ctx.store.insertObservations(changed);
|
|
21
|
+
observations += changed.length;
|
|
22
|
+
});
|
|
23
|
+
let sessions = 0;
|
|
24
|
+
for (const { project } of await ctx.store.listProjects()) {
|
|
25
|
+
for (const session of await ctx.store.recentSessions(project, SESSION_LIMIT)) {
|
|
26
|
+
if (!session.summary)
|
|
27
|
+
continue;
|
|
28
|
+
const summary = redact(session.summary);
|
|
29
|
+
if (summary === session.summary)
|
|
30
|
+
continue;
|
|
31
|
+
await ctx.store.upsertSession({ ...session, summary, updatedAt: now });
|
|
32
|
+
sessions += 1;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return { observations, sessions };
|
|
36
|
+
}
|
|
37
|
+
export function scrubDone(database) {
|
|
38
|
+
return jobMark(JOB, database) >= REDACT_VERSION;
|
|
39
|
+
}
|
|
40
|
+
export function claimScrub(database, now = Date.now()) {
|
|
41
|
+
return !scrubDone(database) && claimJob(JOB, database, now);
|
|
42
|
+
}
|
|
43
|
+
export function finishScrub(database) {
|
|
44
|
+
finishJob(JOB, database, REDACT_VERSION);
|
|
45
|
+
}
|
|
46
|
+
export function releaseScrub(database) {
|
|
47
|
+
releaseJob(JOB, database);
|
|
48
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { runHeadless } from '../util/claude-cli.js';
|
|
2
2
|
const MAX_INPUT_CHARS = 8000;
|
|
3
3
|
const MAX_OUTPUT_CHARS = 800;
|
|
4
4
|
const TIMEOUT_MS = 45_000;
|
|
@@ -21,12 +21,6 @@ export function validateSummary(raw) {
|
|
|
21
21
|
export async function aiSummary(sessionText, model) {
|
|
22
22
|
if (!sessionText.trim())
|
|
23
23
|
return null;
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
execFile('claude', ['-p', buildSummaryPrompt(sessionText), '--model', model, '--effort', 'low'], { timeout: TIMEOUT_MS, maxBuffer: 1024 * 1024 }, (error, stdout) => done(error ? null : validateSummary(stdout)));
|
|
27
|
-
}
|
|
28
|
-
catch {
|
|
29
|
-
done(null);
|
|
30
|
-
}
|
|
31
|
-
});
|
|
24
|
+
const stdout = await runHeadless(buildSummaryPrompt(sessionText), model, TIMEOUT_MS);
|
|
25
|
+
return stdout === null ? null : validateSummary(stdout);
|
|
32
26
|
}
|
|
@@ -9,6 +9,7 @@ export interface Turn {
|
|
|
9
9
|
export interface TranscriptRead {
|
|
10
10
|
turns: Turn[];
|
|
11
11
|
nextOffset: number;
|
|
12
|
+
lastTimestamp: number;
|
|
12
13
|
}
|
|
13
14
|
export declare function readTranscript(path: string, fromOffset?: number): TranscriptRead;
|
|
14
15
|
export declare function transcriptPathFor(project: string, sessionId: string): string;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { closeSync, fstatSync, openSync, readdirSync, readSync } from 'node:fs';
|
|
2
2
|
import { homedir } from 'node:os';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
|
+
import { isRelayedMessage } from '../util/prompt.js';
|
|
4
5
|
import { resolveProject } from '../util/project.js';
|
|
5
6
|
export function readTranscript(path, fromOffset = 0) {
|
|
6
7
|
let fd;
|
|
@@ -8,14 +9,14 @@ export function readTranscript(path, fromOffset = 0) {
|
|
|
8
9
|
fd = openSync(path, 'r');
|
|
9
10
|
}
|
|
10
11
|
catch {
|
|
11
|
-
return { turns: [], nextOffset: fromOffset };
|
|
12
|
+
return { turns: [], nextOffset: fromOffset, lastTimestamp: 0 };
|
|
12
13
|
}
|
|
13
14
|
try {
|
|
14
15
|
const size = fstatSync(fd).size;
|
|
15
16
|
const start = fromOffset > size ? 0 : fromOffset;
|
|
16
17
|
const length = size - start;
|
|
17
18
|
if (length <= 0)
|
|
18
|
-
return { turns: [], nextOffset: size };
|
|
19
|
+
return { turns: [], nextOffset: size, lastTimestamp: 0 };
|
|
19
20
|
const buffer = Buffer.allocUnsafe(length);
|
|
20
21
|
const bytesRead = readSync(fd, buffer, 0, length, start);
|
|
21
22
|
const raw = buffer.subarray(0, bytesRead).toString('utf8');
|
|
@@ -35,7 +36,11 @@ export function readTranscript(path, fromOffset = 0) {
|
|
|
35
36
|
const complete = lastNewline < 0 ? start : start + Buffer.byteLength(raw.slice(0, lastNewline + 1), 'utf8');
|
|
36
37
|
const turns = groupIntoTurns(entries);
|
|
37
38
|
const nextOffset = turns.length > 0 ? (turns[turns.length - 1]?.offset ?? complete) : complete;
|
|
38
|
-
|
|
39
|
+
const lastTimestamp = entries.reduce((latest, { entry }) => {
|
|
40
|
+
const at = Date.parse(entry.timestamp ?? '');
|
|
41
|
+
return Number.isNaN(at) ? latest : Math.max(latest, at);
|
|
42
|
+
}, 0);
|
|
43
|
+
return { turns, nextOffset, lastTimestamp };
|
|
39
44
|
}
|
|
40
45
|
finally {
|
|
41
46
|
closeSync(fd);
|
|
@@ -51,6 +56,8 @@ function groupIntoTurns(entries) {
|
|
|
51
56
|
if (!Number.isNaN(parsed))
|
|
52
57
|
lastKnown = parsed;
|
|
53
58
|
if (entry.type === 'user') {
|
|
59
|
+
if (entry.isMeta || entry.origin?.kind === 'task-notification')
|
|
60
|
+
continue;
|
|
54
61
|
const text = extractText(entry.message?.content);
|
|
55
62
|
if (!text || entry.isCompactSummary || isSyntheticPrompt(text))
|
|
56
63
|
continue;
|
|
@@ -114,6 +121,8 @@ function isSyntheticPrompt(text) {
|
|
|
114
121
|
return (text.includes('<system-reminder>') ||
|
|
115
122
|
text.includes('<project-memory>') ||
|
|
116
123
|
text.includes('<recalled-memory>') ||
|
|
124
|
+
text.startsWith('<memory') ||
|
|
125
|
+
isRelayedMessage(text) ||
|
|
117
126
|
text.startsWith('Caveat:') ||
|
|
118
127
|
text.startsWith('[Request interrupted') ||
|
|
119
128
|
text.startsWith('<local-command'));
|
|
@@ -4,4 +4,3 @@ import type { Turn } from './transcript.js';
|
|
|
4
4
|
export declare function observationsFromTurns(turns: Turn[], sessionId: string, project: string, config: Config): Observation[];
|
|
5
5
|
export declare function topLevelDirs(files: string[], project: string): string[];
|
|
6
6
|
export declare function classifyTurn(turn: Turn): ObservationKind;
|
|
7
|
-
export declare function redact(text: string): string;
|
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import { classifyCommand } from './command.js';
|
|
2
|
-
import { isSearchable } from '../util/prompt.js';
|
|
2
|
+
import { isSearchable, readablePrompt } from '../util/prompt.js';
|
|
3
3
|
import { currentAuthor, observationId } from './identity.js';
|
|
4
|
+
import { redact } from './redact.js';
|
|
4
5
|
export function observationsFromTurns(turns, sessionId, project, config) {
|
|
5
6
|
return turns
|
|
6
7
|
.map(withoutPrivate)
|
|
7
|
-
.
|
|
8
|
-
|
|
8
|
+
.map((turn) => ({
|
|
9
|
+
id: observationId(sessionId, turn.timestamp, turn.prompt),
|
|
10
|
+
turn: { ...turn, prompt: readablePrompt(turn.prompt) },
|
|
11
|
+
}))
|
|
12
|
+
.filter(({ turn }) => isSubstantive(turn, config))
|
|
13
|
+
.map(({ id, turn }) => buildObservation(turn, id, sessionId, project, config));
|
|
9
14
|
}
|
|
10
15
|
function withoutPrivate(turn) {
|
|
11
16
|
const strip = (text) => text.replace(/<private>[\s\S]*?(?:<\/private>|$)/gi, '[private]');
|
|
@@ -22,7 +27,7 @@ function answers(reasoning) {
|
|
|
22
27
|
const scored = scoreSentences(stripMarkdown(redact(reasoning)));
|
|
23
28
|
return scored.length >= 2 && scored.some(([, score]) => score > 0);
|
|
24
29
|
}
|
|
25
|
-
function buildObservation(turn, sessionId, project, config) {
|
|
30
|
+
function buildObservation(turn, id, sessionId, project, config) {
|
|
26
31
|
const files = [...new Set(turn.files.filter((f) => !isExcluded(f, config.capture.exclude)))];
|
|
27
32
|
const commands = turn.commands
|
|
28
33
|
.map(classifyCommand)
|
|
@@ -39,7 +44,7 @@ function buildObservation(turn, sessionId, project, config) {
|
|
|
39
44
|
const author = currentAuthor();
|
|
40
45
|
const kind = classifyTurn(turn);
|
|
41
46
|
return {
|
|
42
|
-
id
|
|
47
|
+
id,
|
|
43
48
|
sessionId,
|
|
44
49
|
project,
|
|
45
50
|
...(author ? { author } : {}),
|
|
@@ -165,20 +170,6 @@ function shortPath(path) {
|
|
|
165
170
|
function isExcluded(file, patterns) {
|
|
166
171
|
return patterns.some((pattern) => file.includes(pattern));
|
|
167
172
|
}
|
|
168
|
-
const PRIVATE_BLOCK = /<private>[\s\S]*?(?:<\/private>|$)/gi;
|
|
169
|
-
const PRIVATE_KEY = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g;
|
|
170
|
-
export function redact(text) {
|
|
171
|
-
return text
|
|
172
|
-
.replace(PRIVATE_BLOCK, '[redacted]')
|
|
173
|
-
.replace(PRIVATE_KEY, '[redacted-private-key]')
|
|
174
|
-
.replace(/\/\/[^\s:/@]+:[^\s@]+@/g, '//[redacted]@')
|
|
175
|
-
.replace(/\b(sk-[A-Za-z0-9_-]{16,})\b/g, '[redacted-key]')
|
|
176
|
-
.replace(/\b(gh[pousr]_[A-Za-z0-9]{16,})\b/g, '[redacted-token]')
|
|
177
|
-
.replace(/\bAKIA[0-9A-Z]{16}\b/g, '[redacted-key]')
|
|
178
|
-
.replace(/\bxox[baprs]-[A-Za-z0-9-]{10,}/g, '[redacted-token]')
|
|
179
|
-
.replace(/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g, '[redacted-jwt]')
|
|
180
|
-
.replace(/(?<=(password|secret|token|api[_-]?key)"?\s*[:=]\s*)"[^"]+"/gi, '"[redacted]"');
|
|
181
|
-
}
|
|
182
173
|
function truncate(text, max) {
|
|
183
174
|
return text.length <= max ? text : `${text.slice(0, max)}\n... [truncated]`;
|
|
184
175
|
}
|
|
@@ -33,9 +33,9 @@ export async function cmdAdoption() {
|
|
|
33
33
|
continue;
|
|
34
34
|
seen.add(item.id);
|
|
35
35
|
const name = typeof item.name === 'string' ? item.name : '';
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
36
|
+
const command = item.input?.command;
|
|
37
|
+
if (name === 'Bash' && typeof command === 'string')
|
|
38
|
+
bash.push(command);
|
|
39
39
|
if (name.startsWith('mcp__memory__')) {
|
|
40
40
|
const tool = name.slice('mcp__memory__'.length);
|
|
41
41
|
memory.set(tool, (memory.get(tool) ?? 0) + 1);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function cmdDistill(argv: (string | undefined)[]): Promise<void>;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { setConfigValue } from '../../config/index.js';
|
|
2
|
+
import { createContext } from '../../context.js';
|
|
3
|
+
import { distillUsage } from '../../facts/budget.js';
|
|
4
|
+
import { importClaudeMemory } from '../../facts/claude-memory.js';
|
|
5
|
+
import { FACTS_JOB, backfill, distillSession, factCounts, pendingSessions, } from '../../facts/distill.js';
|
|
6
|
+
import { finishJob, releaseJob } from '../../util/job-lock.js';
|
|
7
|
+
import { resolveProject } from '../../util/project.js';
|
|
8
|
+
export async function cmdDistill(argv) {
|
|
9
|
+
const [first] = argv;
|
|
10
|
+
if (first === 'on' || first === 'off') {
|
|
11
|
+
setConfigValue('distill', 'enabled', first === 'on');
|
|
12
|
+
console.log(first === 'on'
|
|
13
|
+
? 'Chats will be turned into facts after they end, with one small Haiku call each.'
|
|
14
|
+
: 'Turning chats into facts is off. Chats are still recorded; nothing calls Claude.');
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
const project = resolveProject(undefined);
|
|
18
|
+
if (first === '--backfill') {
|
|
19
|
+
await runBackfill(project);
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
const ctx = await createContext();
|
|
23
|
+
try {
|
|
24
|
+
const session = argv[argv.indexOf('--session') + 1];
|
|
25
|
+
if (first === '--session' && session) {
|
|
26
|
+
if (ctx.config.distill.enabled)
|
|
27
|
+
await distillSession(ctx, project, session);
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
await printStatus(ctx, project);
|
|
31
|
+
}
|
|
32
|
+
finally {
|
|
33
|
+
await ctx.close();
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
async function runBackfill(project) {
|
|
37
|
+
try {
|
|
38
|
+
const ctx = await createContext();
|
|
39
|
+
try {
|
|
40
|
+
await importClaudeMemory(ctx, project);
|
|
41
|
+
if (ctx.config.distill.enabled)
|
|
42
|
+
await backfill(ctx, project);
|
|
43
|
+
}
|
|
44
|
+
finally {
|
|
45
|
+
await ctx.close();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
releaseJob(FACTS_JOB, project);
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
finishJob(FACTS_JOB, project, Date.now());
|
|
53
|
+
}
|
|
54
|
+
async function printStatus(ctx, project) {
|
|
55
|
+
const { distill } = ctx.config;
|
|
56
|
+
const usage = distillUsage();
|
|
57
|
+
const facts = await factCounts(ctx, project);
|
|
58
|
+
console.log(`distill : ${distill.enabled ? 'on' : 'off'} (${distill.model})`);
|
|
59
|
+
console.log(`today : ${usage.used} of ${distill.dailyLimit} calls used`);
|
|
60
|
+
if (usage.pausedUntil > Date.now()) {
|
|
61
|
+
console.log(`paused : until ${new Date(usage.pausedUntil).toISOString()} after a failed call`);
|
|
62
|
+
}
|
|
63
|
+
console.log(`facts : ${facts.project} for this project, ${facts.you} about you`);
|
|
64
|
+
console.log(`waiting : ${(await pendingSessions(ctx, project)).length} chat(s) not yet turned into facts`);
|
|
65
|
+
if (!distill.enabled)
|
|
66
|
+
console.log('\nTurn it on with: claude-db distill on');
|
|
67
|
+
}
|
|
@@ -4,6 +4,7 @@ import { packageVersion } from '../../update.js';
|
|
|
4
4
|
import { randomUUID } from 'node:crypto';
|
|
5
5
|
import { redact, remember } from '../../capture/index.js';
|
|
6
6
|
import { resolveProject } from '../../util/project.js';
|
|
7
|
+
import { ourHookFile } from '../install.js';
|
|
7
8
|
import { settingsPathFor } from '../paths.js';
|
|
8
9
|
import { existsSync, readFileSync } from 'node:fs';
|
|
9
10
|
import { toShortId } from '../../util/shortid.js';
|
|
@@ -46,9 +47,16 @@ function checkWiring(project) {
|
|
|
46
47
|
for (const [event, entries] of Object.entries(settings.hooks ?? {})) {
|
|
47
48
|
const ours = entries
|
|
48
49
|
.flatMap((entry) => entry.hooks.map((hook) => hook.command))
|
|
49
|
-
.filter((command) =>
|
|
50
|
-
|
|
51
|
-
|
|
50
|
+
.filter((command) => ourHookFile(command) !== null);
|
|
51
|
+
const counts = new Map();
|
|
52
|
+
for (const command of ours) {
|
|
53
|
+
const file = ourHookFile(command) ?? '';
|
|
54
|
+
counts.set(file, (counts.get(file) ?? 0) + 1);
|
|
55
|
+
}
|
|
56
|
+
for (const [file, count] of counts) {
|
|
57
|
+
if (count > 1)
|
|
58
|
+
problems.push(`${event} ${file} registered ${count}x in ${path}`);
|
|
59
|
+
}
|
|
52
60
|
for (const command of ours) {
|
|
53
61
|
const file = command.replace(/^node\s+/, '');
|
|
54
62
|
if (!existsSync(file))
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { createContext } from '../../context.js';
|
|
2
|
-
import { eachObservation } from '
|
|
2
|
+
import { eachObservation } from '../../store/each.js';
|
|
3
3
|
import { resolve } from 'node:path';
|
|
4
4
|
import { resolveProject } from '../../util/project.js';
|
|
5
|
-
import { redact } from '../../capture/
|
|
5
|
+
import { redact } from '../../capture/redact.js';
|
|
6
6
|
export async function cmdStats() {
|
|
7
7
|
const project = resolveProject(undefined);
|
|
8
8
|
const ctx = await createContext();
|
|
@@ -10,10 +10,15 @@ export async function cmdStats() {
|
|
|
10
10
|
const kinds = new Map();
|
|
11
11
|
const tags = new Map();
|
|
12
12
|
let embedded = 0;
|
|
13
|
+
let replaced = 0;
|
|
13
14
|
let earliest = Number.POSITIVE_INFINITY;
|
|
14
15
|
let latest = 0;
|
|
15
|
-
const
|
|
16
|
+
const stored = await eachObservation(ctx, { project }, (batch) => {
|
|
16
17
|
for (const obs of batch) {
|
|
18
|
+
if (obs.status === 'replaced') {
|
|
19
|
+
replaced += 1;
|
|
20
|
+
continue;
|
|
21
|
+
}
|
|
17
22
|
kinds.set(obs.kind, (kinds.get(obs.kind) ?? 0) + 1);
|
|
18
23
|
for (const tag of obs.tags)
|
|
19
24
|
tags.set(tag, (tags.get(tag) ?? 0) + 1);
|
|
@@ -23,13 +28,17 @@ export async function cmdStats() {
|
|
|
23
28
|
latest = Math.max(latest, obs.createdAt);
|
|
24
29
|
}
|
|
25
30
|
});
|
|
31
|
+
const total = stored - replaced;
|
|
26
32
|
if (total === 0) {
|
|
27
|
-
|
|
33
|
+
const note = replaced > 0 ? ` (${replaced} replaced, kept but left out of search)` : '';
|
|
34
|
+
console.log(`No memory stored for ${project}${note}.`);
|
|
28
35
|
return;
|
|
29
36
|
}
|
|
30
37
|
const day = (ms) => new Date(ms).toISOString().slice(0, 10);
|
|
31
38
|
console.log(`project : ${project}`);
|
|
32
39
|
console.log(`observations: ${total}`);
|
|
40
|
+
if (replaced > 0)
|
|
41
|
+
console.log(`replaced : ${replaced}, kept but left out of search`);
|
|
33
42
|
console.log(`range : ${day(earliest)} to ${day(latest)}`);
|
|
34
43
|
console.log(`embedded : ${embedded} of ${total}`);
|
|
35
44
|
console.log('\nby kind');
|
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
export declare function cmdReset(argv: (string | undefined)[]): Promise<void>;
|
|
2
|
-
export declare function cmdFlush(): Promise<void>;
|
|
2
|
+
export declare function cmdFlush(argv: (string | undefined)[]): Promise<void>;
|
|
3
3
|
export declare function cmdUpdate(argv: (string | undefined)[]): Promise<void>;
|
|
4
|
+
export declare function cmdRedact(argv: (string | undefined)[]): Promise<void>;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { CONFIG_DIR, loadConfig } from '../../config/index.js';
|
|
2
|
-
import {
|
|
2
|
+
import { join } from 'node:path';
|
|
3
3
|
import { checkForUpdate } from '../../update.js';
|
|
4
4
|
import { createContext } from '../../context.js';
|
|
5
|
-
import {
|
|
5
|
+
import { finishReingest, finishScrub, redact, reingestProject, releaseReingest, releaseScrub, rememberedTranscripts, scrubSecrets, sweepCursors, transcriptsFor, } from '../../capture/index.js';
|
|
6
6
|
import { resolveProject } from '../../util/project.js';
|
|
7
7
|
import { rmSync } from 'node:fs';
|
|
8
8
|
export async function cmdReset(argv) {
|
|
@@ -32,34 +32,48 @@ export async function cmdReset(argv) {
|
|
|
32
32
|
function clearLocalState() {
|
|
33
33
|
rmSync(join(CONFIG_DIR, 'cursors'), { recursive: true, force: true });
|
|
34
34
|
}
|
|
35
|
-
export async function cmdFlush() {
|
|
35
|
+
export async function cmdFlush(argv) {
|
|
36
36
|
const project = resolveProject(undefined);
|
|
37
|
+
const repair = argv.includes('--repair');
|
|
37
38
|
const transcripts = transcriptsFor(project);
|
|
38
39
|
if (transcripts.length === 0) {
|
|
40
|
+
finishReingest(project);
|
|
39
41
|
console.error(`No transcripts found for ${project}`);
|
|
40
42
|
process.exit(1);
|
|
41
43
|
}
|
|
42
|
-
|
|
43
|
-
let total = 0;
|
|
44
|
+
let total;
|
|
44
45
|
try {
|
|
45
|
-
|
|
46
|
-
const sessionId = basename(path, '.jsonl');
|
|
47
|
-
resetCursor(sessionId);
|
|
48
|
-
const result = await flushSession(ctx, sessionId, project, path, true);
|
|
49
|
-
if (result.observations > 0) {
|
|
50
|
-
console.log(`${sessionId.slice(0, 8)} ${String(result.observations).padStart(4)} observations`);
|
|
51
|
-
total += result.observations;
|
|
52
|
-
}
|
|
53
|
-
}
|
|
46
|
+
total = await reingest(project, transcripts, repair);
|
|
54
47
|
}
|
|
55
|
-
|
|
56
|
-
|
|
48
|
+
catch (error) {
|
|
49
|
+
releaseReingest(project);
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
finishReingest(project);
|
|
53
|
+
console.log(`\n${total.saved} observations from ${total.transcripts} transcript(s).`);
|
|
54
|
+
if (total.replaced > 0) {
|
|
55
|
+
console.log(`${total.replaced} older observation(s) saved under the previous capture rules were ` +
|
|
56
|
+
'marked replaced. They stay in the database but no longer appear in search.');
|
|
57
57
|
}
|
|
58
|
-
console.log(`\n${total} observations from ${transcripts.length} transcript(s).`);
|
|
59
58
|
const swept = sweepCursors();
|
|
60
59
|
if (swept > 0)
|
|
61
60
|
console.log(`Swept ${swept} cursor(s) for transcripts that no longer exist.`);
|
|
62
61
|
}
|
|
62
|
+
async function reingest(project, transcripts, repair) {
|
|
63
|
+
const ctx = await createContext();
|
|
64
|
+
try {
|
|
65
|
+
const chosen = repair ? await rememberedTranscripts(ctx, project, transcripts) : transcripts;
|
|
66
|
+
return await reingestProject(ctx, project, chosen, (result) => {
|
|
67
|
+
if (result.saved === 0 && result.replaced === 0)
|
|
68
|
+
return;
|
|
69
|
+
const replaced = result.replaced > 0 ? `, ${result.replaced} replaced` : '';
|
|
70
|
+
console.log(`${result.sessionId.slice(0, 8)} ${String(result.saved).padStart(4)} observations${replaced}`);
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
await ctx.close();
|
|
75
|
+
}
|
|
76
|
+
}
|
|
63
77
|
export async function cmdUpdate(argv) {
|
|
64
78
|
const quiet = argv.includes('--quiet');
|
|
65
79
|
const config = loadConfig();
|
|
@@ -75,3 +89,25 @@ export async function cmdUpdate(argv) {
|
|
|
75
89
|
else
|
|
76
90
|
console.log(`Up to date (${result.current}).`);
|
|
77
91
|
}
|
|
92
|
+
export async function cmdRedact(argv) {
|
|
93
|
+
const background = argv.includes('--background');
|
|
94
|
+
const ctx = await createContext();
|
|
95
|
+
const database = ctx.config.database;
|
|
96
|
+
let result;
|
|
97
|
+
try {
|
|
98
|
+
result = await scrubSecrets(ctx);
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
releaseScrub(database);
|
|
102
|
+
throw error;
|
|
103
|
+
}
|
|
104
|
+
finally {
|
|
105
|
+
await ctx.close();
|
|
106
|
+
}
|
|
107
|
+
finishScrub(database);
|
|
108
|
+
if (background)
|
|
109
|
+
return;
|
|
110
|
+
console.log(result.observations + result.sessions === 0
|
|
111
|
+
? 'Nothing to redact: no saved memory holds a secret the current rules catch.'
|
|
112
|
+
: `Redacted ${result.observations} memory row(s) and ${result.sessions} chat summary(ies).`);
|
|
113
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { loadConfig, setConfigValue } from '../../config/index.js';
|
|
2
|
+
import { PICK_BUDGET } from '../../pick/run.js';
|
|
3
|
+
import { budgetUsage } from '../../util/daily-budget.js';
|
|
4
|
+
export function pickStatus(config, now = Date.now()) {
|
|
5
|
+
const { pick } = config;
|
|
6
|
+
if (!pick.enabled) {
|
|
7
|
+
return 'pick : off, prompts get memory only on a strong word match (claude-db pick on)';
|
|
8
|
+
}
|
|
9
|
+
const usage = budgetUsage(PICK_BUDGET, now);
|
|
10
|
+
const paused = usage.pausedUntil > now
|
|
11
|
+
? `, paused until ${new Date(usage.pausedUntil).toISOString()} after a failed call`
|
|
12
|
+
: '';
|
|
13
|
+
return `pick : on (${pick.model}), ${usage.used} of ${pick.dailyLimit} picks used today${paused}`;
|
|
14
|
+
}
|
|
15
|
+
export function cmdPick(argv) {
|
|
16
|
+
const [first] = argv;
|
|
17
|
+
if (first === 'on' || first === 'off') {
|
|
18
|
+
setConfigValue('pick', 'enabled', first === 'on');
|
|
19
|
+
console.log(first === 'on'
|
|
20
|
+
? 'Haiku will pick the memory shown with each prompt, up to the daily limit.'
|
|
21
|
+
: 'Haiku picking is off. Prompts get memory only on a strong word match; nothing calls Claude.');
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
console.log(pickStatus(loadConfig()));
|
|
25
|
+
}
|
|
@@ -10,6 +10,8 @@ import { readFileSync, statSync } from 'node:fs';
|
|
|
10
10
|
import { resolveProject } from '../../util/project.js';
|
|
11
11
|
import { mentionsPath } from '../../util/paths.js';
|
|
12
12
|
import { transcriptsFor } from '../../capture/index.js';
|
|
13
|
+
import { factCounts, pendingSessions } from '../../facts/distill.js';
|
|
14
|
+
import { pickStatus } from './pick.js';
|
|
13
15
|
export async function cmdInstall(scope) {
|
|
14
16
|
const project = resolveProject(undefined);
|
|
15
17
|
try {
|
|
@@ -51,8 +53,9 @@ function warnIfCommittable(project) {
|
|
|
51
53
|
.split('\n')
|
|
52
54
|
.some((line) => line.trim().replace(/^\//, '') === '.mcp.json');
|
|
53
55
|
}
|
|
54
|
-
catch {
|
|
55
|
-
|
|
56
|
+
catch (error) {
|
|
57
|
+
if (error.code !== 'ENOENT')
|
|
58
|
+
return;
|
|
56
59
|
}
|
|
57
60
|
if (ignored)
|
|
58
61
|
return;
|
|
@@ -94,6 +97,12 @@ export async function cmdStatus() {
|
|
|
94
97
|
const worked = workedAt(project);
|
|
95
98
|
console.log(`worked : ${worked.last > 0 ? ago(worked.last) : 'never'}`);
|
|
96
99
|
console.log(`recorded : ${lastSaved > 0 ? ago(lastSaved) : 'never'}`);
|
|
100
|
+
const facts = await factCounts(ctx, project);
|
|
101
|
+
console.log(`facts : ${facts.project} for this project, ${facts.you} about you`);
|
|
102
|
+
console.log(ctx.config.distill.enabled
|
|
103
|
+
? `waiting : ${(await pendingSessions(ctx, project)).length} chat(s) not yet turned into facts`
|
|
104
|
+
: 'distill : off, so chats are not turned into facts (claude-db distill on)');
|
|
105
|
+
console.log(pickStatus(ctx.config));
|
|
97
106
|
await closeLandedWork(ctx.store, project);
|
|
98
107
|
const open = await openWork(ctx.store, project);
|
|
99
108
|
if (open.length > 0) {
|