claude-db 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. package/CHANGELOG.md +252 -0
  2. package/README.md +74 -113
  3. package/dist/capture/flush.d.ts +1 -0
  4. package/dist/capture/flush.js +4 -3
  5. package/dist/capture/git.js +2 -1
  6. package/dist/capture/index.d.ts +5 -1
  7. package/dist/capture/index.js +4 -1
  8. package/dist/capture/manual.js +1 -1
  9. package/dist/capture/progress.js +1 -0
  10. package/dist/capture/redact.d.ts +1 -0
  11. package/dist/capture/redact.js +17 -0
  12. package/dist/capture/reingest.d.ts +19 -0
  13. package/dist/capture/reingest.js +56 -0
  14. package/dist/capture/scrub.d.ts +11 -0
  15. package/dist/capture/scrub.js +48 -0
  16. package/dist/capture/summarize.js +3 -9
  17. package/dist/capture/transcript.d.ts +1 -0
  18. package/dist/capture/transcript.js +12 -3
  19. package/dist/capture/turn-extractor.d.ts +0 -1
  20. package/dist/capture/turn-extractor.js +10 -19
  21. package/dist/cli/commands/adoption.js +3 -3
  22. package/dist/cli/commands/distill.d.ts +1 -0
  23. package/dist/cli/commands/distill.js +67 -0
  24. package/dist/cli/commands/doctor.js +13 -5
  25. package/dist/cli/commands/insight.js +14 -4
  26. package/dist/cli/commands/maintain.d.ts +2 -1
  27. package/dist/cli/commands/maintain.js +54 -18
  28. package/dist/cli/commands/pick.d.ts +3 -0
  29. package/dist/cli/commands/pick.js +25 -0
  30. package/dist/cli/commands/setup.js +14 -5
  31. package/dist/cli/commands/sync.js +73 -12
  32. package/dist/cli/commands/transfer.d.ts +0 -5
  33. package/dist/cli/commands/transfer.js +1 -24
  34. package/dist/cli/commands/use.js +1 -1
  35. package/dist/cli/commands/view.js +31 -28
  36. package/dist/cli/index.js +13 -2
  37. package/dist/cli/install.d.ts +2 -0
  38. package/dist/cli/install.js +44 -21
  39. package/dist/cli/instructions.js +4 -2
  40. package/dist/cli/refresh.js +5 -1
  41. package/dist/cli/usage.js +4 -0
  42. package/dist/config/dir.d.ts +1 -0
  43. package/dist/config/dir.js +3 -0
  44. package/dist/config/index.d.ts +1 -1
  45. package/dist/config/index.js +1 -1
  46. package/dist/config/load.d.ts +4 -1
  47. package/dist/config/load.js +30 -13
  48. package/dist/config/schema.d.ts +19 -0
  49. package/dist/config/schema.js +17 -2
  50. package/dist/embed/index.d.ts +1 -1
  51. package/dist/embed/index.js +1 -1
  52. package/dist/embed/local.js +2 -1
  53. package/dist/facts/budget.d.ts +4 -0
  54. package/dist/facts/budget.js +12 -0
  55. package/dist/facts/claude-memory.d.ts +19 -0
  56. package/dist/facts/claude-memory.js +100 -0
  57. package/dist/facts/distill.d.ts +25 -0
  58. package/dist/facts/distill.js +141 -0
  59. package/dist/facts/model.d.ts +28 -0
  60. package/dist/facts/model.js +82 -0
  61. package/dist/facts/notice.d.ts +2 -0
  62. package/dist/facts/notice.js +22 -0
  63. package/dist/facts/ops.d.ts +23 -0
  64. package/dist/facts/ops.js +91 -0
  65. package/dist/facts/render.d.ts +4 -0
  66. package/dist/facts/render.js +13 -0
  67. package/dist/facts/start.d.ts +6 -0
  68. package/dist/facts/start.js +57 -0
  69. package/dist/graph/index.d.ts +2 -6
  70. package/dist/graph/index.js +2 -3
  71. package/dist/graph/languages/basic.js +1 -1
  72. package/dist/graph/languages/index.d.ts +1 -1
  73. package/dist/graph/parser.js +1 -1
  74. package/dist/graph/query/index.d.ts +0 -2
  75. package/dist/graph/query/index.js +0 -1
  76. package/dist/graph/scan/extract.js +1 -1
  77. package/dist/graph/scan/files.js +1 -1
  78. package/dist/graph/scan/index.d.ts +1 -2
  79. package/dist/graph/scan/index.js +1 -1
  80. package/dist/hooks/background.d.ts +3 -0
  81. package/dist/hooks/background.js +43 -0
  82. package/dist/hooks/compact.d.ts +8 -0
  83. package/dist/hooks/compact.js +42 -0
  84. package/dist/hooks/payload.d.ts +3 -0
  85. package/dist/hooks/payload.js +18 -1
  86. package/dist/hooks/pick-deliver.d.ts +2 -0
  87. package/dist/hooks/pick-deliver.js +16 -0
  88. package/dist/hooks/prompt-recall.d.ts +21 -0
  89. package/dist/hooks/prompt-recall.js +36 -0
  90. package/dist/hooks/relevance.d.ts +11 -3
  91. package/dist/hooks/relevance.js +56 -26
  92. package/dist/hooks/session-end.js +7 -1
  93. package/dist/hooks/session-start.js +31 -59
  94. package/dist/hooks/shown.d.ts +1 -0
  95. package/dist/hooks/shown.js +5 -2
  96. package/dist/hooks/start-legacy.d.ts +6 -0
  97. package/dist/hooks/start-legacy.js +47 -0
  98. package/dist/hooks/user-prompt.js +29 -25
  99. package/dist/mcp/server.js +13 -1
  100. package/dist/pick/pending.d.ts +17 -0
  101. package/dist/pick/pending.js +106 -0
  102. package/dist/pick/prompt.d.ts +14 -0
  103. package/dist/pick/prompt.js +80 -0
  104. package/dist/pick/run.d.ts +22 -0
  105. package/dist/pick/run.js +36 -0
  106. package/dist/pick/worker.d.ts +2 -0
  107. package/dist/pick/worker.js +32 -0
  108. package/dist/store/adapter.d.ts +1 -0
  109. package/dist/store/each.d.ts +5 -0
  110. package/dist/store/each.js +25 -0
  111. package/dist/store/mongo/docs.d.ts +2 -0
  112. package/dist/store/mongo/docs.js +4 -0
  113. package/dist/store/mongo/driver.js +3 -2
  114. package/dist/store/mongo/filters.d.ts +1 -0
  115. package/dist/store/mongo/filters.js +8 -0
  116. package/dist/store/mongo/index.d.ts +1 -0
  117. package/dist/store/mongo/index.js +5 -2
  118. package/dist/store/mongo/observations.d.ts +2 -1
  119. package/dist/store/mongo/observations.js +11 -1
  120. package/dist/store/mongo/search.js +12 -3
  121. package/dist/store/mongo/sessions.js +7 -1
  122. package/dist/store/mongo/vectorSearch.js +9 -11
  123. package/dist/store/postgres/driver.js +3 -2
  124. package/dist/store/postgres/filters.js +5 -0
  125. package/dist/store/postgres/index.d.ts +1 -0
  126. package/dist/store/postgres/index.js +3 -0
  127. package/dist/store/postgres/meta.js +2 -1
  128. package/dist/store/postgres/observations.d.ts +1 -0
  129. package/dist/store/postgres/observations.js +14 -3
  130. package/dist/store/postgres/rows.js +4 -0
  131. package/dist/store/postgres/schema.d.ts +2 -2
  132. package/dist/store/postgres/schema.js +4 -1
  133. package/dist/store/postgres/sessions.js +14 -7
  134. package/dist/store/session-time.d.ts +2 -0
  135. package/dist/store/session-time.js +5 -0
  136. package/dist/store/sqlite/filters.js +5 -0
  137. package/dist/store/sqlite/index.d.ts +1 -0
  138. package/dist/store/sqlite/index.js +23 -4
  139. package/dist/store/sqlite/observations.d.ts +1 -0
  140. package/dist/store/sqlite/observations.js +14 -1
  141. package/dist/store/sqlite/rows.js +4 -0
  142. package/dist/store/sqlite/schema.sql +3 -1
  143. package/dist/store/sqlite/search.js +7 -3
  144. package/dist/store/sqlite/sessions.js +13 -8
  145. package/dist/types.d.ts +5 -1
  146. package/dist/usages/find.js +5 -3
  147. package/dist/usages/index.d.ts +0 -1
  148. package/dist/util/claude-cli.d.ts +6 -0
  149. package/dist/util/claude-cli.js +38 -0
  150. package/dist/util/daily-budget.d.ts +7 -0
  151. package/dist/util/daily-budget.js +38 -0
  152. package/dist/util/day.d.ts +1 -0
  153. package/dist/util/day.js +11 -0
  154. package/dist/util/job-lock.d.ts +4 -0
  155. package/dist/util/job-lock.js +56 -0
  156. package/dist/util/prompt.d.ts +5 -0
  157. package/dist/util/prompt.js +31 -0
  158. package/docs/how-it-works.md +69 -3
  159. package/docs/setup-guide.md +339 -0
  160. package/docs/with-and-without.md +43 -36
  161. package/package.json +23 -7
@@ -0,0 +1,56 @@
1
+ import { basename } from 'node:path';
2
+ import { claimJob, finishJob, jobMark, releaseJob } from '../util/job-lock.js';
3
+ import { flushSession, resetCursor } from './flush.js';
4
+ import { readTranscript, transcriptsFor } from './transcript.js';
5
+ import { observationsFromTurns } from './turn-extractor.js';
6
+ export const REINGEST_VERSION = 1;
7
+ const SESSION_ROWS = 10_000;
8
+ export async function reingestSession(ctx, project, path) {
9
+ const sessionId = basename(path, '.jsonl');
10
+ const { turns, lastTimestamp } = readTranscript(path, 0);
11
+ resetCursor(sessionId);
12
+ const { observations: saved } = await flushSession(ctx, sessionId, project, path, true);
13
+ if (lastTimestamp === 0)
14
+ return { sessionId, saved, replaced: 0 };
15
+ const current = new Set(observationsFromTurns(turns, sessionId, project, ctx.config).map((obs) => obs.id));
16
+ const stored = await ctx.store.list({ project, sessionId, limit: SESSION_ROWS });
17
+ const stale = stored
18
+ .filter((obs) => obs.status !== 'replaced' && obs.createdAt <= lastTimestamp)
19
+ .filter((obs) => !current.has(obs.id))
20
+ .map((obs) => obs.id);
21
+ return { sessionId, saved, replaced: await ctx.store.markReplaced(stale) };
22
+ }
23
+ export async function rememberedTranscripts(ctx, project, paths = transcriptsFor(project)) {
24
+ const kept = [];
25
+ for (const path of paths) {
26
+ const sessionId = basename(path, '.jsonl');
27
+ if ((await ctx.store.list({ project, sessionId, limit: 1 })).length > 0)
28
+ kept.push(path);
29
+ }
30
+ return kept;
31
+ }
32
+ export async function reingestProject(ctx, project, paths = transcriptsFor(project), onSession = () => { }) {
33
+ const total = { transcripts: paths.length, saved: 0, replaced: 0 };
34
+ for (const path of paths) {
35
+ const result = await reingestSession(ctx, project, path);
36
+ total.saved += result.saved;
37
+ total.replaced += result.replaced;
38
+ onSession(result);
39
+ }
40
+ return total;
41
+ }
42
+ const JOB = 'reingest';
43
+ export function reingestDone(project) {
44
+ return jobMark(JOB, project) >= REINGEST_VERSION;
45
+ }
46
+ export function claimReingest(project, now = Date.now()) {
47
+ if (reingestDone(project))
48
+ return false;
49
+ return claimJob(JOB, project, now);
50
+ }
51
+ export function finishReingest(project) {
52
+ finishJob(JOB, project, REINGEST_VERSION);
53
+ }
54
+ export function releaseReingest(project) {
55
+ releaseJob(JOB, project);
56
+ }
@@ -0,0 +1,11 @@
1
+ import type { RecallContext } from '../context.js';
2
+ export declare const REDACT_VERSION = 1;
3
+ export interface ScrubResult {
4
+ observations: number;
5
+ sessions: number;
6
+ }
7
+ export declare function scrubSecrets(ctx: RecallContext, now?: number): Promise<ScrubResult>;
8
+ export declare function scrubDone(database: string): boolean;
9
+ export declare function claimScrub(database: string, now?: number): boolean;
10
+ export declare function finishScrub(database: string): void;
11
+ export declare function releaseScrub(database: string): void;
@@ -0,0 +1,48 @@
1
+ import { eachObservation } from '../store/each.js';
2
+ import { claimJob, finishJob, jobMark, releaseJob } from '../util/job-lock.js';
3
+ import { embedObservations } from './flush.js';
4
+ import { redact } from './redact.js';
5
+ export const REDACT_VERSION = 1;
6
+ const JOB = 'redact';
7
+ const SESSION_LIMIT = 100_000;
8
+ function redacted(obs) {
9
+ const title = redact(obs.title);
10
+ const body = redact(obs.body);
11
+ return title === obs.title && body === obs.body ? null : { ...obs, title, body };
12
+ }
13
+ export async function scrubSecrets(ctx, now = Date.now()) {
14
+ let observations = 0;
15
+ await eachObservation(ctx, {}, async (batch) => {
16
+ const changed = batch.map(redacted).filter((obs) => obs !== null);
17
+ if (changed.length === 0)
18
+ return;
19
+ await embedObservations(ctx, changed);
20
+ await ctx.store.insertObservations(changed);
21
+ observations += changed.length;
22
+ });
23
+ let sessions = 0;
24
+ for (const { project } of await ctx.store.listProjects()) {
25
+ for (const session of await ctx.store.recentSessions(project, SESSION_LIMIT)) {
26
+ if (!session.summary)
27
+ continue;
28
+ const summary = redact(session.summary);
29
+ if (summary === session.summary)
30
+ continue;
31
+ await ctx.store.upsertSession({ ...session, summary, updatedAt: now });
32
+ sessions += 1;
33
+ }
34
+ }
35
+ return { observations, sessions };
36
+ }
37
+ export function scrubDone(database) {
38
+ return jobMark(JOB, database) >= REDACT_VERSION;
39
+ }
40
+ export function claimScrub(database, now = Date.now()) {
41
+ return !scrubDone(database) && claimJob(JOB, database, now);
42
+ }
43
+ export function finishScrub(database) {
44
+ finishJob(JOB, database, REDACT_VERSION);
45
+ }
46
+ export function releaseScrub(database) {
47
+ releaseJob(JOB, database);
48
+ }
@@ -1,4 +1,4 @@
1
- import { execFile } from 'node:child_process';
1
+ import { runHeadless } from '../util/claude-cli.js';
2
2
  const MAX_INPUT_CHARS = 8000;
3
3
  const MAX_OUTPUT_CHARS = 800;
4
4
  const TIMEOUT_MS = 45_000;
@@ -21,12 +21,6 @@ export function validateSummary(raw) {
21
21
  export async function aiSummary(sessionText, model) {
22
22
  if (!sessionText.trim())
23
23
  return null;
24
- return new Promise((done) => {
25
- try {
26
- execFile('claude', ['-p', buildSummaryPrompt(sessionText), '--model', model, '--effort', 'low'], { timeout: TIMEOUT_MS, maxBuffer: 1024 * 1024 }, (error, stdout) => done(error ? null : validateSummary(stdout)));
27
- }
28
- catch {
29
- done(null);
30
- }
31
- });
24
+ const stdout = await runHeadless(buildSummaryPrompt(sessionText), model, TIMEOUT_MS);
25
+ return stdout === null ? null : validateSummary(stdout);
32
26
  }
@@ -9,6 +9,7 @@ export interface Turn {
9
9
  export interface TranscriptRead {
10
10
  turns: Turn[];
11
11
  nextOffset: number;
12
+ lastTimestamp: number;
12
13
  }
13
14
  export declare function readTranscript(path: string, fromOffset?: number): TranscriptRead;
14
15
  export declare function transcriptPathFor(project: string, sessionId: string): string;
@@ -1,6 +1,7 @@
1
1
  import { closeSync, fstatSync, openSync, readdirSync, readSync } from 'node:fs';
2
2
  import { homedir } from 'node:os';
3
3
  import { join } from 'node:path';
4
+ import { isRelayedMessage } from '../util/prompt.js';
4
5
  import { resolveProject } from '../util/project.js';
5
6
  export function readTranscript(path, fromOffset = 0) {
6
7
  let fd;
@@ -8,14 +9,14 @@ export function readTranscript(path, fromOffset = 0) {
8
9
  fd = openSync(path, 'r');
9
10
  }
10
11
  catch {
11
- return { turns: [], nextOffset: fromOffset };
12
+ return { turns: [], nextOffset: fromOffset, lastTimestamp: 0 };
12
13
  }
13
14
  try {
14
15
  const size = fstatSync(fd).size;
15
16
  const start = fromOffset > size ? 0 : fromOffset;
16
17
  const length = size - start;
17
18
  if (length <= 0)
18
- return { turns: [], nextOffset: size };
19
+ return { turns: [], nextOffset: size, lastTimestamp: 0 };
19
20
  const buffer = Buffer.allocUnsafe(length);
20
21
  const bytesRead = readSync(fd, buffer, 0, length, start);
21
22
  const raw = buffer.subarray(0, bytesRead).toString('utf8');
@@ -35,7 +36,11 @@ export function readTranscript(path, fromOffset = 0) {
35
36
  const complete = lastNewline < 0 ? start : start + Buffer.byteLength(raw.slice(0, lastNewline + 1), 'utf8');
36
37
  const turns = groupIntoTurns(entries);
37
38
  const nextOffset = turns.length > 0 ? (turns[turns.length - 1]?.offset ?? complete) : complete;
38
- return { turns, nextOffset };
39
+ const lastTimestamp = entries.reduce((latest, { entry }) => {
40
+ const at = Date.parse(entry.timestamp ?? '');
41
+ return Number.isNaN(at) ? latest : Math.max(latest, at);
42
+ }, 0);
43
+ return { turns, nextOffset, lastTimestamp };
39
44
  }
40
45
  finally {
41
46
  closeSync(fd);
@@ -51,6 +56,8 @@ function groupIntoTurns(entries) {
51
56
  if (!Number.isNaN(parsed))
52
57
  lastKnown = parsed;
53
58
  if (entry.type === 'user') {
59
+ if (entry.isMeta || entry.origin?.kind === 'task-notification')
60
+ continue;
54
61
  const text = extractText(entry.message?.content);
55
62
  if (!text || entry.isCompactSummary || isSyntheticPrompt(text))
56
63
  continue;
@@ -114,6 +121,8 @@ function isSyntheticPrompt(text) {
114
121
  return (text.includes('<system-reminder>') ||
115
122
  text.includes('<project-memory>') ||
116
123
  text.includes('<recalled-memory>') ||
124
+ text.startsWith('<memory') ||
125
+ isRelayedMessage(text) ||
117
126
  text.startsWith('Caveat:') ||
118
127
  text.startsWith('[Request interrupted') ||
119
128
  text.startsWith('<local-command'));
@@ -4,4 +4,3 @@ import type { Turn } from './transcript.js';
4
4
  export declare function observationsFromTurns(turns: Turn[], sessionId: string, project: string, config: Config): Observation[];
5
5
  export declare function topLevelDirs(files: string[], project: string): string[];
6
6
  export declare function classifyTurn(turn: Turn): ObservationKind;
7
- export declare function redact(text: string): string;
@@ -1,11 +1,16 @@
1
1
  import { classifyCommand } from './command.js';
2
- import { isSearchable } from '../util/prompt.js';
2
+ import { isSearchable, readablePrompt } from '../util/prompt.js';
3
3
  import { currentAuthor, observationId } from './identity.js';
4
+ import { redact } from './redact.js';
4
5
  export function observationsFromTurns(turns, sessionId, project, config) {
5
6
  return turns
6
7
  .map(withoutPrivate)
7
- .filter((turn) => isSubstantive(turn, config))
8
- .map((turn) => buildObservation(turn, sessionId, project, config));
8
+ .map((turn) => ({
9
+ id: observationId(sessionId, turn.timestamp, turn.prompt),
10
+ turn: { ...turn, prompt: readablePrompt(turn.prompt) },
11
+ }))
12
+ .filter(({ turn }) => isSubstantive(turn, config))
13
+ .map(({ id, turn }) => buildObservation(turn, id, sessionId, project, config));
9
14
  }
10
15
  function withoutPrivate(turn) {
11
16
  const strip = (text) => text.replace(/<private>[\s\S]*?(?:<\/private>|$)/gi, '[private]');
@@ -22,7 +27,7 @@ function answers(reasoning) {
22
27
  const scored = scoreSentences(stripMarkdown(redact(reasoning)));
23
28
  return scored.length >= 2 && scored.some(([, score]) => score > 0);
24
29
  }
25
- function buildObservation(turn, sessionId, project, config) {
30
+ function buildObservation(turn, id, sessionId, project, config) {
26
31
  const files = [...new Set(turn.files.filter((f) => !isExcluded(f, config.capture.exclude)))];
27
32
  const commands = turn.commands
28
33
  .map(classifyCommand)
@@ -39,7 +44,7 @@ function buildObservation(turn, sessionId, project, config) {
39
44
  const author = currentAuthor();
40
45
  const kind = classifyTurn(turn);
41
46
  return {
42
- id: observationId(sessionId, turn.timestamp, turn.prompt),
47
+ id,
43
48
  sessionId,
44
49
  project,
45
50
  ...(author ? { author } : {}),
@@ -165,20 +170,6 @@ function shortPath(path) {
165
170
  function isExcluded(file, patterns) {
166
171
  return patterns.some((pattern) => file.includes(pattern));
167
172
  }
168
- const PRIVATE_BLOCK = /<private>[\s\S]*?(?:<\/private>|$)/gi;
169
- const PRIVATE_KEY = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g;
170
- export function redact(text) {
171
- return text
172
- .replace(PRIVATE_BLOCK, '[redacted]')
173
- .replace(PRIVATE_KEY, '[redacted-private-key]')
174
- .replace(/\/\/[^\s:/@]+:[^\s@]+@/g, '//[redacted]@')
175
- .replace(/\b(sk-[A-Za-z0-9_-]{16,})\b/g, '[redacted-key]')
176
- .replace(/\b(gh[pousr]_[A-Za-z0-9]{16,})\b/g, '[redacted-token]')
177
- .replace(/\bAKIA[0-9A-Z]{16}\b/g, '[redacted-key]')
178
- .replace(/\bxox[baprs]-[A-Za-z0-9-]{10,}/g, '[redacted-token]')
179
- .replace(/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g, '[redacted-jwt]')
180
- .replace(/(?<=(password|secret|token|api[_-]?key)"?\s*[:=]\s*)"[^"]+"/gi, '"[redacted]"');
181
- }
182
173
  function truncate(text, max) {
183
174
  return text.length <= max ? text : `${text.slice(0, max)}\n... [truncated]`;
184
175
  }
@@ -33,9 +33,9 @@ export async function cmdAdoption() {
33
33
  continue;
34
34
  seen.add(item.id);
35
35
  const name = typeof item.name === 'string' ? item.name : '';
36
- if (name === 'Bash' && typeof item.input?.command === 'string') {
37
- bash.push(item.input.command);
38
- }
36
+ const command = item.input?.command;
37
+ if (name === 'Bash' && typeof command === 'string')
38
+ bash.push(command);
39
39
  if (name.startsWith('mcp__memory__')) {
40
40
  const tool = name.slice('mcp__memory__'.length);
41
41
  memory.set(tool, (memory.get(tool) ?? 0) + 1);
@@ -0,0 +1 @@
1
+ export declare function cmdDistill(argv: (string | undefined)[]): Promise<void>;
@@ -0,0 +1,67 @@
1
+ import { setConfigValue } from '../../config/index.js';
2
+ import { createContext } from '../../context.js';
3
+ import { distillUsage } from '../../facts/budget.js';
4
+ import { importClaudeMemory } from '../../facts/claude-memory.js';
5
+ import { FACTS_JOB, backfill, distillSession, factCounts, pendingSessions, } from '../../facts/distill.js';
6
+ import { finishJob, releaseJob } from '../../util/job-lock.js';
7
+ import { resolveProject } from '../../util/project.js';
8
+ export async function cmdDistill(argv) {
9
+ const [first] = argv;
10
+ if (first === 'on' || first === 'off') {
11
+ setConfigValue('distill', 'enabled', first === 'on');
12
+ console.log(first === 'on'
13
+ ? 'Chats will be turned into facts after they end, with one small Haiku call each.'
14
+ : 'Turning chats into facts is off. Chats are still recorded; nothing calls Claude.');
15
+ return;
16
+ }
17
+ const project = resolveProject(undefined);
18
+ if (first === '--backfill') {
19
+ await runBackfill(project);
20
+ return;
21
+ }
22
+ const ctx = await createContext();
23
+ try {
24
+ const session = argv[argv.indexOf('--session') + 1];
25
+ if (first === '--session' && session) {
26
+ if (ctx.config.distill.enabled)
27
+ await distillSession(ctx, project, session);
28
+ return;
29
+ }
30
+ await printStatus(ctx, project);
31
+ }
32
+ finally {
33
+ await ctx.close();
34
+ }
35
+ }
36
+ async function runBackfill(project) {
37
+ try {
38
+ const ctx = await createContext();
39
+ try {
40
+ await importClaudeMemory(ctx, project);
41
+ if (ctx.config.distill.enabled)
42
+ await backfill(ctx, project);
43
+ }
44
+ finally {
45
+ await ctx.close();
46
+ }
47
+ }
48
+ catch (error) {
49
+ releaseJob(FACTS_JOB, project);
50
+ throw error;
51
+ }
52
+ finishJob(FACTS_JOB, project, Date.now());
53
+ }
54
+ async function printStatus(ctx, project) {
55
+ const { distill } = ctx.config;
56
+ const usage = distillUsage();
57
+ const facts = await factCounts(ctx, project);
58
+ console.log(`distill : ${distill.enabled ? 'on' : 'off'} (${distill.model})`);
59
+ console.log(`today : ${usage.used} of ${distill.dailyLimit} calls used`);
60
+ if (usage.pausedUntil > Date.now()) {
61
+ console.log(`paused : until ${new Date(usage.pausedUntil).toISOString()} after a failed call`);
62
+ }
63
+ console.log(`facts : ${facts.project} for this project, ${facts.you} about you`);
64
+ console.log(`waiting : ${(await pendingSessions(ctx, project)).length} chat(s) not yet turned into facts`);
65
+ if (!distill.enabled)
66
+ console.log('\nTurn it on with: claude-db distill on');
67
+ }
@@ -2,8 +2,9 @@ import { createContext } from '../../context.js';
2
2
  import { loadConfig } from '../../config/index.js';
3
3
  import { packageVersion } from '../../update.js';
4
4
  import { randomUUID } from 'node:crypto';
5
- import { remember } from '../../capture/index.js';
5
+ import { redact, remember } from '../../capture/index.js';
6
6
  import { resolveProject } from '../../util/project.js';
7
+ import { ourHookFile } from '../install.js';
7
8
  import { settingsPathFor } from '../paths.js';
8
9
  import { existsSync, readFileSync } from 'node:fs';
9
10
  import { toShortId } from '../../util/shortid.js';
@@ -16,7 +17,7 @@ export async function cmdDoctor(argv) {
16
17
  const embedder = await ctx.embedder();
17
18
  const vectors = await probeEmbedder(embedder);
18
19
  console.log(`version : ${packageVersion()}`);
19
- console.log(`database : ${ctx.config.database}`);
20
+ console.log(`database : ${redact(ctx.config.database)}`);
20
21
  console.log(`adapter : ${ctx.store.kind}`);
21
22
  console.log(`reachable: ${reachable ? 'yes' : 'no'}`);
22
23
  console.log(`requested: embeddings.provider = ${ctx.config.embeddings.provider}`);
@@ -46,9 +47,16 @@ function checkWiring(project) {
46
47
  for (const [event, entries] of Object.entries(settings.hooks ?? {})) {
47
48
  const ours = entries
48
49
  .flatMap((entry) => entry.hooks.map((hook) => hook.command))
49
- .filter((command) => /[\\/]hooks[\\/](session-start|session-end|user-prompt|prefer-usages)\.js$/.test(command.replace(/\\/g, '/')));
50
- if (ours.length > 1)
51
- problems.push(`${event} registered ${ours.length}x in ${path}`);
50
+ .filter((command) => ourHookFile(command) !== null);
51
+ const counts = new Map();
52
+ for (const command of ours) {
53
+ const file = ourHookFile(command) ?? '';
54
+ counts.set(file, (counts.get(file) ?? 0) + 1);
55
+ }
56
+ for (const [file, count] of counts) {
57
+ if (count > 1)
58
+ problems.push(`${event} ${file} registered ${count}x in ${path}`);
59
+ }
52
60
  for (const command of ours) {
53
61
  const file = command.replace(/^node\s+/, '');
54
62
  if (!existsSync(file))
@@ -1,7 +1,8 @@
1
1
  import { createContext } from '../../context.js';
2
- import { eachObservation } from '../commands/transfer.js';
2
+ import { eachObservation } from '../../store/each.js';
3
3
  import { resolve } from 'node:path';
4
4
  import { resolveProject } from '../../util/project.js';
5
+ import { redact } from '../../capture/redact.js';
5
6
  export async function cmdStats() {
6
7
  const project = resolveProject(undefined);
7
8
  const ctx = await createContext();
@@ -9,10 +10,15 @@ export async function cmdStats() {
9
10
  const kinds = new Map();
10
11
  const tags = new Map();
11
12
  let embedded = 0;
13
+ let replaced = 0;
12
14
  let earliest = Number.POSITIVE_INFINITY;
13
15
  let latest = 0;
14
- const total = await eachObservation(ctx, { project }, (batch) => {
16
+ const stored = await eachObservation(ctx, { project }, (batch) => {
15
17
  for (const obs of batch) {
18
+ if (obs.status === 'replaced') {
19
+ replaced += 1;
20
+ continue;
21
+ }
16
22
  kinds.set(obs.kind, (kinds.get(obs.kind) ?? 0) + 1);
17
23
  for (const tag of obs.tags)
18
24
  tags.set(tag, (tags.get(tag) ?? 0) + 1);
@@ -22,13 +28,17 @@ export async function cmdStats() {
22
28
  latest = Math.max(latest, obs.createdAt);
23
29
  }
24
30
  });
31
+ const total = stored - replaced;
25
32
  if (total === 0) {
26
- console.log(`No memory stored for ${project}.`);
33
+ const note = replaced > 0 ? ` (${replaced} replaced, kept but left out of search)` : '';
34
+ console.log(`No memory stored for ${project}${note}.`);
27
35
  return;
28
36
  }
29
37
  const day = (ms) => new Date(ms).toISOString().slice(0, 10);
30
38
  console.log(`project : ${project}`);
31
39
  console.log(`observations: ${total}`);
40
+ if (replaced > 0)
41
+ console.log(`replaced : ${replaced}, kept but left out of search`);
32
42
  console.log(`range : ${day(earliest)} to ${day(latest)}`);
33
43
  console.log(`embedded : ${embedded} of ${total}`);
34
44
  console.log('\nby kind');
@@ -55,7 +65,7 @@ export async function cmdProjects() {
55
65
  console.log('No memory stored yet.');
56
66
  return;
57
67
  }
58
- console.log(`database: ${ctx.config.database}\n`);
68
+ console.log(`database: ${redact(ctx.config.database)}\n`);
59
69
  for (const entry of projects) {
60
70
  const marker = entry.project === here ? '*' : ' ';
61
71
  const date = new Date(entry.lastActive).toISOString().slice(0, 10);
@@ -1,3 +1,4 @@
1
1
  export declare function cmdReset(argv: (string | undefined)[]): Promise<void>;
2
- export declare function cmdFlush(): Promise<void>;
2
+ export declare function cmdFlush(argv: (string | undefined)[]): Promise<void>;
3
3
  export declare function cmdUpdate(argv: (string | undefined)[]): Promise<void>;
4
+ export declare function cmdRedact(argv: (string | undefined)[]): Promise<void>;
@@ -1,8 +1,8 @@
1
1
  import { CONFIG_DIR, loadConfig } from '../../config/index.js';
2
- import { basename, join } from 'node:path';
2
+ import { join } from 'node:path';
3
3
  import { checkForUpdate } from '../../update.js';
4
4
  import { createContext } from '../../context.js';
5
- import { flushSession, resetCursor, sweepCursors, transcriptsFor } from '../../capture/index.js';
5
+ import { finishReingest, finishScrub, redact, reingestProject, releaseReingest, releaseScrub, rememberedTranscripts, scrubSecrets, sweepCursors, transcriptsFor, } from '../../capture/index.js';
6
6
  import { resolveProject } from '../../util/project.js';
7
7
  import { rmSync } from 'node:fs';
8
8
  export async function cmdReset(argv) {
@@ -11,7 +11,7 @@ export async function cmdReset(argv) {
11
11
  const project = resolveProject(undefined);
12
12
  const ctx = await createContext();
13
13
  try {
14
- const target = scoped ? project : ctx.config.database;
14
+ const target = scoped ? project : redact(ctx.config.database);
15
15
  if (!confirmed) {
16
16
  console.log(`This would delete ${scoped ? "this project's" : 'ALL'} memory from:`);
17
17
  console.log(` ${target}`);
@@ -32,34 +32,48 @@ export async function cmdReset(argv) {
32
32
  function clearLocalState() {
33
33
  rmSync(join(CONFIG_DIR, 'cursors'), { recursive: true, force: true });
34
34
  }
35
- export async function cmdFlush() {
35
+ export async function cmdFlush(argv) {
36
36
  const project = resolveProject(undefined);
37
+ const repair = argv.includes('--repair');
37
38
  const transcripts = transcriptsFor(project);
38
39
  if (transcripts.length === 0) {
40
+ finishReingest(project);
39
41
  console.error(`No transcripts found for ${project}`);
40
42
  process.exit(1);
41
43
  }
42
- const ctx = await createContext();
43
- let total = 0;
44
+ let total;
44
45
  try {
45
- for (const path of transcripts) {
46
- const sessionId = basename(path, '.jsonl');
47
- resetCursor(sessionId);
48
- const result = await flushSession(ctx, sessionId, project, path, true);
49
- if (result.observations > 0) {
50
- console.log(`${sessionId.slice(0, 8)} ${String(result.observations).padStart(4)} observations`);
51
- total += result.observations;
52
- }
53
- }
46
+ total = await reingest(project, transcripts, repair);
54
47
  }
55
- finally {
56
- await ctx.close();
48
+ catch (error) {
49
+ releaseReingest(project);
50
+ throw error;
51
+ }
52
+ finishReingest(project);
53
+ console.log(`\n${total.saved} observations from ${total.transcripts} transcript(s).`);
54
+ if (total.replaced > 0) {
55
+ console.log(`${total.replaced} older observation(s) saved under the previous capture rules were ` +
56
+ 'marked replaced. They stay in the database but no longer appear in search.');
57
57
  }
58
- console.log(`\n${total} observations from ${transcripts.length} transcript(s).`);
59
58
  const swept = sweepCursors();
60
59
  if (swept > 0)
61
60
  console.log(`Swept ${swept} cursor(s) for transcripts that no longer exist.`);
62
61
  }
62
+ async function reingest(project, transcripts, repair) {
63
+ const ctx = await createContext();
64
+ try {
65
+ const chosen = repair ? await rememberedTranscripts(ctx, project, transcripts) : transcripts;
66
+ return await reingestProject(ctx, project, chosen, (result) => {
67
+ if (result.saved === 0 && result.replaced === 0)
68
+ return;
69
+ const replaced = result.replaced > 0 ? `, ${result.replaced} replaced` : '';
70
+ console.log(`${result.sessionId.slice(0, 8)} ${String(result.saved).padStart(4)} observations${replaced}`);
71
+ });
72
+ }
73
+ finally {
74
+ await ctx.close();
75
+ }
76
+ }
63
77
  export async function cmdUpdate(argv) {
64
78
  const quiet = argv.includes('--quiet');
65
79
  const config = loadConfig();
@@ -75,3 +89,25 @@ export async function cmdUpdate(argv) {
75
89
  else
76
90
  console.log(`Up to date (${result.current}).`);
77
91
  }
92
+ export async function cmdRedact(argv) {
93
+ const background = argv.includes('--background');
94
+ const ctx = await createContext();
95
+ const database = ctx.config.database;
96
+ let result;
97
+ try {
98
+ result = await scrubSecrets(ctx);
99
+ }
100
+ catch (error) {
101
+ releaseScrub(database);
102
+ throw error;
103
+ }
104
+ finally {
105
+ await ctx.close();
106
+ }
107
+ finishScrub(database);
108
+ if (background)
109
+ return;
110
+ console.log(result.observations + result.sessions === 0
111
+ ? 'Nothing to redact: no saved memory holds a secret the current rules catch.'
112
+ : `Redacted ${result.observations} memory row(s) and ${result.sessions} chat summary(ies).`);
113
+ }
@@ -0,0 +1,3 @@
1
+ import type { Config } from '../../config/index.js';
2
+ export declare function pickStatus(config: Config, now?: number): string;
3
+ export declare function cmdPick(argv: (string | undefined)[]): void;
@@ -0,0 +1,25 @@
1
+ import { loadConfig, setConfigValue } from '../../config/index.js';
2
+ import { PICK_BUDGET } from '../../pick/run.js';
3
+ import { budgetUsage } from '../../util/daily-budget.js';
4
+ export function pickStatus(config, now = Date.now()) {
5
+ const { pick } = config;
6
+ if (!pick.enabled) {
7
+ return 'pick : off, prompts get memory only on a strong word match (claude-db pick on)';
8
+ }
9
+ const usage = budgetUsage(PICK_BUDGET, now);
10
+ const paused = usage.pausedUntil > now
11
+ ? `, paused until ${new Date(usage.pausedUntil).toISOString()} after a failed call`
12
+ : '';
13
+ return `pick : on (${pick.model}), ${usage.used} of ${pick.dailyLimit} picks used today${paused}`;
14
+ }
15
+ export function cmdPick(argv) {
16
+ const [first] = argv;
17
+ if (first === 'on' || first === 'off') {
18
+ setConfigValue('pick', 'enabled', first === 'on');
19
+ console.log(first === 'on'
20
+ ? 'Haiku will pick the memory shown with each prompt, up to the daily limit.'
21
+ : 'Haiku picking is off. Prompts get memory only on a strong word match; nothing calls Claude.');
22
+ return;
23
+ }
24
+ console.log(pickStatus(loadConfig()));
25
+ }
@@ -1,5 +1,5 @@
1
1
  import { toShortId } from '../../util/shortid.js';
2
- import { closeLandedWork, openWork } from '../../capture/index.js';
2
+ import { closeLandedWork, openWork, redact } from '../../capture/index.js';
3
3
  import { CONFIG_PATH } from '../../config/index.js';
4
4
  import { DIST_DIR } from '../constants.js';
5
5
  import { assertStableLocation, install, uninstall } from '../install.js';
@@ -10,6 +10,8 @@ import { readFileSync, statSync } from 'node:fs';
10
10
  import { resolveProject } from '../../util/project.js';
11
11
  import { mentionsPath } from '../../util/paths.js';
12
12
  import { transcriptsFor } from '../../capture/index.js';
13
+ import { factCounts, pendingSessions } from '../../facts/distill.js';
14
+ import { pickStatus } from './pick.js';
13
15
  export async function cmdInstall(scope) {
14
16
  const project = resolveProject(undefined);
15
17
  try {
@@ -28,7 +30,7 @@ export async function cmdInstall(scope) {
28
30
  console.log(`Guidance : ${instructionsPathFor(scope, project)}`);
29
31
  console.log(`Skill : ${skillPathFor(scope, project)} (/cdb-scan)`);
30
32
  console.log(`Config : ${CONFIG_PATH}`);
31
- console.log(`Database : ${ctx.config.database}`);
33
+ console.log(`Database : ${redact(ctx.config.database)}`);
32
34
  if (embedder.id === 'builtin-hashing') {
33
35
  console.log('Search : keyword + basic vectors');
34
36
  console.log(' for semantic search: npm i -g @xenova/transformers');
@@ -51,8 +53,9 @@ function warnIfCommittable(project) {
51
53
  .split('\n')
52
54
  .some((line) => line.trim().replace(/^\//, '') === '.mcp.json');
53
55
  }
54
- catch {
55
- return;
56
+ catch (error) {
57
+ if (error.code !== 'ENOENT')
58
+ return;
56
59
  }
57
60
  if (ignored)
58
61
  return;
@@ -84,7 +87,7 @@ export async function cmdStatus() {
84
87
  }));
85
88
  const active = wiring.filter((entry) => entry.installed);
86
89
  console.log(`project : ${project}`);
87
- console.log(`database : ${ctx.config.database}`);
90
+ console.log(`database : ${redact(ctx.config.database)}`);
88
91
  console.log(`hooks : ${active.length > 0 ? active.map((entry) => entry.label).join(' + ') : 'NOT INSTALLED'}`);
89
92
  const mcpFile = [mcpPathFor('project', project), mcpPathFor('global', project)].find((candidate) => fileMentions(candidate, resolve(DIST_DIR, 'mcp', 'server.js')));
90
93
  console.log(`mcp : ${mcpFile ? `registered (${mcpFile})` : 'NOT REGISTERED'}`);
@@ -94,6 +97,12 @@ export async function cmdStatus() {
94
97
  const worked = workedAt(project);
95
98
  console.log(`worked : ${worked.last > 0 ? ago(worked.last) : 'never'}`);
96
99
  console.log(`recorded : ${lastSaved > 0 ? ago(lastSaved) : 'never'}`);
100
+ const facts = await factCounts(ctx, project);
101
+ console.log(`facts : ${facts.project} for this project, ${facts.you} about you`);
102
+ console.log(ctx.config.distill.enabled
103
+ ? `waiting : ${(await pendingSessions(ctx, project)).length} chat(s) not yet turned into facts`
104
+ : 'distill : off, so chats are not turned into facts (claude-db distill on)');
105
+ console.log(pickStatus(ctx.config));
97
106
  await closeLandedWork(ctx.store, project);
98
107
  const open = await openWork(ctx.store, project);
99
108
  if (open.length > 0) {