claude-db 0.12.2 → 0.12.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +42 -0
  2. package/dist/capture/flush.js +41 -4
  3. package/dist/capture/index.d.ts +1 -0
  4. package/dist/capture/index.js +1 -0
  5. package/dist/capture/progress.js +8 -1
  6. package/dist/capture/vectors.d.ts +7 -0
  7. package/dist/capture/vectors.js +38 -0
  8. package/dist/catchup/gather.js +1 -0
  9. package/dist/cli/commands/adoption.js +41 -29
  10. package/dist/cli/commands/languages.js +10 -6
  11. package/dist/cli/commands/sync.js +5 -4
  12. package/dist/cli/commands/transfer.js +28 -8
  13. package/dist/facts/claude-memory.js +7 -2
  14. package/dist/facts/distill.js +4 -1
  15. package/dist/facts/handoff.js +6 -1
  16. package/dist/facts/health.js +2 -1
  17. package/dist/facts/known.js +2 -0
  18. package/dist/facts/start.js +1 -1
  19. package/dist/graph/query/answer.js +1 -1
  20. package/dist/graph/query/lookup.d.ts +1 -0
  21. package/dist/graph/query/lookup.js +2 -1
  22. package/dist/graph/query/path.js +69 -37
  23. package/dist/graph/query/suggest.js +1 -2
  24. package/dist/hooks/background.js +5 -1
  25. package/dist/hooks/compact.js +7 -1
  26. package/dist/hooks/detached.d.ts +1 -0
  27. package/dist/hooks/detached.js +37 -2
  28. package/dist/hooks/prefer-usages.js +1 -1
  29. package/dist/hooks/session-start.js +1 -1
  30. package/dist/hooks/start-legacy.js +1 -1
  31. package/dist/store/adapter.d.ts +5 -2
  32. package/dist/store/each.d.ts +2 -4
  33. package/dist/store/each.js +1 -0
  34. package/dist/store/mongo/graph.d.ts +2 -1
  35. package/dist/store/mongo/graph.js +17 -2
  36. package/dist/store/mongo/index.d.ts +4 -2
  37. package/dist/store/mongo/index.js +8 -2
  38. package/dist/store/mongo/observations.d.ts +2 -1
  39. package/dist/store/mongo/observations.js +25 -3
  40. package/dist/store/postgres/graph.d.ts +2 -1
  41. package/dist/store/postgres/graph.js +22 -7
  42. package/dist/store/postgres/index.d.ts +5 -2
  43. package/dist/store/postgres/index.js +14 -7
  44. package/dist/store/postgres/insert.js +3 -1
  45. package/dist/store/postgres/meta.d.ts +1 -0
  46. package/dist/store/postgres/meta.js +4 -0
  47. package/dist/store/postgres/observations.d.ts +3 -2
  48. package/dist/store/postgres/observations.js +40 -6
  49. package/dist/store/postgres/rows.js +1 -1
  50. package/dist/store/scoped-store.js +4 -0
  51. package/dist/store/sqlite/graph.d.ts +2 -1
  52. package/dist/store/sqlite/graph.js +19 -2
  53. package/dist/store/sqlite/index.d.ts +4 -2
  54. package/dist/store/sqlite/index.js +8 -2
  55. package/dist/store/sqlite/observations.d.ts +2 -1
  56. package/dist/store/sqlite/observations.js +32 -5
  57. package/dist/types.d.ts +9 -0
  58. package/package.json +6 -8
package/CHANGELOG.md CHANGED
@@ -1,5 +1,47 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.12.4
4
+
5
+ ### Fixed
6
+
7
+ - **`find_usages` finds a path between two symbols anywhere in the code graph.** The search used to load only
8
+ the first 20,000 links of a project, so on a large repository it often said there was no path when there was
9
+ one. It now walks out from both ends and asks the database only for the links it needs.
10
+ - **Each request's memories are saved once.** The last request of a chat is read again on the next save, and it
11
+ was embedded and written again even when nothing in it had changed. It is now skipped.
12
+
13
+ ### Changed
14
+
15
+ - **claude-db downloads far less from a remote database.** Memories are read without their vectors unless a copy
16
+ needs them, and without their text where only titles are shown. Counting facts, finding imported Claude
17
+ memories and finding rows that need a vector now happen in the database instead of reading every row. This
18
+ matters most on a hosted Postgres that bills for data sent out, such as Neon.
19
+ - **`graphology` and `graphology-shortest-path` are no longer dependencies.**
20
+
21
+ By @alok-debnath (#36).
22
+
23
+ ## 0.12.3
24
+
25
+ ### Fixed
26
+
27
+ - **`distill` no longer stops on Postgres.** A Claude memory file with no `modified` time made it fail with
28
+ `invalid input syntax for type bigint`, so chats waited for days. Times are now whole milliseconds.
29
+ - **Rows saved without a vector get one later.** A row is saved as text only when the embedding model is still
30
+ loading, or when its width differs from the Postgres column. The next chat start fills such rows in the
31
+ background, at most every 6 hours, once an embedder of the right width loads. It never replaces a vector from
32
+ another embedder.
33
+ - **Background jobs keep their output.** It goes to `~/.claude-memory/logs/background.log`, which rolls over at
34
+ 1 MB.
35
+ - **The waiting-chats warning names `claude-db distill --backfill`**, which works through them.
36
+ - **`claude-db languages` lists TypeScript, TSX and JavaScript** as built in.
37
+
38
+ ### Changed
39
+
40
+ - **A grep for a symbol that is not in the graph is faster.** The hook no longer runs a "did you mean" scan it
41
+ never shows.
42
+ - **`claude-db adoption` is faster.** It looks each symbol up once, several at a time, and shows a counter on a
43
+ terminal.
44
+
3
45
  ## 0.12.2
4
46
 
5
47
  ### Fixed
@@ -1,9 +1,11 @@
1
+ import { createHash } from 'node:crypto';
1
2
  import { mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
2
3
  import { dirname, join } from 'node:path';
3
4
  import { CONFIG_DIR } from '../config/index.js';
4
5
  import { readTranscript, sessionIdsOnDisk, transcriptPathFor } from './transcript.js';
5
6
  import { observationsFromTurns } from './turn-extractor.js';
6
7
  import { closeLandedWork } from './progress.js';
8
+ import { noteMissingVectors } from './vectors.js';
7
9
  export async function flushSession(ctx, sessionId, project, transcriptPath, rebuild = false, finalReply = '') {
8
10
  const path = transcriptPath ?? transcriptPathFor(project, sessionId);
9
11
  const cursor = readCursor(sessionId);
@@ -22,8 +24,12 @@ export async function flushSession(ctx, sessionId, project, transcriptPath, rebu
22
24
  writeCursor(sessionId, nextOffset);
23
25
  return { observations: 0, summary: null, lastReply };
24
26
  }
25
- await embedObservations(ctx, observations);
26
- await ctx.store.insertObservations(observations);
27
+ const written = rebuild ? new Map() : readWritten(sessionId);
28
+ const prints = new Map(observations.map((obs) => [obs.id, printOf(obs)]));
29
+ const fresh = observations.filter((obs) => written.get(obs.id) !== prints.get(obs.id));
30
+ await embedObservations(ctx, fresh);
31
+ await ctx.store.insertObservations(fresh);
32
+ writeWritten(sessionId, prints);
27
33
  await closeLandedWork(ctx.store, project);
28
34
  const previous = rebuild ? null : await ctx.store.getSession(sessionId);
29
35
  const summary = summarize(observations, previous?.summary);
@@ -37,10 +43,15 @@ export async function flushSession(ctx, sessionId, project, transcriptPath, rebu
37
43
  return { observations: observations.length, summary, lastReply };
38
44
  }
39
45
  export async function embedObservations(ctx, observations) {
46
+ if (observations.length === 0)
47
+ return;
40
48
  try {
41
49
  const embedder = await ctx.embedder();
42
- if (embedder.dimensions === 0)
50
+ if (embedder.dimensions === 0) {
51
+ if (ctx.config.embeddings.provider !== 'none')
52
+ noteMissingVectors(observations);
43
53
  return;
54
+ }
44
55
  const size = ctx.config.embeddings.batchSize;
45
56
  for (let start = 0; start < observations.length; start += size) {
46
57
  const chunk = observations.slice(start, start + size);
@@ -53,8 +64,11 @@ export async function embedObservations(ctx, observations) {
53
64
  }
54
65
  });
55
66
  }
67
+ const stored = ctx.store.storedVectorDims?.() ?? null;
68
+ noteMissingVectors(observations, stored !== null && stored !== embedder.dimensions);
56
69
  }
57
70
  catch (error) {
71
+ noteMissingVectors(observations);
58
72
  process.stderr.write(`[claude-db] embedding skipped: ` +
59
73
  `${error instanceof Error ? error.message : String(error)}\n`);
60
74
  }
@@ -91,6 +105,27 @@ function cursorName(sessionId) {
91
105
  function cursorPath(sessionId) {
92
106
  return join(CONFIG_DIR, 'cursors', cursorName(sessionId));
93
107
  }
108
+ const writtenPath = (sessionId) => cursorPath(sessionId).replace(/\.offset$/, '.written');
109
+ const printOf = (obs) => createHash('sha256').update(JSON.stringify(obs)).digest('base64url');
110
+ function readWritten(sessionId) {
111
+ try {
112
+ const parsed = JSON.parse(readFileSync(writtenPath(sessionId), 'utf8'));
113
+ if (!parsed || typeof parsed !== 'object')
114
+ return new Map();
115
+ return new Map(Object.entries(parsed).filter((entry) => typeof entry[1] === 'string'));
116
+ }
117
+ catch {
118
+ return new Map();
119
+ }
120
+ }
121
+ function writeWritten(sessionId, prints) {
122
+ const path = writtenPath(sessionId);
123
+ try {
124
+ mkdirSync(dirname(path), { recursive: true });
125
+ writeFileSync(path, JSON.stringify(Object.fromEntries(prints)), 'utf8');
126
+ }
127
+ catch { }
128
+ }
94
129
  function readCursor(sessionId) {
95
130
  try {
96
131
  const value = Number.parseInt(readFileSync(cursorPath(sessionId), 'utf8'), 10);
@@ -110,10 +145,12 @@ function writeCursor(sessionId, offset) {
110
145
  }
111
146
  export function resetCursor(sessionId) {
112
147
  writeCursor(sessionId, 0);
148
+ rmSync(writtenPath(sessionId), { force: true });
113
149
  }
114
150
  export function clearCursor(sessionId) {
115
151
  rmSync(cursorPath(sessionId), { force: true });
116
152
  rmSync(cursorPath(sessionId).replace(/\.offset$/, '.shown'), { force: true });
153
+ rmSync(writtenPath(sessionId), { force: true });
117
154
  }
118
155
  export function sweepCursors() {
119
156
  const dir = join(CONFIG_DIR, 'cursors');
@@ -127,7 +164,7 @@ export function sweepCursors() {
127
164
  const live = new Set(sessionIdsOnDisk().map((id) => id.replace(/[^\w-]/g, '_')));
128
165
  let removed = 0;
129
166
  for (const file of files) {
130
- const match = /^(.+)\.(offset|shown)$/.exec(file);
167
+ const match = /^(.+)\.(offset|shown|written)$/.exec(file);
131
168
  if (!match?.[1] || live.has(match[1]))
132
169
  continue;
133
170
  rmSync(join(dir, file), { force: true });
@@ -10,6 +10,7 @@ export { redact } from './redact.js';
10
10
  export { claimScrub, finishScrub, releaseScrub, scrubSecrets } from './scrub.js';
11
11
  export { flushSession, resetCursor, clearCursor, sweepCursors, summarize, embedObservations, } from './flush.js';
12
12
  export type { FlushResult } from './flush.js';
13
+ export { claimReembed, finishReembed, releaseReembed, startReembed, vectorsMissing, } from './vectors.js';
13
14
  export { closeLandedWork, openWork } from './progress.js';
14
15
  export { REINGEST_VERSION, claimReingest, finishReingest, reingestDone, reingestProject, rememberedTranscripts, reingestSession, releaseReingest, } from './reingest.js';
15
16
  export type { ProjectReingest, SessionReingest } from './reingest.js';
@@ -7,6 +7,7 @@ export { observationsFromTurns } from './turn-extractor.js';
7
7
  export { redact } from './redact.js';
8
8
  export { claimScrub, finishScrub, releaseScrub, scrubSecrets } from './scrub.js';
9
9
  export { flushSession, resetCursor, clearCursor, sweepCursors, summarize, embedObservations, } from './flush.js';
10
+ export { claimReembed, finishReembed, releaseReembed, startReembed, vectorsMissing, } from './vectors.js';
10
11
  export { closeLandedWork, openWork } from './progress.js';
11
12
  export { REINGEST_VERSION, claimReingest, finishReingest, reingestDone, reingestProject, rememberedTranscripts, reingestSession, releaseReingest, } from './reingest.js';
12
13
  export { aiSummary, buildSummaryPrompt, validateSummary } from './summarize.js';
@@ -43,6 +43,7 @@ export async function closeLandedWork(store, project) {
43
43
  ...(after === undefined ? {} : { after }),
44
44
  ...(afterId === undefined ? {} : { afterId }),
45
45
  limit: OPEN_LIMIT,
46
+ bodies: false,
46
47
  });
47
48
  if (batch.length === 0)
48
49
  break;
@@ -61,6 +62,12 @@ export async function closeLandedWork(store, project) {
61
62
  return store.closeObservations(landed);
62
63
  }
63
64
  export async function openWork(store, project) {
64
- const open = await store.list({ project, status: 'open', limit: OPEN_LIMIT, newest: true });
65
+ const open = await store.list({
66
+ project,
67
+ status: 'open',
68
+ limit: OPEN_LIMIT,
69
+ newest: true,
70
+ bodies: false,
71
+ });
65
72
  return open.sort((a, b) => b.createdAt - a.createdAt);
66
73
  }
@@ -0,0 +1,7 @@
1
+ import type { Observation } from '../types.js';
2
+ export declare function noteMissingVectors(observations: Observation[], everyRow?: boolean): void;
3
+ export declare const vectorsMissing: (project: string) => boolean;
4
+ export declare function claimReembed(project: string, now?: number): boolean;
5
+ export declare function startReembed(project: string): void;
6
+ export declare function finishReembed(project: string, now?: number): void;
7
+ export declare function releaseReembed(project: string): void;
@@ -0,0 +1,38 @@
1
+ import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
2
+ import { join } from 'node:path';
3
+ import { CONFIG_DIR } from '../config/index.js';
4
+ import { claimJob, finishJob, jobMark, releaseJob } from '../util/job-lock.js';
5
+ import { scopeToken } from '../util/scope.js';
6
+ const JOB = 'reembed';
7
+ const FLAGS = 'vectors-missing';
8
+ const REEMBED_EVERY_MS = 6 * 60 * 60 * 1000;
9
+ const flagPath = (project) => join(CONFIG_DIR, FLAGS, `${scopeToken(project)}.flag`);
10
+ export function noteMissingVectors(observations, everyRow = false) {
11
+ const projects = new Set(observations.filter((obs) => everyRow || !obs.embedding?.length).map((obs) => obs.project));
12
+ for (const project of projects) {
13
+ try {
14
+ mkdirSync(join(CONFIG_DIR, FLAGS), { recursive: true });
15
+ writeFileSync(flagPath(project), '');
16
+ }
17
+ catch {
18
+ continue;
19
+ }
20
+ }
21
+ }
22
+ export const vectorsMissing = (project) => existsSync(flagPath(project));
23
+ export function claimReembed(project, now = Date.now()) {
24
+ if (!vectorsMissing(project))
25
+ return false;
26
+ if (now - jobMark(JOB, project) < REEMBED_EVERY_MS)
27
+ return false;
28
+ return claimJob(JOB, project, now);
29
+ }
30
+ export function startReembed(project) {
31
+ rmSync(flagPath(project), { force: true });
32
+ }
33
+ export function finishReembed(project, now = Date.now()) {
34
+ finishJob(JOB, project, now);
35
+ }
36
+ export function releaseReembed(project) {
37
+ releaseJob(JOB, project);
38
+ }
@@ -18,6 +18,7 @@ export async function gatherCatchup(ctx, project, current, now = Date.now()) {
18
18
  sessionId: FACT_SESSION,
19
19
  newest: true,
20
20
  limit: FACTS_SCANNED,
21
+ bodies: false,
21
22
  });
22
23
  const kept = onePerKey(facts.filter((obs) => obs.status !== 'replaced'));
23
24
  const todos = kept
@@ -4,6 +4,8 @@ import { queryGraph } from '../../graph/index.js';
4
4
  import { transcriptsFor } from '../../capture/transcript.js';
5
5
  import { resolveProject } from '../../util/project.js';
6
6
  import { DECLARED, isSymbol, symbolsGreppedIn } from '../../hooks/grep-symbols.js';
7
+ const MAX_SYMBOLS = 2;
8
+ const LOOKUPS_IN_FLIGHT = 8;
7
9
  export async function cmdAdoption() {
8
10
  const project = resolveProject(undefined);
9
11
  const files = transcriptsFor(project);
@@ -45,35 +47,8 @@ export async function cmdAdoption() {
45
47
  }
46
48
  const greps = bash.filter((command) => /\b(?:grep|rg)\b/.test(command));
47
49
  const candidates = greps.map((command) => symbolsGreppedIn(command)).filter((s) => s.length > 0);
48
- const ctx = await createContext();
49
- let fires = 0;
50
- const cache = new Map();
51
- try {
52
- for (const symbols of candidates) {
53
- let hit = false;
54
- for (const symbol of symbols.slice(0, 2)) {
55
- let fired = cache.get(symbol);
56
- if (fired === undefined) {
57
- const answer = await queryGraph(ctx.store, project, {
58
- mode: 'usages',
59
- symbol,
60
- limit: 20,
61
- });
62
- fired =
63
- !answer.empty &&
64
- (isSymbol(symbol) || answer.definitions.some((d) => DECLARED.has(d.kind)));
65
- cache.set(symbol, fired);
66
- }
67
- if (fired)
68
- hit = true;
69
- }
70
- if (hit)
71
- fires++;
72
- }
73
- }
74
- finally {
75
- await ctx.close();
76
- }
50
+ const firing = await symbolsThatFire(project, candidates.flatMap((symbols) => symbols.slice(0, MAX_SYMBOLS)));
51
+ const fires = candidates.filter((symbols) => symbols.slice(0, MAX_SYMBOLS).some((symbol) => firing.has(symbol))).length;
77
52
  const memoryTotal = [...memory.values()].reduce((a, b) => a + b, 0);
78
53
  const pct = (n, of) => (of === 0 ? '0.0%' : `${((n / of) * 100).toFixed(1)}%`);
79
54
  console.log(`Adoption for ${project} (${files.length} transcripts)\n`);
@@ -92,3 +67,40 @@ export async function cmdAdoption() {
92
67
  console.log(`\n ${ratio}`);
93
68
  }
94
69
  }
70
+ async function symbolsThatFire(project, symbols) {
71
+ const unique = [...new Set(symbols)];
72
+ const firing = new Set();
73
+ if (unique.length === 0)
74
+ return firing;
75
+ const ctx = await createContext();
76
+ const tty = process.stderr.isTTY === true;
77
+ let next = 0;
78
+ let done = 0;
79
+ const lookup = async () => {
80
+ while (next < unique.length) {
81
+ const symbol = unique[next++];
82
+ const answer = await queryGraph(ctx.store, project, {
83
+ mode: 'usages',
84
+ symbol,
85
+ limit: 20,
86
+ suggest: false,
87
+ });
88
+ if (!answer.empty &&
89
+ (isSymbol(symbol) || answer.definitions.some((d) => DECLARED.has(d.kind)))) {
90
+ firing.add(symbol);
91
+ }
92
+ done += 1;
93
+ if (tty)
94
+ process.stderr.write(`\rchecking symbols ${done}/${unique.length}`);
95
+ }
96
+ };
97
+ try {
98
+ await Promise.all(Array.from({ length: Math.min(LOOKUPS_IN_FLIGHT, unique.length) }, () => lookup()));
99
+ }
100
+ finally {
101
+ if (tty)
102
+ process.stderr.write('\r\x1b[K');
103
+ await ctx.close();
104
+ }
105
+ return firing;
106
+ }
@@ -1,13 +1,17 @@
1
1
  import { GRAMMARS, PLATFORM, hasGrammar } from '../../graph/grammars.js';
2
- import { languageNames } from '../../graph/index.js';
2
+ import { LANGUAGES, languageNames } from '../../graph/index.js';
3
3
  const SHIPPED = 'Grammars ship with claude-db for each platform, so there is nothing to add or remove.';
4
- const stateOf = (name) => hasGrammar(name)
5
- ? 'read with real syntax'
6
- : `read by pattern: no grammar for ${PLATFORM} in this install`;
4
+ const stateOf = (name) => {
5
+ if (!GRAMMARS.includes(name))
6
+ return 'read with real syntax, built in';
7
+ return hasGrammar(name)
8
+ ? 'read with real syntax'
9
+ : `read by pattern: no grammar for ${PLATFORM} in this install`;
10
+ };
7
11
  function listLanguages() {
8
12
  console.log(`read with real syntax: ${languageNames()}`);
9
- for (const name of GRAMMARS)
10
- console.log(` ${name.padEnd(8)} ${stateOf(name)}`);
13
+ for (const { id, label } of LANGUAGES)
14
+ console.log(` ${label.padEnd(10)} ${stateOf(id)}`);
11
15
  if (GRAMMARS.some((name) => !hasGrammar(name))) {
12
16
  console.log('Reinstall claude-db to add the grammars for this platform.');
13
17
  }
@@ -18,13 +18,13 @@ export async function cmdSync(argv) {
18
18
  try {
19
19
  await remote.init();
20
20
  const localStatus = new Map();
21
- await eachRemoteObservation(local.store, (batch) => {
21
+ await eachRemoteObservation(local.store, false, (batch) => {
22
22
  for (const obs of batch)
23
23
  localStatus.set(obs.id, obs.status ?? 'done');
24
24
  });
25
25
  const remoteStatus = new Map();
26
26
  let pulled = 0;
27
- await eachRemoteObservation(remote, async (batch) => {
27
+ await eachRemoteObservation(remote, true, async (batch) => {
28
28
  for (const obs of batch)
29
29
  remoteStatus.set(obs.id, obs.status ?? 'done');
30
30
  const fresh = batch.filter((obs) => !localStatus.has(obs.id));
@@ -35,7 +35,7 @@ export async function cmdSync(argv) {
35
35
  await local.store.insertObservations(fresh);
36
36
  });
37
37
  let pushed = 0;
38
- await eachRemoteObservation(local.store, async (batch) => {
38
+ await eachRemoteObservation(local.store, true, async (batch) => {
39
39
  const fresh = batch.filter((obs) => !remoteStatus.has(obs.id));
40
40
  if (fresh.length === 0)
41
41
  return;
@@ -139,7 +139,7 @@ async function reconcileSession(session, from, to) {
139
139
  }
140
140
  return false;
141
141
  }
142
- async function eachRemoteObservation(store, visit) {
142
+ async function eachRemoteObservation(store, vectors, visit) {
143
143
  let after;
144
144
  let afterId;
145
145
  for (;;) {
@@ -147,6 +147,7 @@ async function eachRemoteObservation(store, visit) {
147
147
  ...(after === undefined ? {} : { after }),
148
148
  ...(afterId === undefined ? {} : { afterId }),
149
149
  limit: BATCH,
150
+ vectors,
150
151
  });
151
152
  if (batch.length === 0)
152
153
  return;
@@ -1,7 +1,7 @@
1
1
  import { BATCH } from '../constants.js';
2
2
  const SESSION_LIMIT = 10000;
3
3
  import { createContext } from '../../context.js';
4
- import { embedObservations } from '../../capture/index.js';
4
+ import { embedObservations, finishReembed, releaseReembed, startReembed, } from '../../capture/index.js';
5
5
  import { loadConfig } from '../../config/index.js';
6
6
  import { readFileSync } from 'node:fs';
7
7
  import { resolveProject } from '../../util/project.js';
@@ -127,39 +127,59 @@ export async function cmdPrune(argv) {
127
127
  }
128
128
  export async function cmdReembed(argv = []) {
129
129
  const scoped = argv.includes('--project') || argv.includes('-p');
130
+ const background = argv.includes('--background');
131
+ const project = resolveProject(undefined);
130
132
  const base = loadConfig();
131
133
  const ctx = await createContext({ embeddings: { ...base.embeddings, timeoutMs: 0 } });
134
+ if (background)
135
+ startReembed(project);
136
+ let finished = false;
132
137
  try {
133
138
  const embedder = await ctx.embedder();
134
139
  if (embedder.dimensions === 0) {
135
140
  console.error('No embedder available; nothing to do.');
136
141
  process.exit(1);
137
142
  }
143
+ const stored = ctx.store.storedVectorDims?.() ?? null;
144
+ if (background && stored !== null && stored !== embedder.dimensions) {
145
+ finished = true;
146
+ return;
147
+ }
138
148
  let migrated = false;
139
149
  if (!scoped && ctx.store.migrateVectorDims) {
140
150
  migrated = await ctx.store.migrateVectorDims(embedder.dimensions);
141
151
  }
142
152
  let updated = 0;
143
- let skipped = 0;
144
- const filter = scoped ? { project: resolveProject(undefined) } : {};
145
- const scanned = await eachObservation(ctx, filter, async (batch) => {
146
- const stale = batch.filter((obs) => obs.embedder !== embedder.id || !obs.embedding?.length);
147
- skipped += batch.length - stale.length;
153
+ const filter = scoped ? { project } : {};
154
+ const scanned = await ctx.store.count(filter);
155
+ const vectorStale = background ? {} : { embedder: embedder.id };
156
+ await eachObservation(ctx, { ...filter, vectorStale }, async (batch) => {
157
+ const stale = batch.filter((obs) => !obs.embedding?.length || (!background && obs.embedder !== embedder.id));
148
158
  if (stale.length === 0)
149
159
  return;
150
160
  await embedObservations(ctx, stale);
151
161
  await ctx.store.insertObservations(stale);
152
162
  updated += stale.length;
153
- process.stderr.write(`\r${updated} re-embedded...`);
163
+ if (!background)
164
+ process.stderr.write(`\r${updated} re-embedded...`);
154
165
  });
155
- process.stderr.write('\r');
166
+ if (!background)
167
+ process.stderr.write('\r');
168
+ const skipped = scanned - updated;
156
169
  if (migrated) {
157
170
  console.log(`Rebuilt vector storage at ${embedder.dimensions}d.`);
158
171
  }
159
172
  console.log(`Scanned ${scanned}, re-embedded ${updated} with ${embedder.id}` +
160
173
  `${skipped > 0 ? `, ${skipped} already current` : ''}.`);
174
+ finished = true;
161
175
  }
162
176
  finally {
177
+ if (background) {
178
+ if (finished)
179
+ finishReembed(project);
180
+ else
181
+ releaseReembed(project);
182
+ }
163
183
  await ctx.close();
164
184
  }
165
185
  }
@@ -76,7 +76,7 @@ function memoryFiles(dir) {
76
76
  for (const name of names) {
77
77
  try {
78
78
  const path = join(dir, name);
79
- const parsed = parseMemoryFile(readFileSync(path, 'utf8'), statSync(path).mtimeMs);
79
+ const parsed = parseMemoryFile(readFileSync(path, 'utf8'), Math.trunc(statSync(path).mtimeMs));
80
80
  if (parsed)
81
81
  files.push(parsed);
82
82
  }
@@ -88,7 +88,12 @@ async function importedFacts(ctx, project) {
88
88
  const mine = [machineTag(), originTag(project)];
89
89
  const rows = [];
90
90
  for (const owner of [project, youScope()]) {
91
- const batch = await ctx.store.list({ project: owner, sessionId: FACT_SESSION, limit: 5000 });
91
+ const batch = await ctx.store.list({
92
+ project: owner,
93
+ sessionId: FACT_SESSION,
94
+ tag: CLAUDE_MEMORY_TAG,
95
+ limit: 5000,
96
+ });
92
97
  rows.push(...batch.filter((obs) => obs.tags.includes(CLAUDE_MEMORY_TAG) && mine.every((tag) => obs.tags.includes(tag))));
93
98
  }
94
99
  return rows;
@@ -128,6 +128,9 @@ function clip(text, max) {
128
128
  return text.length <= max ? text : `${text.slice(0, max)}…`;
129
129
  }
130
130
  export async function factCounts(ctx, project) {
131
- const count = async (owner) => (await ctx.store.list({ project: owner, sessionId: FACT_SESSION, limit: 100_000 })).filter((obs) => obs.status !== 'replaced').length;
131
+ const count = async (owner) => {
132
+ const scope = { project: owner, sessionId: FACT_SESSION };
133
+ return ((await ctx.store.count(scope)) - (await ctx.store.count({ ...scope, status: 'replaced' })));
134
+ };
132
135
  return { project: await count(project), you: await count(youScope()) };
133
136
  }
@@ -19,11 +19,16 @@ export async function newestHandoff(ctx, project, now = Date.now()) {
19
19
  sessionId: MANUAL_SESSION,
20
20
  newest: true,
21
21
  limit: MANUAL_NOTES_SCANNED,
22
+ bodies: false,
22
23
  });
23
24
  const current = (obs) => obs.status !== 'replaced' &&
24
25
  obs.tags.includes(HANDOFF_TAG) &&
25
26
  now - obs.createdAt <= HANDOFF_MAX_AGE_MS;
26
- return notes.find(current) ?? null;
27
+ const found = notes.find(current);
28
+ if (!found)
29
+ return null;
30
+ const [full] = await ctx.store.getObservations([found.id]);
31
+ return full ?? null;
27
32
  }
28
33
  function clipLine(line) {
29
34
  return line.length > HANDOFF_LINE_CHARS
@@ -6,5 +6,6 @@ export function waitingWarning(waiting, oldestAt, usage, now = Date.now()) {
6
6
  return null;
7
7
  if (usage.used > 0 || usage.pausedUntil > now)
8
8
  return null;
9
- return `warning : ${waiting} chat(s) have waited over a day and no facts were tried today. Run claude-db doctor`;
9
+ return (`warning : ${waiting} chat(s) have waited over a day and no facts were tried today. ` +
10
+ 'Run claude-db distill --backfill to work through them now, or claude-db doctor to check Haiku');
10
11
  }
@@ -14,6 +14,7 @@ export async function existingFacts(ctx, project) {
14
14
  sessionId: FACT_SESSION,
15
15
  newest: true,
16
16
  limit: EXISTING_LIMIT,
17
+ bodies: false,
17
18
  });
18
19
  for (const obs of onePerKey(rows.filter((row) => row.status !== 'replaced'))) {
19
20
  const key = factKey(obs);
@@ -31,6 +32,7 @@ export async function notedByUser(ctx, project) {
31
32
  sessionId: MANUAL_SESSION,
32
33
  newest: true,
33
34
  limit: EXISTING_LIMIT,
35
+ bodies: false,
34
36
  });
35
37
  return rows
36
38
  .filter((row) => row.status !== 'replaced' && !row.tags.includes(HANDOFF_TAG))
@@ -13,7 +13,7 @@ const UNCOMMITTED = 3;
13
13
  const ORDER = ['rule', 'decision', 'deadend', 'fact'];
14
14
  export async function startFacts(ctx, project, hidden = new Set(), current) {
15
15
  const visible = (obs) => obs.status !== 'replaced' && !hidden.has(obs.id) && !seenByClaude(obs, project);
16
- const list = async (owner, sessionId, limit) => (await ctx.store.list({ project: owner, sessionId, newest: true, limit })).filter(visible);
16
+ const list = async (owner, sessionId, limit) => (await ctx.store.list({ project: owner, sessionId, newest: true, limit, bodies: false })).filter(visible);
17
17
  const you = onePerKey(await list(youScope(), FACT_SESSION, 50));
18
18
  const facts = onePerKey(await list(project, FACT_SESSION, 300));
19
19
  const handoff = await newestHandoff(ctx, project);
@@ -10,7 +10,7 @@ export function addText(answer, root) {
10
10
  return;
11
11
  answer.text = unlinkedText(root, answer.symbol, answer.definitions, answer.inbound);
12
12
  }
13
- export const hasGraph = async (store, project) => (await store.scannedFiles(project)).length > 0;
13
+ export const hasGraph = async (store, project) => (await store.scannedFiles(project, 1)).length > 0;
14
14
  export async function answerQuery(request) {
15
15
  const { store, root, project, query } = request;
16
16
  const refreshed = request.refresh ? await refreshGraph(store, root, project) : [];
@@ -20,5 +20,6 @@ export interface GraphQuery {
20
20
  symbol: string;
21
21
  target?: string;
22
22
  limit: number;
23
+ suggest?: boolean;
23
24
  }
24
25
  export declare function queryGraph(store: MemoryStore, project: string, query: GraphQuery): Promise<GraphAnswer>;
@@ -56,7 +56,8 @@ export async function queryGraph(store, project, query) {
56
56
  answer.inbound = [...direct, ...(await throughAliases(store, project, direct, edgeLimit))];
57
57
  answer.outbound = edges.filter((edge) => idSet.has(edge.srcId) && !pointsAtSymbol(edge));
58
58
  answer.empty = answer.definitions.length === 0 && answer.inbound.length === 0;
59
- if (answer.empty)
59
+ if (answer.empty && query.suggest !== false) {
60
60
  answer.suggestions = await suggestFor(store, project, query.symbol);
61
+ }
61
62
  return answer;
62
63
  }