claude-db 0.12.3 → 0.12.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +20 -0
  2. package/dist/capture/flush.js +31 -3
  3. package/dist/capture/progress.js +8 -1
  4. package/dist/catchup/gather.js +1 -0
  5. package/dist/cli/commands/sync.js +5 -4
  6. package/dist/cli/commands/transfer.js +4 -3
  7. package/dist/facts/claude-memory.js +6 -1
  8. package/dist/facts/distill.js +4 -1
  9. package/dist/facts/handoff.js +6 -1
  10. package/dist/facts/known.js +2 -0
  11. package/dist/facts/start.js +1 -1
  12. package/dist/graph/query/answer.js +1 -1
  13. package/dist/graph/query/path.js +69 -37
  14. package/dist/graph/query/suggest.js +1 -2
  15. package/dist/hooks/compact.js +7 -1
  16. package/dist/hooks/session-start.js +1 -1
  17. package/dist/hooks/start-legacy.js +1 -1
  18. package/dist/store/adapter.d.ts +4 -2
  19. package/dist/store/each.d.ts +2 -4
  20. package/dist/store/each.js +1 -0
  21. package/dist/store/mongo/graph.d.ts +2 -1
  22. package/dist/store/mongo/graph.js +17 -2
  23. package/dist/store/mongo/index.d.ts +4 -2
  24. package/dist/store/mongo/index.js +8 -2
  25. package/dist/store/mongo/observations.d.ts +2 -1
  26. package/dist/store/mongo/observations.js +25 -3
  27. package/dist/store/postgres/graph.d.ts +2 -1
  28. package/dist/store/postgres/graph.js +22 -7
  29. package/dist/store/postgres/index.d.ts +4 -2
  30. package/dist/store/postgres/index.js +11 -7
  31. package/dist/store/postgres/insert.js +3 -1
  32. package/dist/store/postgres/meta.d.ts +1 -0
  33. package/dist/store/postgres/meta.js +4 -0
  34. package/dist/store/postgres/observations.d.ts +3 -2
  35. package/dist/store/postgres/observations.js +40 -6
  36. package/dist/store/postgres/rows.js +1 -1
  37. package/dist/store/scoped-store.js +4 -0
  38. package/dist/store/sqlite/graph.d.ts +2 -1
  39. package/dist/store/sqlite/graph.js +19 -2
  40. package/dist/store/sqlite/index.d.ts +4 -2
  41. package/dist/store/sqlite/index.js +8 -2
  42. package/dist/store/sqlite/observations.d.ts +2 -1
  43. package/dist/store/sqlite/observations.js +32 -5
  44. package/dist/types.d.ts +9 -0
  45. package/package.json +6 -8
package/CHANGELOG.md CHANGED
@@ -1,5 +1,25 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.12.4
4
+
5
+ ### Fixed
6
+
7
+ - **`find_usages` finds a path between two symbols anywhere in the code graph.** The search used to load only
8
+ the first 20,000 links of a project, so on a large repository it often said there was no path when there was
9
+ one. It now walks out from both ends and asks the database only for the links it needs.
10
+ - **Each request's memories are saved once.** The last request of a chat is read again on the next save, and it
11
+ was embedded and written again even when nothing in it had changed. It is now skipped.
12
+
13
+ ### Changed
14
+
15
+ - **claude-db downloads far less from a remote database.** Memories are read without their vectors unless a copy
16
+ needs them, and without their text where only titles are shown. Counting facts, finding imported Claude
17
+ memories and finding rows that need a vector now happen in the database instead of reading every row. This
18
+ matters most on a hosted Postgres that bills for data sent out, such as Neon.
19
+ - **`graphology` and `graphology-shortest-path` are no longer dependencies.**
20
+
21
+ By @alok-debnath (#36).
22
+
3
23
  ## 0.12.3
4
24
 
5
25
  ### Fixed
@@ -1,3 +1,4 @@
1
+ import { createHash } from 'node:crypto';
1
2
  import { mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
2
3
  import { dirname, join } from 'node:path';
3
4
  import { CONFIG_DIR } from '../config/index.js';
@@ -23,8 +24,12 @@ export async function flushSession(ctx, sessionId, project, transcriptPath, rebu
23
24
  writeCursor(sessionId, nextOffset);
24
25
  return { observations: 0, summary: null, lastReply };
25
26
  }
26
- await embedObservations(ctx, observations);
27
- await ctx.store.insertObservations(observations);
27
+ const written = rebuild ? new Map() : readWritten(sessionId);
28
+ const prints = new Map(observations.map((obs) => [obs.id, printOf(obs)]));
29
+ const fresh = observations.filter((obs) => written.get(obs.id) !== prints.get(obs.id));
30
+ await embedObservations(ctx, fresh);
31
+ await ctx.store.insertObservations(fresh);
32
+ writeWritten(sessionId, prints);
28
33
  await closeLandedWork(ctx.store, project);
29
34
  const previous = rebuild ? null : await ctx.store.getSession(sessionId);
30
35
  const summary = summarize(observations, previous?.summary);
@@ -100,6 +105,27 @@ function cursorName(sessionId) {
100
105
  function cursorPath(sessionId) {
101
106
  return join(CONFIG_DIR, 'cursors', cursorName(sessionId));
102
107
  }
108
+ const writtenPath = (sessionId) => cursorPath(sessionId).replace(/\.offset$/, '.written');
109
+ const printOf = (obs) => createHash('sha256').update(JSON.stringify(obs)).digest('base64url');
110
+ function readWritten(sessionId) {
111
+ try {
112
+ const parsed = JSON.parse(readFileSync(writtenPath(sessionId), 'utf8'));
113
+ if (!parsed || typeof parsed !== 'object')
114
+ return new Map();
115
+ return new Map(Object.entries(parsed).filter((entry) => typeof entry[1] === 'string'));
116
+ }
117
+ catch {
118
+ return new Map();
119
+ }
120
+ }
121
+ function writeWritten(sessionId, prints) {
122
+ const path = writtenPath(sessionId);
123
+ try {
124
+ mkdirSync(dirname(path), { recursive: true });
125
+ writeFileSync(path, JSON.stringify(Object.fromEntries(prints)), 'utf8');
126
+ }
127
+ catch { }
128
+ }
103
129
  function readCursor(sessionId) {
104
130
  try {
105
131
  const value = Number.parseInt(readFileSync(cursorPath(sessionId), 'utf8'), 10);
@@ -119,10 +145,12 @@ function writeCursor(sessionId, offset) {
119
145
  }
120
146
  export function resetCursor(sessionId) {
121
147
  writeCursor(sessionId, 0);
148
+ rmSync(writtenPath(sessionId), { force: true });
122
149
  }
123
150
  export function clearCursor(sessionId) {
124
151
  rmSync(cursorPath(sessionId), { force: true });
125
152
  rmSync(cursorPath(sessionId).replace(/\.offset$/, '.shown'), { force: true });
153
+ rmSync(writtenPath(sessionId), { force: true });
126
154
  }
127
155
  export function sweepCursors() {
128
156
  const dir = join(CONFIG_DIR, 'cursors');
@@ -136,7 +164,7 @@ export function sweepCursors() {
136
164
  const live = new Set(sessionIdsOnDisk().map((id) => id.replace(/[^\w-]/g, '_')));
137
165
  let removed = 0;
138
166
  for (const file of files) {
139
- const match = /^(.+)\.(offset|shown)$/.exec(file);
167
+ const match = /^(.+)\.(offset|shown|written)$/.exec(file);
140
168
  if (!match?.[1] || live.has(match[1]))
141
169
  continue;
142
170
  rmSync(join(dir, file), { force: true });
@@ -43,6 +43,7 @@ export async function closeLandedWork(store, project) {
43
43
  ...(after === undefined ? {} : { after }),
44
44
  ...(afterId === undefined ? {} : { afterId }),
45
45
  limit: OPEN_LIMIT,
46
+ bodies: false,
46
47
  });
47
48
  if (batch.length === 0)
48
49
  break;
@@ -61,6 +62,12 @@ export async function closeLandedWork(store, project) {
61
62
  return store.closeObservations(landed);
62
63
  }
63
64
  export async function openWork(store, project) {
64
- const open = await store.list({ project, status: 'open', limit: OPEN_LIMIT, newest: true });
65
+ const open = await store.list({
66
+ project,
67
+ status: 'open',
68
+ limit: OPEN_LIMIT,
69
+ newest: true,
70
+ bodies: false,
71
+ });
65
72
  return open.sort((a, b) => b.createdAt - a.createdAt);
66
73
  }
@@ -18,6 +18,7 @@ export async function gatherCatchup(ctx, project, current, now = Date.now()) {
18
18
  sessionId: FACT_SESSION,
19
19
  newest: true,
20
20
  limit: FACTS_SCANNED,
21
+ bodies: false,
21
22
  });
22
23
  const kept = onePerKey(facts.filter((obs) => obs.status !== 'replaced'));
23
24
  const todos = kept
@@ -18,13 +18,13 @@ export async function cmdSync(argv) {
18
18
  try {
19
19
  await remote.init();
20
20
  const localStatus = new Map();
21
- await eachRemoteObservation(local.store, (batch) => {
21
+ await eachRemoteObservation(local.store, false, (batch) => {
22
22
  for (const obs of batch)
23
23
  localStatus.set(obs.id, obs.status ?? 'done');
24
24
  });
25
25
  const remoteStatus = new Map();
26
26
  let pulled = 0;
27
- await eachRemoteObservation(remote, async (batch) => {
27
+ await eachRemoteObservation(remote, true, async (batch) => {
28
28
  for (const obs of batch)
29
29
  remoteStatus.set(obs.id, obs.status ?? 'done');
30
30
  const fresh = batch.filter((obs) => !localStatus.has(obs.id));
@@ -35,7 +35,7 @@ export async function cmdSync(argv) {
35
35
  await local.store.insertObservations(fresh);
36
36
  });
37
37
  let pushed = 0;
38
- await eachRemoteObservation(local.store, async (batch) => {
38
+ await eachRemoteObservation(local.store, true, async (batch) => {
39
39
  const fresh = batch.filter((obs) => !remoteStatus.has(obs.id));
40
40
  if (fresh.length === 0)
41
41
  return;
@@ -139,7 +139,7 @@ async function reconcileSession(session, from, to) {
139
139
  }
140
140
  return false;
141
141
  }
142
- async function eachRemoteObservation(store, visit) {
142
+ async function eachRemoteObservation(store, vectors, visit) {
143
143
  let after;
144
144
  let afterId;
145
145
  for (;;) {
@@ -147,6 +147,7 @@ async function eachRemoteObservation(store, visit) {
147
147
  ...(after === undefined ? {} : { after }),
148
148
  ...(afterId === undefined ? {} : { afterId }),
149
149
  limit: BATCH,
150
+ vectors,
150
151
  });
151
152
  if (batch.length === 0)
152
153
  return;
@@ -150,11 +150,11 @@ export async function cmdReembed(argv = []) {
150
150
  migrated = await ctx.store.migrateVectorDims(embedder.dimensions);
151
151
  }
152
152
  let updated = 0;
153
- let skipped = 0;
154
153
  const filter = scoped ? { project } : {};
155
- const scanned = await eachObservation(ctx, filter, async (batch) => {
154
+ const scanned = await ctx.store.count(filter);
155
+ const vectorStale = background ? {} : { embedder: embedder.id };
156
+ await eachObservation(ctx, { ...filter, vectorStale }, async (batch) => {
156
157
  const stale = batch.filter((obs) => !obs.embedding?.length || (!background && obs.embedder !== embedder.id));
157
- skipped += batch.length - stale.length;
158
158
  if (stale.length === 0)
159
159
  return;
160
160
  await embedObservations(ctx, stale);
@@ -165,6 +165,7 @@ export async function cmdReembed(argv = []) {
165
165
  });
166
166
  if (!background)
167
167
  process.stderr.write('\r');
168
+ const skipped = scanned - updated;
168
169
  if (migrated) {
169
170
  console.log(`Rebuilt vector storage at ${embedder.dimensions}d.`);
170
171
  }
@@ -88,7 +88,12 @@ async function importedFacts(ctx, project) {
88
88
  const mine = [machineTag(), originTag(project)];
89
89
  const rows = [];
90
90
  for (const owner of [project, youScope()]) {
91
- const batch = await ctx.store.list({ project: owner, sessionId: FACT_SESSION, limit: 5000 });
91
+ const batch = await ctx.store.list({
92
+ project: owner,
93
+ sessionId: FACT_SESSION,
94
+ tag: CLAUDE_MEMORY_TAG,
95
+ limit: 5000,
96
+ });
92
97
  rows.push(...batch.filter((obs) => obs.tags.includes(CLAUDE_MEMORY_TAG) && mine.every((tag) => obs.tags.includes(tag))));
93
98
  }
94
99
  return rows;
@@ -128,6 +128,9 @@ function clip(text, max) {
128
128
  return text.length <= max ? text : `${text.slice(0, max)}…`;
129
129
  }
130
130
  export async function factCounts(ctx, project) {
131
- const count = async (owner) => (await ctx.store.list({ project: owner, sessionId: FACT_SESSION, limit: 100_000 })).filter((obs) => obs.status !== 'replaced').length;
131
+ const count = async (owner) => {
132
+ const scope = { project: owner, sessionId: FACT_SESSION };
133
+ return ((await ctx.store.count(scope)) - (await ctx.store.count({ ...scope, status: 'replaced' })));
134
+ };
132
135
  return { project: await count(project), you: await count(youScope()) };
133
136
  }
@@ -19,11 +19,16 @@ export async function newestHandoff(ctx, project, now = Date.now()) {
19
19
  sessionId: MANUAL_SESSION,
20
20
  newest: true,
21
21
  limit: MANUAL_NOTES_SCANNED,
22
+ bodies: false,
22
23
  });
23
24
  const current = (obs) => obs.status !== 'replaced' &&
24
25
  obs.tags.includes(HANDOFF_TAG) &&
25
26
  now - obs.createdAt <= HANDOFF_MAX_AGE_MS;
26
- return notes.find(current) ?? null;
27
+ const found = notes.find(current);
28
+ if (!found)
29
+ return null;
30
+ const [full] = await ctx.store.getObservations([found.id]);
31
+ return full ?? null;
27
32
  }
28
33
  function clipLine(line) {
29
34
  return line.length > HANDOFF_LINE_CHARS
@@ -14,6 +14,7 @@ export async function existingFacts(ctx, project) {
14
14
  sessionId: FACT_SESSION,
15
15
  newest: true,
16
16
  limit: EXISTING_LIMIT,
17
+ bodies: false,
17
18
  });
18
19
  for (const obs of onePerKey(rows.filter((row) => row.status !== 'replaced'))) {
19
20
  const key = factKey(obs);
@@ -31,6 +32,7 @@ export async function notedByUser(ctx, project) {
31
32
  sessionId: MANUAL_SESSION,
32
33
  newest: true,
33
34
  limit: EXISTING_LIMIT,
35
+ bodies: false,
34
36
  });
35
37
  return rows
36
38
  .filter((row) => row.status !== 'replaced' && !row.tags.includes(HANDOFF_TAG))
@@ -13,7 +13,7 @@ const UNCOMMITTED = 3;
13
13
  const ORDER = ['rule', 'decision', 'deadend', 'fact'];
14
14
  export async function startFacts(ctx, project, hidden = new Set(), current) {
15
15
  const visible = (obs) => obs.status !== 'replaced' && !hidden.has(obs.id) && !seenByClaude(obs, project);
16
- const list = async (owner, sessionId, limit) => (await ctx.store.list({ project: owner, sessionId, newest: true, limit })).filter(visible);
16
+ const list = async (owner, sessionId, limit) => (await ctx.store.list({ project: owner, sessionId, newest: true, limit, bodies: false })).filter(visible);
17
17
  const you = onePerKey(await list(youScope(), FACT_SESSION, 50));
18
18
  const facts = onePerKey(await list(project, FACT_SESSION, 300));
19
19
  const handoff = await newestHandoff(ctx, project);
@@ -10,7 +10,7 @@ export function addText(answer, root) {
10
10
  return;
11
11
  answer.text = unlinkedText(root, answer.symbol, answer.definitions, answer.inbound);
12
12
  }
13
- export const hasGraph = async (store, project) => (await store.scannedFiles(project)).length > 0;
13
+ export const hasGraph = async (store, project) => (await store.scannedFiles(project, 1)).length > 0;
14
14
  export async function answerQuery(request) {
15
15
  const { store, root, project, query } = request;
16
16
  const refreshed = request.refresh ? await refreshGraph(store, root, project) : [];
@@ -1,48 +1,80 @@
1
1
  const MAX_EDGES = 20000;
2
- const MAX_SYMBOLS = 20000;
2
+ const MAX_ENDPOINTS = 5000;
3
+ const UNRESOLVED = 'name:';
4
+ const nodeFor = (id, name) => (id ? id : `${UNRESOLVED}${name}`);
5
+ function sideFrom(nodes) {
6
+ return {
7
+ parent: new Map(nodes.map((node) => [node, null])),
8
+ depth: new Map(nodes.map((node) => [node, 0])),
9
+ frontier: nodes,
10
+ };
11
+ }
12
+ async function endpoints(store, project, name) {
13
+ const symbols = await store.findSymbols({ project, name, limit: MAX_ENDPOINTS });
14
+ return [...new Set(symbols.map((symbol) => symbol.id)), `${UNRESOLVED}${name}`];
15
+ }
16
+ function edgesOf(store, project, frontier, forward, limit) {
17
+ const ids = frontier.filter((node) => !node.startsWith(UNRESOLVED));
18
+ const names = frontier
19
+ .filter((node) => node.startsWith(UNRESOLVED))
20
+ .map((node) => node.slice(UNRESOLVED.length));
21
+ return store.findEdges(forward
22
+ ? { project, srcIds: ids, unresolvedSrcNames: names, limit }
23
+ : { project, dstIds: ids, unresolvedDstNames: names, limit });
24
+ }
25
+ function chain(side, from) {
26
+ const nodes = [];
27
+ for (let node = from; node; node = side.parent.get(node)) {
28
+ nodes.push(node);
29
+ }
30
+ return nodes;
31
+ }
3
32
  export async function shortestPath(store, project, from, to) {
4
33
  if (!from || !to)
5
34
  return [];
6
35
  if (from === to)
7
36
  return [from];
8
- const graphology = (await import('graphology'));
9
- const Graph = graphology.default;
10
- const { bidirectional } = (await import('graphology-shortest-path/unweighted.js'));
11
- const [symbols, edges] = await Promise.all([
12
- store.findSymbols({ project, limit: MAX_SYMBOLS }),
13
- store.findEdges({ project, limit: MAX_EDGES }),
14
- ]);
15
37
  const label = new Map();
16
- for (const symbol of symbols)
17
- label.set(symbol.id, symbol.name);
18
- const graph = new Graph({ type: 'directed', multi: false, allowSelfLoops: false });
19
- const nodeFor = (id, name) => (id ? id : `name:${name}`);
20
- for (const edge of edges) {
21
- const src = nodeFor(edge.srcId, edge.srcName);
22
- const dst = nodeFor(edge.dstId, edge.dstName);
23
- if (src === dst)
24
- continue;
25
- graph.mergeDirectedEdge(src, dst);
26
- }
27
- const endpoints = (name) => {
28
- const ids = symbols.filter((symbol) => symbol.name === name).map((symbol) => symbol.id);
29
- const nodes = ids.filter((id) => graph.hasNode(id));
30
- const unresolved = `name:${name}`;
31
- if (graph.hasNode(unresolved))
32
- nodes.push(unresolved);
33
- return nodes;
34
- };
35
- let best = null;
36
- for (const start of endpoints(from)) {
37
- for (const end of endpoints(to)) {
38
- if (start === end)
38
+ const ahead = sideFrom(await endpoints(store, project, from));
39
+ const behind = sideFrom(await endpoints(store, project, to));
40
+ for (const node of ahead.frontier)
41
+ label.set(node, from);
42
+ for (const node of behind.frontier)
43
+ label.set(node, to);
44
+ let spent = 0;
45
+ while (ahead.frontier.length > 0 && behind.frontier.length > 0) {
46
+ const forward = ahead.frontier.length <= behind.frontier.length;
47
+ const [mine, other] = forward ? [ahead, behind] : [behind, ahead];
48
+ const current = new Set(mine.frontier);
49
+ const edges = await edgesOf(store, project, mine.frontier, forward, MAX_EDGES - spent + 1);
50
+ spent += edges.length;
51
+ if (spent > MAX_EDGES)
52
+ return [];
53
+ const next = [];
54
+ let meet = null;
55
+ for (const edge of edges) {
56
+ const src = nodeFor(edge.srcId, edge.srcName);
57
+ const dst = nodeFor(edge.dstId, edge.dstName);
58
+ if (src === dst)
59
+ continue;
60
+ if (!label.has(src))
61
+ label.set(src, edge.srcName);
62
+ const [near, far] = forward ? [src, dst] : [dst, src];
63
+ if (!current.has(near) || mine.parent.has(far))
39
64
  continue;
40
- const found = bidirectional(graph, start, end);
41
- if (found && (!best || found.length < best.length))
42
- best = found;
65
+ mine.parent.set(far, near);
66
+ mine.depth.set(far, (mine.depth.get(near) ?? 0) + 1);
67
+ next.push(far);
68
+ const reached = other.depth.get(far);
69
+ if (reached !== undefined && (meet === null || reached < (other.depth.get(meet) ?? 0))) {
70
+ meet = far;
71
+ }
43
72
  }
73
+ if (meet !== null) {
74
+ const nodes = [...chain(ahead, meet).reverse(), ...chain(behind, meet).slice(1)];
75
+ return nodes.map((node) => label.get(node) ?? node.replace(/^name:/, ''));
76
+ }
77
+ mine.frontier = next;
44
78
  }
45
- if (!best)
46
- return [];
47
- return best.map((node) => label.get(node) ?? node.replace(/^name:/, ''));
79
+ return [];
48
80
  }
@@ -44,8 +44,7 @@ export async function suggestFor(store, project, symbol) {
44
44
  if (!symbol)
45
45
  return [];
46
46
  try {
47
- const symbols = await store.findSymbols({ project, limit: CANDIDATES });
48
- return nearest(symbols.map((entry) => entry.name), symbol);
47
+ return nearest(await store.symbolNames(project, CANDIDATES), symbol);
49
48
  }
50
49
  catch {
51
50
  return [];
@@ -14,7 +14,13 @@ export async function recoverAfterCompact(ctx, project, sessionId, transcriptPat
14
14
  if (!capturingDisabled(ctx.config.capture.scripted)) {
15
15
  await flushSession(ctx, sessionId, project, transcriptPath);
16
16
  }
17
- const observations = await ctx.store.list({ project, sessionId, newest: true, limit: SCANNED });
17
+ const observations = await ctx.store.list({
18
+ project,
19
+ sessionId,
20
+ newest: true,
21
+ limit: SCANNED,
22
+ bodies: false,
23
+ });
18
24
  return renderEarlierInChat(observations);
19
25
  }
20
26
  export function renderEarlierInChat(newestFirst) {
@@ -73,7 +73,7 @@ await runHook(async () => {
73
73
  parts.push("Search this project's full history with the memory MCP tools before " +
74
74
  'asking the user to re-explain prior decisions.');
75
75
  }
76
- const scanned = (await ctx.store.scannedFiles(project)).length > 0;
76
+ const scanned = (await ctx.store.scannedFiles(project, 1)).length > 0;
77
77
  if (!scanned)
78
78
  parts.push(SCAN_HINT);
79
79
  const update = ctx.config.updates === 'off' ? null : updateNotice();
@@ -24,7 +24,7 @@ export async function legacyBlock(ctx, project, restored) {
24
24
  for (const obs of open.slice(0, 3))
25
25
  lines.push(`- ${obs.title}`);
26
26
  }
27
- const rules = (await ctx.store.list({ project, kind: 'preference', limit: 100, newest: true }))
27
+ const rules = (await ctx.store.list({ project, kind: 'preference', limit: 100, newest: true, bodies: false }))
28
28
  .filter((obs) => obs.status !== 'replaced' && !restored.has(obs.id))
29
29
  .sort((a, b) => {
30
30
  const manual = Number(b.sessionId === 'manual') - Number(a.sessionId === 'manual');
@@ -1,4 +1,4 @@
1
- import type { CodeEdge, CodeSymbol, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../types.js';
1
+ import type { CodeEdge, CodeSymbol, CountFilter, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../types.js';
2
2
  export interface ProjectSummary {
3
3
  project: string;
4
4
  observations: number;
@@ -22,6 +22,7 @@ export interface MemoryStore {
22
22
  storedVectorDims?(): number | null;
23
23
  remove(filter: RemoveFilter): Promise<number>;
24
24
  list(filter: ListFilter): Promise<Observation[]>;
25
+ count(filter: CountFilter): Promise<number>;
25
26
  listProjects(): Promise<ProjectSummary[]>;
26
27
  inventory(): Promise<string[]>;
27
28
  searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
@@ -35,8 +36,9 @@ export interface MemoryStore {
35
36
  files: ScannedFile[];
36
37
  }): Promise<void>;
37
38
  findSymbols(filter: SymbolFilter): Promise<CodeSymbol[]>;
39
+ symbolNames(project: string, limit: number): Promise<string[]>;
38
40
  findEdges(filter: EdgeFilter): Promise<CodeEdge[]>;
39
- scannedFiles(project: string): Promise<ScannedFile[]>;
41
+ scannedFiles(project: string, limit?: number): Promise<ScannedFile[]>;
40
42
  removeGraph(project: string, files?: string[]): Promise<number>;
41
43
  }
42
44
  export declare function isWholeScope(filter: RemoveFilter): boolean;
@@ -1,5 +1,3 @@
1
1
  import type { RecallContext } from '../context.js';
2
- import type { Observation } from '../types.js';
3
- export declare function eachObservation(ctx: RecallContext, filter: {
4
- project?: string;
5
- }, visit: (batch: Observation[]) => Promise<void> | void): Promise<number>;
2
+ import type { ListFilter, Observation } from '../types.js';
3
+ export declare function eachObservation(ctx: RecallContext, filter: Pick<ListFilter, 'project' | 'vectorStale'>, visit: (batch: Observation[]) => Promise<void> | void): Promise<number>;
@@ -9,6 +9,7 @@ export async function eachObservation(ctx, filter, visit) {
9
9
  ...(after === undefined ? {} : { after }),
10
10
  ...(afterId === undefined ? {} : { afterId }),
11
11
  limit: PAGE,
12
+ vectors: true,
12
13
  });
13
14
  if (batch.length === 0)
14
15
  return total;
@@ -7,6 +7,7 @@ export declare function upsertGraph(symbols: Collection<SymbolDoc>, edges: Colle
7
7
  files: ScannedFile[];
8
8
  }): Promise<void>;
9
9
  export declare function findSymbols(symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, filter: SymbolFilter): Promise<CodeSymbol[]>;
10
+ export declare function symbolNames(symbols: Collection<SymbolDoc>, project: string, limit: number): Promise<string[]>;
10
11
  export declare function findEdges(symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, filter: EdgeFilter): Promise<CodeEdge[]>;
11
- export declare function scannedFiles(symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, project: string): Promise<ScannedFile[]>;
12
+ export declare function scannedFiles(symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, project: string, limit?: number): Promise<ScannedFile[]>;
12
13
  export declare function removeGraph(symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, project: string, files?: string[]): Promise<number>;
@@ -24,6 +24,14 @@ export async function findSymbols(symbols, edges, scanned, filter) {
24
24
  .toArray();
25
25
  return docs.map(toSymbol);
26
26
  }
27
+ export async function symbolNames(symbols, project, limit) {
28
+ const docs = await symbols
29
+ .find({ project }, { projection: { name: 1 } })
30
+ .sort({ file: 1, line: 1 })
31
+ .limit(limit)
32
+ .toArray();
33
+ return docs.map((doc) => doc.name);
34
+ }
27
35
  export async function findEdges(symbols, edges, scanned, filter) {
28
36
  const query = { project: filter.project };
29
37
  const alternatives = [];
@@ -33,6 +41,12 @@ export async function findEdges(symbols, edges, scanned, filter) {
33
41
  if (filter.dstIds && filter.dstIds.length > 0) {
34
42
  alternatives.push({ dstId: { $in: filter.dstIds } });
35
43
  }
44
+ if (filter.unresolvedSrcNames && filter.unresolvedSrcNames.length > 0) {
45
+ alternatives.push({ srcId: { $in: ['', null] }, srcName: { $in: filter.unresolvedSrcNames } });
46
+ }
47
+ if (filter.unresolvedDstNames && filter.unresolvedDstNames.length > 0) {
48
+ alternatives.push({ dstId: { $in: ['', null] }, dstName: { $in: filter.unresolvedDstNames } });
49
+ }
36
50
  if (alternatives.length > 0)
37
51
  query['$or'] = alternatives;
38
52
  if (filter.dstName !== undefined) {
@@ -47,8 +61,9 @@ export async function findEdges(symbols, edges, scanned, filter) {
47
61
  .toArray();
48
62
  return docs.map(toEdge);
49
63
  }
50
- export async function scannedFiles(symbols, edges, scanned, project) {
51
- const docs = await scanned.find({ project }).toArray();
64
+ export async function scannedFiles(symbols, edges, scanned, project, limit) {
65
+ const cursor = scanned.find({ project });
66
+ const docs = await (limit === undefined ? cursor : cursor.limit(limit)).toArray();
52
67
  return docs.map((doc) => ({
53
68
  project: doc.project,
54
69
  path: doc.path,
@@ -1,4 +1,4 @@
1
- import type { CodeEdge, CodeSymbol, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
1
+ import type { CodeEdge, CodeSymbol, CountFilter, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
2
2
  import type { MemoryStore, ProjectSummary } from '../adapter.js';
3
3
  export declare class MongoStore implements MemoryStore {
4
4
  private readonly client;
@@ -27,6 +27,7 @@ export declare class MongoStore implements MemoryStore {
27
27
  getObservations(ids: string[]): Promise<Observation[]>;
28
28
  remove(filter: RemoveFilter): Promise<number>;
29
29
  list(filter: ListFilter): Promise<Observation[]>;
30
+ count(filter: CountFilter): Promise<number>;
30
31
  listProjects(): Promise<ProjectSummary[]>;
31
32
  inventory(): Promise<string[]>;
32
33
  searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
@@ -41,7 +42,8 @@ export declare class MongoStore implements MemoryStore {
41
42
  files: ScannedFile[];
42
43
  }): Promise<void>;
43
44
  findSymbols(filter: SymbolFilter): Promise<CodeSymbol[]>;
45
+ symbolNames(project: string, limit: number): Promise<string[]>;
44
46
  findEdges(filter: EdgeFilter): Promise<CodeEdge[]>;
45
- scannedFiles(project: string): Promise<ScannedFile[]>;
47
+ scannedFiles(project: string, limit?: number): Promise<ScannedFile[]>;
46
48
  removeGraph(project: string, files?: string[]): Promise<number>;
47
49
  }
@@ -89,6 +89,9 @@ export class MongoStore {
89
89
  async list(filter) {
90
90
  return observationsOps.list(this.observations, this.sessions, this.symbols, this.edges, this.scanned, filter);
91
91
  }
92
+ async count(filter) {
93
+ return observationsOps.count(this.observations, filter);
94
+ }
92
95
  async listProjects() {
93
96
  return observationsOps.listProjects(this.observations);
94
97
  }
@@ -121,11 +124,14 @@ export class MongoStore {
121
124
  async findSymbols(filter) {
122
125
  return graphOps.findSymbols(this.symbols, this.edges, this.scanned, filter);
123
126
  }
127
+ async symbolNames(project, limit) {
128
+ return graphOps.symbolNames(this.symbols, project, limit);
129
+ }
124
130
  async findEdges(filter) {
125
131
  return graphOps.findEdges(this.symbols, this.edges, this.scanned, filter);
126
132
  }
127
- async scannedFiles(project) {
128
- return graphOps.scannedFiles(this.symbols, this.edges, this.scanned, project);
133
+ async scannedFiles(project, limit) {
134
+ return graphOps.scannedFiles(this.symbols, this.edges, this.scanned, project, limit);
129
135
  }
130
136
  async removeGraph(project, files) {
131
137
  return graphOps.removeGraph(this.symbols, this.edges, this.scanned, project, files);
@@ -1,10 +1,11 @@
1
1
  import type { Collection } from './driver.js';
2
- import type { ListFilter, Observation, RemoveFilter } from '../../types.js';
2
+ import type { CountFilter, ListFilter, Observation, RemoveFilter } from '../../types.js';
3
3
  import type { ProjectSummary } from '../adapter.js';
4
4
  import type { EdgeDoc, ObservationDoc, ScannedFileDoc, SessionDoc, SymbolDoc } from './docs.js';
5
5
  export declare function insertObservations(collection: Collection<ObservationDoc>, sessions: Collection<SessionDoc>, symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, rows: Observation[]): Promise<void>;
6
6
  export declare function getObservations(observations: Collection<ObservationDoc>, sessions: Collection<SessionDoc>, symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, ids: string[]): Promise<Observation[]>;
7
7
  export declare function remove(observations: Collection<ObservationDoc>, sessions: Collection<SessionDoc>, symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, filter: RemoveFilter): Promise<number>;
8
+ export declare function count(observations: Collection<ObservationDoc>, filter: CountFilter): Promise<number>;
8
9
  export declare function list(observations: Collection<ObservationDoc>, sessions: Collection<SessionDoc>, symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, filter: ListFilter): Promise<Observation[]>;
9
10
  export declare function listProjects(observations: Collection<ObservationDoc>): Promise<ProjectSummary[]>;
10
11
  export declare function closeObservations(observations: Collection<ObservationDoc>, ids: string[]): Promise<number>;
@@ -60,9 +60,7 @@ export async function remove(observations, sessions, symbols, edges, scanned, fi
60
60
  }
61
61
  return count;
62
62
  }
63
- export async function list(observations, sessions, symbols, edges, scanned, filter) {
64
- if (noProjects(filter.project))
65
- return [];
63
+ function scopeQuery(filter) {
66
64
  const query = {};
67
65
  Object.assign(query, projectMatch(filter.project));
68
66
  Object.assign(query, sessionMatch(filter.sessionId, filter.excludeSessions));
@@ -70,6 +68,19 @@ export async function list(observations, sessions, symbols, edges, scanned, filt
70
68
  query['kind'] = filter.kind;
71
69
  if (filter.status)
72
70
  query['status'] = filter.status;
71
+ if (filter.tag)
72
+ query['tags'] = filter.tag;
73
+ return query;
74
+ }
75
+ export async function count(observations, filter) {
76
+ if (noProjects(filter.project))
77
+ return 0;
78
+ return observations.countDocuments(scopeQuery(filter));
79
+ }
80
+ export async function list(observations, sessions, symbols, edges, scanned, filter) {
81
+ if (noProjects(filter.project))
82
+ return [];
83
+ const query = scopeQuery(filter);
73
84
  if (filter.after !== undefined) {
74
85
  if (filter.afterId === undefined)
75
86
  query['createdAt'] = { $gt: filter.after };
@@ -79,6 +90,17 @@ export async function list(observations, sessions, symbols, edges, scanned, filt
79
90
  { createdAt: filter.after, _id: { $gt: filter.afterId } },
80
91
  ];
81
92
  }
93
+ if (filter.vectorStale) {
94
+ const { embedder } = filter.vectorStale;
95
+ query['$and'] = [
96
+ {
97
+ $or: [
98
+ { embedding: null },
99
+ embedder === undefined ? { embedder: null } : { embedder: { $ne: embedder } },
100
+ ],
101
+ },
102
+ ];
103
+ }
82
104
  const direction = filter.newest ? -1 : 1;
83
105
  const docs = await observations
84
106
  .find(query)
@@ -6,6 +6,7 @@ export declare function upsertGraph(pool: Pool, scan: {
6
6
  files: ScannedFile[];
7
7
  }): Promise<void>;
8
8
  export declare function findSymbols(pool: Pool, filter: SymbolFilter): Promise<CodeSymbol[]>;
9
+ export declare function symbolNames(pool: Pool, project: string, limit: number): Promise<string[]>;
9
10
  export declare function findEdges(pool: Pool, filter: EdgeFilter): Promise<CodeEdge[]>;
10
- export declare function scannedFiles(pool: Pool, project: string): Promise<ScannedFile[]>;
11
+ export declare function scannedFiles(pool: Pool, project: string, limit?: number): Promise<ScannedFile[]>;
11
12
  export declare function removeGraph(pool: Pool, project: string, files?: string[]): Promise<number>;
@@ -68,6 +68,8 @@ export async function upsertGraph(pool, scan) {
68
68
  client.release();
69
69
  }
70
70
  }
71
+ const SYMBOL_COLUMNS = 'id, name, kind, file, line, lang, signature';
72
+ const EDGE_COLUMNS = 'id, src_id, src_name, dst_id, dst_name, relation, confidence, score, file, line';
71
73
  export async function findSymbols(pool, filter) {
72
74
  const values = [filter.project];
73
75
  const conditions = ['project = $1'];
@@ -80,9 +82,13 @@ export async function findSymbols(pool, filter) {
80
82
  conditions.push(`file = $${values.length}`);
81
83
  }
82
84
  values.push(filter.limit ?? 500);
83
- const res = await pool.query(`SELECT * FROM symbols WHERE ${conditions.join(' AND ')}
85
+ const res = await pool.query(`SELECT ${SYMBOL_COLUMNS} FROM symbols WHERE ${conditions.join(' AND ')}
84
86
  ORDER BY file, line LIMIT $${values.length}`, values);
85
- return res.rows.map(toSymbol);
87
+ return res.rows.map((row) => toSymbol({ ...row, project: filter.project }));
88
+ }
89
+ export async function symbolNames(pool, project, limit) {
90
+ const res = await pool.query('SELECT name FROM symbols WHERE project = $1 ORDER BY file, line LIMIT $2', [project, limit]);
91
+ return res.rows.map((row) => row['name']);
86
92
  }
87
93
  export async function findEdges(pool, filter) {
88
94
  const values = [filter.project];
@@ -96,20 +102,29 @@ export async function findEdges(pool, filter) {
96
102
  values.push(ids);
97
103
  sides.push(`${column} = ANY($${values.length}::text[])`);
98
104
  }
105
+ for (const [side, names] of [
106
+ ['src', filter.unresolvedSrcNames],
107
+ ['dst', filter.unresolvedDstNames],
108
+ ]) {
109
+ if (!names || names.length === 0)
110
+ continue;
111
+ values.push(names);
112
+ sides.push(`(${side}_id = '' AND ${side}_name = ANY($${values.length}::text[]))`);
113
+ }
99
114
  let where = sides.length > 0 ? ` AND (${sides.join(' OR ')})` : '';
100
115
  if (filter.dstName !== undefined) {
101
116
  values.push(filter.dstName, `%.${filter.dstName}`);
102
117
  where += ` AND (dst_name = $${values.length - 1} OR dst_name LIKE $${values.length})`;
103
118
  }
104
119
  values.push(filter.limit ?? 2000);
105
- const res = await pool.query(`SELECT * FROM symbol_edges WHERE project = $1${where}
120
+ const res = await pool.query(`SELECT ${EDGE_COLUMNS} FROM symbol_edges WHERE project = $1${where}
106
121
  ORDER BY file, line, id LIMIT $${values.length}`, values);
107
- return res.rows.map(toEdge);
122
+ return res.rows.map((row) => toEdge({ ...row, project: filter.project }));
108
123
  }
109
- export async function scannedFiles(pool, project) {
110
- const res = await pool.query('SELECT * FROM scanned_files WHERE project = $1', [project]);
124
+ export async function scannedFiles(pool, project, limit) {
125
+ const res = await pool.query(`SELECT path, hash, scanned_at FROM scanned_files WHERE project = $1${limit === undefined ? '' : ' LIMIT $2'}`, limit === undefined ? [project] : [project, limit]);
111
126
  return res.rows.map((row) => ({
112
- project: row['project'],
127
+ project,
113
128
  path: row['path'],
114
129
  hash: row['hash'],
115
130
  scannedAt: Number(row['scanned_at']),
@@ -1,4 +1,4 @@
1
- import type { CodeEdge, CodeSymbol, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
1
+ import type { CodeEdge, CodeSymbol, CountFilter, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
2
2
  import type { MemoryStore, ProjectSummary } from '../adapter.js';
3
3
  export declare class PostgresStore implements MemoryStore {
4
4
  private readonly pool;
@@ -33,6 +33,7 @@ export declare class PostgresStore implements MemoryStore {
33
33
  getObservations(ids: string[]): Promise<Observation[]>;
34
34
  remove(filter: RemoveFilter): Promise<number>;
35
35
  list(filter: ListFilter): Promise<Observation[]>;
36
+ count(filter: CountFilter): Promise<number>;
36
37
  listProjects(): Promise<ProjectSummary[]>;
37
38
  inventory(): Promise<string[]>;
38
39
  searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
@@ -47,7 +48,8 @@ export declare class PostgresStore implements MemoryStore {
47
48
  files: ScannedFile[];
48
49
  }): Promise<void>;
49
50
  findSymbols(filter: SymbolFilter): Promise<CodeSymbol[]>;
51
+ symbolNames(project: string, limit: number): Promise<string[]>;
50
52
  findEdges(filter: EdgeFilter): Promise<CodeEdge[]>;
51
- scannedFiles(project: string): Promise<ScannedFile[]>;
53
+ scannedFiles(project: string, limit?: number): Promise<ScannedFile[]>;
52
54
  removeGraph(project: string, files?: string[]): Promise<number>;
53
55
  }
@@ -58,8 +58,7 @@ export class PostgresStore {
58
58
  await this.pool.query('ALTER TABLE observations DROP COLUMN IF EXISTS embedding');
59
59
  await this.pool.query(`ALTER TABLE observations ADD COLUMN embedding vector(${dims})`);
60
60
  await this.pool.query('UPDATE observations SET embedder = NULL');
61
- await this.pool.query(`CREATE INDEX IF NOT EXISTS idx_obs_embedding
62
- ON observations USING hnsw (embedding vector_cosine_ops)`);
61
+ await metaOps.indexEmbeddings(this.pool);
63
62
  this.vectorDims = await this.readVectorDims();
64
63
  this.warnedDims = false;
65
64
  await this.writeMeta();
@@ -75,8 +74,7 @@ export class PostgresStore {
75
74
  return false;
76
75
  if (this.vectorDims === null) {
77
76
  await this.pool.query(`ALTER TABLE observations ADD COLUMN IF NOT EXISTS embedding vector(${dims})`);
78
- await this.pool.query(`CREATE INDEX IF NOT EXISTS idx_obs_embedding
79
- ON observations USING hnsw (embedding vector_cosine_ops)`);
77
+ await metaOps.indexEmbeddings(this.pool);
80
78
  this.vectorDims = await this.readVectorDims();
81
79
  await this.writeMeta();
82
80
  }
@@ -125,7 +123,10 @@ export class PostgresStore {
125
123
  return observationsOps.remove(this.pool, filter);
126
124
  }
127
125
  async list(filter) {
128
- return observationsOps.list(this.pool, filter);
126
+ return observationsOps.list(this.pool, filter, this.vectorDims !== null);
127
+ }
128
+ async count(filter) {
129
+ return observationsOps.count(this.pool, filter);
129
130
  }
130
131
  async listProjects() {
131
132
  return observationsOps.listProjects(this.pool);
@@ -157,11 +158,14 @@ export class PostgresStore {
157
158
  async findSymbols(filter) {
158
159
  return graphOps.findSymbols(this.pool, filter);
159
160
  }
161
+ async symbolNames(project, limit) {
162
+ return graphOps.symbolNames(this.pool, project, limit);
163
+ }
160
164
  async findEdges(filter) {
161
165
  return graphOps.findEdges(this.pool, filter);
162
166
  }
163
- async scannedFiles(project) {
164
- return graphOps.scannedFiles(this.pool, project);
167
+ async scannedFiles(project, limit) {
168
+ return graphOps.scannedFiles(this.pool, project, limit);
165
169
  }
166
170
  async removeGraph(project, files) {
167
171
  return graphOps.removeGraph(this.pool, project, files);
@@ -8,6 +8,7 @@ export async function insertObservations(pool, ensureVectorColumn, observations,
8
8
  await client.query('BEGIN');
9
9
  for (const obs of observations) {
10
10
  const vectorStored = vectors && obs.embedding?.length === width;
11
+ const vectorKept = !obs.embedding?.length && obs.embedder != null;
11
12
  const columns = [
12
13
  'id',
13
14
  'session_id',
@@ -43,8 +44,9 @@ export async function insertObservations(pool, ensureVectorColumn, observations,
43
44
  const holes = values.map((_, i) => `$${i + 1}`).join(', ');
44
45
  const updates = columns
45
46
  .slice(1)
47
+ .filter((column) => !(vectorKept && column === 'embedder'))
46
48
  .map((column) => `${column} = EXCLUDED.${column}`)
47
- .concat(!vectorStored && (hasVectorColumn || vectors) ? ['embedding = NULL'] : [])
49
+ .concat(!vectorStored && !vectorKept && (hasVectorColumn || vectors) ? ['embedding = NULL'] : [])
48
50
  .join(', ');
49
51
  await client.query(`INSERT INTO observations (${columns.join(', ')}) VALUES (${holes})
50
52
  ON CONFLICT (id) DO UPDATE SET ${updates}`, values);
@@ -8,3 +8,4 @@ export declare function ensureTagsIndexed(pool: Pool): Promise<void>;
8
8
  export declare function readVectorDims(pool: Pool): Promise<number | null>;
9
9
  export declare function tryEnableVector(pool: Pool): Promise<boolean>;
10
10
  export declare function writeMeta(pool: Pool, vectorEnabled: boolean, vectorDims: number | null): Promise<void>;
11
+ export declare function indexEmbeddings(pool: Pool): Promise<void>;
@@ -65,3 +65,7 @@ export async function writeMeta(pool, vectorEnabled, vectorDims) {
65
65
  vector_enabled = EXCLUDED.vector_enabled,
66
66
  vector_dims = EXCLUDED.vector_dims`, [SCHEMA_VERSION, vectorEnabled, vectorDims]);
67
67
  }
68
+ export async function indexEmbeddings(pool) {
69
+ await pool.query(`CREATE INDEX IF NOT EXISTS idx_obs_embedding
70
+ ON observations USING hnsw (embedding vector_cosine_ops)`);
71
+ }
@@ -1,9 +1,10 @@
1
1
  import type { Pool } from './driver.js';
2
- import type { ListFilter, Observation, ObservationIndexEntry, RemoveFilter, TimelineQuery } from '../../types.js';
2
+ import type { CountFilter, ListFilter, Observation, ObservationIndexEntry, RemoveFilter, TimelineQuery } from '../../types.js';
3
3
  import type { ProjectSummary } from '../adapter.js';
4
4
  export declare function getObservations(pool: Pool, ids: string[]): Promise<Observation[]>;
5
5
  export declare function remove(pool: Pool, filter: RemoveFilter): Promise<number>;
6
- export declare function list(pool: Pool, filter: ListFilter): Promise<Observation[]>;
6
+ export declare function list(pool: Pool, filter: ListFilter, hasVectorColumn?: boolean): Promise<Observation[]>;
7
+ export declare function count(pool: Pool, filter: CountFilter): Promise<number>;
7
8
  export declare function listProjects(pool: Pool): Promise<ProjectSummary[]>;
8
9
  export declare function inventory(pool: Pool): Promise<string[]>;
9
10
  export declare function timeline(pool: Pool, query: TimelineQuery): Promise<ObservationIndexEntry[]>;
@@ -3,6 +3,13 @@ import { partitionIds } from '../../util/shortid.js';
3
3
  import { excludeSessionsClause, projectClause } from './filters.js';
4
4
  import { toIndexEntry, toObservation } from './rows.js';
5
5
  import { noProjects } from '../project-scope.js';
6
+ const COLUMNS = 'id, session_id, project, kind, title, body, files, tags, created_at, embedder, author, status';
7
+ const WITHOUT_BODY = COLUMNS.replace(' body,', '');
8
+ function listColumns(filter) {
9
+ if (filter.vectors)
10
+ return '*';
11
+ return filter.bodies === false ? WITHOUT_BODY : COLUMNS;
12
+ }
6
13
  export async function getObservations(pool, ids) {
7
14
  if (ids.length === 0)
8
15
  return [];
@@ -17,7 +24,7 @@ export async function getObservations(pool, ids) {
17
24
  values.push(`${prefix.replace(/[%_]/g, '')}%`);
18
25
  clauses.push(`id LIKE $${values.length}`);
19
26
  }
20
- const res = await pool.query(`SELECT * FROM observations WHERE ${clauses.join(' OR ')}`, values);
27
+ const res = await pool.query(`SELECT ${COLUMNS} FROM observations WHERE ${clauses.join(' OR ')}`, values);
21
28
  return res.rows.map(toObservation);
22
29
  }
23
30
  export async function remove(pool, filter) {
@@ -76,11 +83,8 @@ export async function remove(pool, filter) {
76
83
  client.release();
77
84
  }
78
85
  }
79
- export async function list(pool, filter) {
80
- if (noProjects(filter.project))
81
- return [];
86
+ function scopeConditions(filter, values) {
82
87
  const conditions = [];
83
- const values = [];
84
88
  const project = projectClause(filter.project, values);
85
89
  if (project)
86
90
  conditions.push(project);
@@ -99,6 +103,17 @@ export async function list(pool, filter) {
99
103
  values.push(filter.status);
100
104
  conditions.push(`status = $${values.length}`);
101
105
  }
106
+ if (filter.tag) {
107
+ values.push(JSON.stringify([filter.tag]));
108
+ conditions.push(`tags @> $${values.length}::jsonb`);
109
+ }
110
+ return conditions;
111
+ }
112
+ export async function list(pool, filter, hasVectorColumn = false) {
113
+ if (noProjects(filter.project))
114
+ return [];
115
+ const values = [];
116
+ const conditions = scopeConditions(filter, values);
102
117
  if (filter.after !== undefined) {
103
118
  if (filter.afterId === undefined) {
104
119
  values.push(filter.after);
@@ -109,13 +124,32 @@ export async function list(pool, filter) {
109
124
  conditions.push(`(created_at > $${values.length - 2} OR (created_at = $${values.length - 1} AND id > $${values.length}))`);
110
125
  }
111
126
  }
127
+ if (filter.vectorStale) {
128
+ const stale = hasVectorColumn
129
+ ? ['embedder IS NULL', 'embedding IS NULL']
130
+ : ['embedder IS NULL'];
131
+ if (filter.vectorStale.embedder !== undefined) {
132
+ values.push(filter.vectorStale.embedder);
133
+ stale.push(`embedder <> $${values.length}`);
134
+ }
135
+ conditions.push(`(${stale.join(' OR ')})`);
136
+ }
112
137
  values.push(filter.limit ?? 1000);
113
138
  const direction = filter.newest ? 'DESC' : 'ASC';
114
- const res = await pool.query(`SELECT * FROM observations
139
+ const res = await pool.query(`SELECT ${listColumns(filter)} FROM observations
115
140
  ${conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''}
116
141
  ORDER BY created_at ${direction}, id ${direction} LIMIT $${values.length}`, values);
117
142
  return res.rows.map(toObservation);
118
143
  }
144
+ export async function count(pool, filter) {
145
+ if (noProjects(filter.project))
146
+ return 0;
147
+ const values = [];
148
+ const conditions = scopeConditions(filter, values);
149
+ const res = await pool.query(`SELECT COUNT(*)::int AS n FROM observations
150
+ ${conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''}`, values);
151
+ return Number(res.rows[0]?.['n'] ?? 0);
152
+ }
119
153
  export async function listProjects(pool) {
120
154
  const res = await pool.query(`SELECT project, COUNT(*)::int AS n, MAX(created_at) AS last
121
155
  FROM observations GROUP BY project ORDER BY last DESC`);
@@ -35,7 +35,7 @@ export function toObservation(row) {
35
35
  project: row['project'],
36
36
  kind: row['kind'],
37
37
  title: row['title'],
38
- body: row['body'],
38
+ body: row['body'] ?? '',
39
39
  files: row['files'] ?? [],
40
40
  tags: row['tags'] ?? [],
41
41
  createdAt: Number(row['created_at']),
@@ -27,6 +27,10 @@ export function withProjectScope(inner, resolver) {
27
27
  const project = await expand(filter.project);
28
28
  return inner.list(project === undefined ? filter : { ...filter, project });
29
29
  },
30
+ async count(filter) {
31
+ const project = await expand(filter.project);
32
+ return inner.count(project === undefined ? filter : { ...filter, project });
33
+ },
30
34
  async searchKeyword(query) {
31
35
  const project = await expand(query.project);
32
36
  return inner.searchKeyword(project === undefined ? query : { ...query, project });
@@ -6,6 +6,7 @@ export declare function upsertGraph(db: DatabaseSync, scan: {
6
6
  files: ScannedFile[];
7
7
  }): Promise<void>;
8
8
  export declare function findSymbols(db: DatabaseSync, filter: SymbolFilter): Promise<CodeSymbol[]>;
9
+ export declare function symbolNames(db: DatabaseSync, project: string, limit: number): Promise<string[]>;
9
10
  export declare function findEdges(db: DatabaseSync, filter: EdgeFilter): Promise<CodeEdge[]>;
10
- export declare function scannedFiles(db: DatabaseSync, project: string): Promise<ScannedFile[]>;
11
+ export declare function scannedFiles(db: DatabaseSync, project: string, limit?: number): Promise<ScannedFile[]>;
11
12
  export declare function removeGraph(db: DatabaseSync, project: string, files?: string[]): Promise<number>;
@@ -45,6 +45,12 @@ export async function findSymbols(db, filter) {
45
45
  .all(...params);
46
46
  return rows.map(toSymbol);
47
47
  }
48
+ export async function symbolNames(db, project, limit) {
49
+ const rows = db
50
+ .prepare('SELECT name FROM symbols WHERE project = ? ORDER BY file, line LIMIT ?')
51
+ .all(project, limit);
52
+ return rows.map((row) => row['name']);
53
+ }
48
54
  export async function findEdges(db, filter) {
49
55
  const params = [filter.project];
50
56
  const sides = [];
@@ -57,6 +63,15 @@ export async function findEdges(db, filter) {
57
63
  sides.push(`${column} IN (${ids.map(() => '?').join(',')})`);
58
64
  params.push(...ids);
59
65
  }
66
+ for (const [side, names] of [
67
+ ['src', filter.unresolvedSrcNames],
68
+ ['dst', filter.unresolvedDstNames],
69
+ ]) {
70
+ if (!names || names.length === 0)
71
+ continue;
72
+ sides.push(`(${side}_id = '' AND ${side}_name IN (${names.map(() => '?').join(',')}))`);
73
+ params.push(...names);
74
+ }
60
75
  let where = sides.length > 0 ? ` AND (${sides.join(' OR ')})` : '';
61
76
  if (filter.dstName !== undefined) {
62
77
  where += ' AND (dst_name = ? OR dst_name LIKE ?)';
@@ -68,8 +83,10 @@ export async function findEdges(db, filter) {
68
83
  .all(...params);
69
84
  return rows.map(toEdge);
70
85
  }
71
- export async function scannedFiles(db, project) {
72
- const rows = db.prepare('SELECT * FROM scanned_files WHERE project = ?').all(project);
86
+ export async function scannedFiles(db, project, limit) {
87
+ const rows = (limit === undefined
88
+ ? db.prepare('SELECT * FROM scanned_files WHERE project = ?').all(project)
89
+ : db.prepare('SELECT * FROM scanned_files WHERE project = ? LIMIT ?').all(project, limit));
73
90
  return rows.map((row) => ({
74
91
  project: row['project'],
75
92
  path: row['path'],
@@ -1,4 +1,4 @@
1
- import type { CodeEdge, CodeSymbol, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
1
+ import type { CodeEdge, CodeSymbol, CountFilter, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
2
2
  import type { MemoryStore, ProjectSummary } from '../adapter.js';
3
3
  export declare const HERE: string;
4
4
  export declare class SqliteStore implements MemoryStore {
@@ -19,6 +19,7 @@ export declare class SqliteStore implements MemoryStore {
19
19
  getObservations(ids: string[]): Promise<Observation[]>;
20
20
  remove(filter: RemoveFilter): Promise<number>;
21
21
  list(filter: ListFilter): Promise<Observation[]>;
22
+ count(filter: CountFilter): Promise<number>;
22
23
  listProjects(): Promise<ProjectSummary[]>;
23
24
  sessionProjects(): Promise<string[]>;
24
25
  inventory(): Promise<string[]>;
@@ -33,7 +34,8 @@ export declare class SqliteStore implements MemoryStore {
33
34
  files: ScannedFile[];
34
35
  }): Promise<void>;
35
36
  findSymbols(filter: SymbolFilter): Promise<CodeSymbol[]>;
37
+ symbolNames(project: string, limit: number): Promise<string[]>;
36
38
  findEdges(filter: EdgeFilter): Promise<CodeEdge[]>;
37
- scannedFiles(project: string): Promise<ScannedFile[]>;
39
+ scannedFiles(project: string, limit?: number): Promise<ScannedFile[]>;
38
40
  removeGraph(project: string, files?: string[]): Promise<number>;
39
41
  }
@@ -97,6 +97,9 @@ export class SqliteStore {
97
97
  async list(filter) {
98
98
  return observationsOps.list(this.db, filter);
99
99
  }
100
+ async count(filter) {
101
+ return observationsOps.count(this.db, filter);
102
+ }
100
103
  async listProjects() {
101
104
  return observationsOps.listProjects(this.db);
102
105
  }
@@ -127,11 +130,14 @@ export class SqliteStore {
127
130
  async findSymbols(filter) {
128
131
  return graphOps.findSymbols(this.db, filter);
129
132
  }
133
+ async symbolNames(project, limit) {
134
+ return graphOps.symbolNames(this.db, project, limit);
135
+ }
130
136
  async findEdges(filter) {
131
137
  return graphOps.findEdges(this.db, filter);
132
138
  }
133
- async scannedFiles(project) {
134
- return graphOps.scannedFiles(this.db, project);
139
+ async scannedFiles(project, limit) {
140
+ return graphOps.scannedFiles(this.db, project, limit);
135
141
  }
136
142
  async removeGraph(project, files) {
137
143
  return graphOps.removeGraph(this.db, project, files);
@@ -1,10 +1,11 @@
1
1
  import type { DatabaseSync } from 'node:sqlite';
2
- import type { ListFilter, Observation, RemoveFilter } from '../../types.js';
2
+ import type { CountFilter, ListFilter, Observation, RemoveFilter } from '../../types.js';
3
3
  import type { ProjectSummary } from '../adapter.js';
4
4
  export declare function insertObservations(db: DatabaseSync, observations: Observation[]): Promise<void>;
5
5
  export declare function getObservations(db: DatabaseSync, ids: string[]): Promise<Observation[]>;
6
6
  export declare function remove(db: DatabaseSync, filter: RemoveFilter): Promise<number>;
7
7
  export declare function list(db: DatabaseSync, filter: ListFilter): Promise<Observation[]>;
8
+ export declare function count(db: DatabaseSync, filter: CountFilter): Promise<number>;
8
9
  export declare function listProjects(db: DatabaseSync): Promise<ProjectSummary[]>;
9
10
  export declare function inventory(db: DatabaseSync): Promise<string[]>;
10
11
  export declare function closeObservations(db: DatabaseSync, ids: string[]): Promise<number>;
@@ -1,6 +1,6 @@
1
1
  import { foreignNames, isWholeScope } from '../adapter.js';
2
2
  import { toObservation } from './rows.js';
3
- import { excludeSessionsClause, projectClause, removeWhere } from './filters.js';
3
+ import { TAG_PREDICATE, excludeSessionsClause, projectClause, removeWhere } from './filters.js';
4
4
  import { partitionIds } from '../../util/shortid.js';
5
5
  import { packVector } from '../../util/vector.js';
6
6
  import { scopeToken } from '../../util/scope.js';
@@ -76,11 +76,8 @@ export async function remove(db, filter) {
76
76
  db.exec('VACUUM');
77
77
  return count?.n ?? 0;
78
78
  }
79
- export async function list(db, filter) {
80
- if (noProjects(filter.project))
81
- return [];
79
+ function scopeConditions(filter, params) {
82
80
  const conditions = [];
83
- const params = [];
84
81
  const project = projectClause('project', filter.project, params);
85
82
  if (project)
86
83
  conditions.push(project);
@@ -99,6 +96,17 @@ export async function list(db, filter) {
99
96
  conditions.push('status = ?');
100
97
  params.push(filter.status);
101
98
  }
99
+ if (filter.tag) {
100
+ conditions.push(TAG_PREDICATE(''));
101
+ params.push(filter.tag);
102
+ }
103
+ return conditions;
104
+ }
105
+ export async function list(db, filter) {
106
+ if (noProjects(filter.project))
107
+ return [];
108
+ const params = [];
109
+ const conditions = scopeConditions(filter, params);
102
110
  if (filter.after !== undefined) {
103
111
  if (filter.afterId === undefined) {
104
112
  conditions.push('created_at > ?');
@@ -109,6 +117,14 @@ export async function list(db, filter) {
109
117
  params.push(filter.after, filter.after, filter.afterId);
110
118
  }
111
119
  }
120
+ if (filter.vectorStale) {
121
+ const stale = ['embedding IS NULL', 'embedder IS NULL'];
122
+ if (filter.vectorStale.embedder !== undefined) {
123
+ stale.push('embedder != ?');
124
+ params.push(filter.vectorStale.embedder);
125
+ }
126
+ conditions.push(`(${stale.join(' OR ')})`);
127
+ }
112
128
  params.push(filter.limit ?? 1000);
113
129
  const direction = filter.newest ? 'DESC' : 'ASC';
114
130
  const rows = db
@@ -118,6 +134,17 @@ export async function list(db, filter) {
118
134
  .all(...params);
119
135
  return rows.map(toObservation);
120
136
  }
137
+ export async function count(db, filter) {
138
+ if (noProjects(filter.project))
139
+ return 0;
140
+ const params = [];
141
+ const conditions = scopeConditions(filter, params);
142
+ const row = db
143
+ .prepare(`SELECT COUNT(*) AS n FROM observations
144
+ ${conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''}`)
145
+ .get(...params);
146
+ return Number(row?.n ?? 0);
147
+ }
121
148
  export async function listProjects(db) {
122
149
  const rows = db
123
150
  .prepare(`SELECT project, COUNT(*) AS n, MAX(created_at) AS last
package/dist/types.d.ts CHANGED
@@ -58,11 +58,18 @@ export interface ListFilter {
58
58
  excludeSessions?: string[];
59
59
  kind?: ObservationKind;
60
60
  status?: ObservationStatus;
61
+ tag?: string;
61
62
  after?: number;
62
63
  afterId?: string;
63
64
  limit?: number;
64
65
  newest?: boolean;
66
+ vectors?: boolean;
67
+ bodies?: boolean;
68
+ vectorStale?: {
69
+ embedder?: string;
70
+ };
65
71
  }
72
+ export type CountFilter = Pick<ListFilter, 'project' | 'sessionId' | 'excludeSessions' | 'kind' | 'status' | 'tag'>;
66
73
  export interface TimelineQuery {
67
74
  observationId: string;
68
75
  before: number;
@@ -104,6 +111,8 @@ export interface EdgeFilter {
104
111
  project: string;
105
112
  srcIds?: string[];
106
113
  dstIds?: string[];
114
+ unresolvedSrcNames?: string[];
115
+ unresolvedDstNames?: string[];
107
116
  dstName?: string;
108
117
  limit?: number;
109
118
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "claude-db",
3
- "version": "0.12.3",
3
+ "version": "0.12.4",
4
4
  "description": "Free, open-source memory for Claude Code. Remembers why decisions were made and maps your code. Your own database (SQLite, Postgres or MongoDB), no cloud, no subscription.",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",
@@ -54,8 +54,6 @@
54
54
  "dependencies": {
55
55
  "@ast-grep/napi": "^0.45.0",
56
56
  "@modelcontextprotocol/sdk": "^1.30.0",
57
- "graphology": "^0.26.0",
58
- "graphology-shortest-path": "^2.1.0",
59
57
  "oxc-resolver": "^11.24.2",
60
58
  "zod": "^4.4.3"
61
59
  },
@@ -130,10 +128,10 @@
130
128
  "access": "public"
131
129
  },
132
130
  "optionalDependencies": {
133
- "claude-db-grammars-linux-x64": "0.12.3",
134
- "claude-db-grammars-linux-arm64": "0.12.3",
135
- "claude-db-grammars-darwin-x64": "0.12.3",
136
- "claude-db-grammars-darwin-arm64": "0.12.3",
137
- "claude-db-grammars-win32-x64": "0.12.3"
131
+ "claude-db-grammars-linux-x64": "0.12.4",
132
+ "claude-db-grammars-linux-arm64": "0.12.4",
133
+ "claude-db-grammars-darwin-x64": "0.12.4",
134
+ "claude-db-grammars-darwin-arm64": "0.12.4",
135
+ "claude-db-grammars-win32-x64": "0.12.4"
138
136
  }
139
137
  }