claude-db 0.12.2 → 0.12.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/dist/capture/flush.js +41 -4
- package/dist/capture/index.d.ts +1 -0
- package/dist/capture/index.js +1 -0
- package/dist/capture/progress.js +8 -1
- package/dist/capture/vectors.d.ts +7 -0
- package/dist/capture/vectors.js +38 -0
- package/dist/catchup/gather.js +1 -0
- package/dist/cli/commands/adoption.js +41 -29
- package/dist/cli/commands/languages.js +10 -6
- package/dist/cli/commands/sync.js +5 -4
- package/dist/cli/commands/transfer.js +28 -8
- package/dist/facts/claude-memory.js +7 -2
- package/dist/facts/distill.js +4 -1
- package/dist/facts/handoff.js +6 -1
- package/dist/facts/health.js +2 -1
- package/dist/facts/known.js +2 -0
- package/dist/facts/start.js +1 -1
- package/dist/graph/query/answer.js +1 -1
- package/dist/graph/query/lookup.d.ts +1 -0
- package/dist/graph/query/lookup.js +2 -1
- package/dist/graph/query/path.js +69 -37
- package/dist/graph/query/suggest.js +1 -2
- package/dist/hooks/background.js +5 -1
- package/dist/hooks/compact.js +7 -1
- package/dist/hooks/detached.d.ts +1 -0
- package/dist/hooks/detached.js +37 -2
- package/dist/hooks/prefer-usages.js +1 -1
- package/dist/hooks/session-start.js +1 -1
- package/dist/hooks/start-legacy.js +1 -1
- package/dist/store/adapter.d.ts +5 -2
- package/dist/store/each.d.ts +2 -4
- package/dist/store/each.js +1 -0
- package/dist/store/mongo/graph.d.ts +2 -1
- package/dist/store/mongo/graph.js +17 -2
- package/dist/store/mongo/index.d.ts +4 -2
- package/dist/store/mongo/index.js +8 -2
- package/dist/store/mongo/observations.d.ts +2 -1
- package/dist/store/mongo/observations.js +25 -3
- package/dist/store/postgres/graph.d.ts +2 -1
- package/dist/store/postgres/graph.js +22 -7
- package/dist/store/postgres/index.d.ts +5 -2
- package/dist/store/postgres/index.js +14 -7
- package/dist/store/postgres/insert.js +3 -1
- package/dist/store/postgres/meta.d.ts +1 -0
- package/dist/store/postgres/meta.js +4 -0
- package/dist/store/postgres/observations.d.ts +3 -2
- package/dist/store/postgres/observations.js +40 -6
- package/dist/store/postgres/rows.js +1 -1
- package/dist/store/scoped-store.js +4 -0
- package/dist/store/sqlite/graph.d.ts +2 -1
- package/dist/store/sqlite/graph.js +19 -2
- package/dist/store/sqlite/index.d.ts +4 -2
- package/dist/store/sqlite/index.js +8 -2
- package/dist/store/sqlite/observations.d.ts +2 -1
- package/dist/store/sqlite/observations.js +32 -5
- package/dist/types.d.ts +9 -0
- package/package.json +6 -8
package/dist/graph/query/path.js
CHANGED
|
@@ -1,48 +1,80 @@
|
|
|
1
1
|
const MAX_EDGES = 20000;
|
|
2
|
-
const
|
|
2
|
+
const MAX_ENDPOINTS = 5000;
|
|
3
|
+
const UNRESOLVED = 'name:';
|
|
4
|
+
const nodeFor = (id, name) => (id ? id : `${UNRESOLVED}${name}`);
|
|
5
|
+
function sideFrom(nodes) {
|
|
6
|
+
return {
|
|
7
|
+
parent: new Map(nodes.map((node) => [node, null])),
|
|
8
|
+
depth: new Map(nodes.map((node) => [node, 0])),
|
|
9
|
+
frontier: nodes,
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
async function endpoints(store, project, name) {
|
|
13
|
+
const symbols = await store.findSymbols({ project, name, limit: MAX_ENDPOINTS });
|
|
14
|
+
return [...new Set(symbols.map((symbol) => symbol.id)), `${UNRESOLVED}${name}`];
|
|
15
|
+
}
|
|
16
|
+
function edgesOf(store, project, frontier, forward, limit) {
|
|
17
|
+
const ids = frontier.filter((node) => !node.startsWith(UNRESOLVED));
|
|
18
|
+
const names = frontier
|
|
19
|
+
.filter((node) => node.startsWith(UNRESOLVED))
|
|
20
|
+
.map((node) => node.slice(UNRESOLVED.length));
|
|
21
|
+
return store.findEdges(forward
|
|
22
|
+
? { project, srcIds: ids, unresolvedSrcNames: names, limit }
|
|
23
|
+
: { project, dstIds: ids, unresolvedDstNames: names, limit });
|
|
24
|
+
}
|
|
25
|
+
function chain(side, from) {
|
|
26
|
+
const nodes = [];
|
|
27
|
+
for (let node = from; node; node = side.parent.get(node)) {
|
|
28
|
+
nodes.push(node);
|
|
29
|
+
}
|
|
30
|
+
return nodes;
|
|
31
|
+
}
|
|
3
32
|
export async function shortestPath(store, project, from, to) {
|
|
4
33
|
if (!from || !to)
|
|
5
34
|
return [];
|
|
6
35
|
if (from === to)
|
|
7
36
|
return [from];
|
|
8
|
-
const graphology = (await import('graphology'));
|
|
9
|
-
const Graph = graphology.default;
|
|
10
|
-
const { bidirectional } = (await import('graphology-shortest-path/unweighted.js'));
|
|
11
|
-
const [symbols, edges] = await Promise.all([
|
|
12
|
-
store.findSymbols({ project, limit: MAX_SYMBOLS }),
|
|
13
|
-
store.findEdges({ project, limit: MAX_EDGES }),
|
|
14
|
-
]);
|
|
15
37
|
const label = new Map();
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
for (const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
if (
|
|
38
|
+
const ahead = sideFrom(await endpoints(store, project, from));
|
|
39
|
+
const behind = sideFrom(await endpoints(store, project, to));
|
|
40
|
+
for (const node of ahead.frontier)
|
|
41
|
+
label.set(node, from);
|
|
42
|
+
for (const node of behind.frontier)
|
|
43
|
+
label.set(node, to);
|
|
44
|
+
let spent = 0;
|
|
45
|
+
while (ahead.frontier.length > 0 && behind.frontier.length > 0) {
|
|
46
|
+
const forward = ahead.frontier.length <= behind.frontier.length;
|
|
47
|
+
const [mine, other] = forward ? [ahead, behind] : [behind, ahead];
|
|
48
|
+
const current = new Set(mine.frontier);
|
|
49
|
+
const edges = await edgesOf(store, project, mine.frontier, forward, MAX_EDGES - spent + 1);
|
|
50
|
+
spent += edges.length;
|
|
51
|
+
if (spent > MAX_EDGES)
|
|
52
|
+
return [];
|
|
53
|
+
const next = [];
|
|
54
|
+
let meet = null;
|
|
55
|
+
for (const edge of edges) {
|
|
56
|
+
const src = nodeFor(edge.srcId, edge.srcName);
|
|
57
|
+
const dst = nodeFor(edge.dstId, edge.dstName);
|
|
58
|
+
if (src === dst)
|
|
59
|
+
continue;
|
|
60
|
+
if (!label.has(src))
|
|
61
|
+
label.set(src, edge.srcName);
|
|
62
|
+
const [near, far] = forward ? [src, dst] : [dst, src];
|
|
63
|
+
if (!current.has(near) || mine.parent.has(far))
|
|
39
64
|
continue;
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
65
|
+
mine.parent.set(far, near);
|
|
66
|
+
mine.depth.set(far, (mine.depth.get(near) ?? 0) + 1);
|
|
67
|
+
next.push(far);
|
|
68
|
+
const reached = other.depth.get(far);
|
|
69
|
+
if (reached !== undefined && (meet === null || reached < (other.depth.get(meet) ?? 0))) {
|
|
70
|
+
meet = far;
|
|
71
|
+
}
|
|
43
72
|
}
|
|
73
|
+
if (meet !== null) {
|
|
74
|
+
const nodes = [...chain(ahead, meet).reverse(), ...chain(behind, meet).slice(1)];
|
|
75
|
+
return nodes.map((node) => label.get(node) ?? node.replace(/^name:/, ''));
|
|
76
|
+
}
|
|
77
|
+
mine.frontier = next;
|
|
44
78
|
}
|
|
45
|
-
|
|
46
|
-
return [];
|
|
47
|
-
return best.map((node) => label.get(node) ?? node.replace(/^name:/, ''));
|
|
79
|
+
return [];
|
|
48
80
|
}
|
|
@@ -44,8 +44,7 @@ export async function suggestFor(store, project, symbol) {
|
|
|
44
44
|
if (!symbol)
|
|
45
45
|
return [];
|
|
46
46
|
try {
|
|
47
|
-
|
|
48
|
-
return nearest(symbols.map((entry) => entry.name), symbol);
|
|
47
|
+
return nearest(await store.symbolNames(project, CANDIDATES), symbol);
|
|
49
48
|
}
|
|
50
49
|
catch {
|
|
51
50
|
return [];
|
package/dist/hooks/background.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { claimReingest, claimScrub, reingestDone, releaseReingest, releaseScrub, } from '../capture/index.js';
|
|
1
|
+
import { claimReembed, claimReingest, claimScrub, reingestDone, releaseReembed, releaseReingest, releaseScrub, } from '../capture/index.js';
|
|
2
2
|
import { FACTS_JOB } from '../facts/distill.js';
|
|
3
3
|
import { claimJob, jobMark, releaseJob } from '../util/job-lock.js';
|
|
4
4
|
import { CLI, PICK_WORKER, runDetached } from './detached.js';
|
|
@@ -14,6 +14,10 @@ export function startBackgroundWork(project, database, now = Date.now()) {
|
|
|
14
14
|
}
|
|
15
15
|
if (!reingestDone(project))
|
|
16
16
|
return;
|
|
17
|
+
if (claimReembed(project, now)) {
|
|
18
|
+
runDetached(CLI, ['reembed', '--project', '--background'], project, () => releaseReembed(project));
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
17
21
|
if (now - jobMark(FACTS_JOB, project) < FACTS_EVERY_MS)
|
|
18
22
|
return;
|
|
19
23
|
if (!claimJob(FACTS_JOB, project, now))
|
package/dist/hooks/compact.js
CHANGED
|
@@ -14,7 +14,13 @@ export async function recoverAfterCompact(ctx, project, sessionId, transcriptPat
|
|
|
14
14
|
if (!capturingDisabled(ctx.config.capture.scripted)) {
|
|
15
15
|
await flushSession(ctx, sessionId, project, transcriptPath);
|
|
16
16
|
}
|
|
17
|
-
const observations = await ctx.store.list({
|
|
17
|
+
const observations = await ctx.store.list({
|
|
18
|
+
project,
|
|
19
|
+
sessionId,
|
|
20
|
+
newest: true,
|
|
21
|
+
limit: SCANNED,
|
|
22
|
+
bodies: false,
|
|
23
|
+
});
|
|
18
24
|
return renderEarlierInChat(observations);
|
|
19
25
|
}
|
|
20
26
|
export function renderEarlierInChat(newestFirst) {
|
package/dist/hooks/detached.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export declare const CLI: string;
|
|
2
2
|
export declare const PICK_WORKER: string;
|
|
3
3
|
export declare const TURN_WORKER: string;
|
|
4
|
+
export declare const backgroundLog: () => string;
|
|
4
5
|
export declare function runDetached(script: string, args: string[], cwd: string, release: () => void): void;
|
package/dist/hooks/detached.js
CHANGED
|
@@ -1,17 +1,52 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
|
-
import {
|
|
2
|
+
import { appendFileSync, closeSync, mkdirSync, openSync, renameSync, statSync } from 'node:fs';
|
|
3
|
+
import { basename, dirname, join, resolve } from 'node:path';
|
|
3
4
|
import { fileURLToPath } from 'node:url';
|
|
5
|
+
import { CONFIG_DIR } from '../config/dir.js';
|
|
4
6
|
const DIST = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
5
7
|
export const CLI = resolve(DIST, 'cli', 'index.js');
|
|
6
8
|
export const PICK_WORKER = resolve(DIST, 'pick', 'worker.js');
|
|
7
9
|
export const TURN_WORKER = resolve(DIST, 'hooks', 'turn-save.js');
|
|
10
|
+
const LOG_LIMIT = 1024 * 1024;
|
|
11
|
+
export const backgroundLog = () => join(CONFIG_DIR, 'logs', 'background.log');
|
|
12
|
+
function rotate(path) {
|
|
13
|
+
try {
|
|
14
|
+
if (statSync(path).size > LOG_LIMIT)
|
|
15
|
+
renameSync(path, `${path}.1`);
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
function openLog(script, args) {
|
|
22
|
+
try {
|
|
23
|
+
const path = backgroundLog();
|
|
24
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
25
|
+
rotate(path);
|
|
26
|
+
const name = script === CLI ? (args[0] ?? 'cli') : basename(script, '.js');
|
|
27
|
+
appendFileSync(path, `${new Date().toISOString()} ${name} started\n`);
|
|
28
|
+
return openSync(path, 'a');
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return 'ignore';
|
|
32
|
+
}
|
|
33
|
+
}
|
|
8
34
|
export function runDetached(script, args, cwd, release) {
|
|
35
|
+
const log = openLog(script, args);
|
|
9
36
|
try {
|
|
10
|
-
spawn(process.execPath, [script, ...args], {
|
|
37
|
+
spawn(process.execPath, [script, ...args], {
|
|
38
|
+
cwd,
|
|
39
|
+
detached: true,
|
|
40
|
+
stdio: ['ignore', log, log],
|
|
41
|
+
})
|
|
11
42
|
.on('error', release)
|
|
12
43
|
.unref();
|
|
13
44
|
}
|
|
14
45
|
catch {
|
|
15
46
|
release();
|
|
16
47
|
}
|
|
48
|
+
finally {
|
|
49
|
+
if (typeof log === 'number')
|
|
50
|
+
closeSync(log);
|
|
51
|
+
}
|
|
17
52
|
}
|
|
@@ -57,7 +57,7 @@ async function answerSymbols(ctx, project, symbols) {
|
|
|
57
57
|
store: ctx.store,
|
|
58
58
|
root: project,
|
|
59
59
|
project,
|
|
60
|
-
query: { mode: 'usages', symbol, limit: EDGE_LIMIT },
|
|
60
|
+
query: { mode: 'usages', symbol, limit: EDGE_LIMIT, suggest: false },
|
|
61
61
|
refresh: false,
|
|
62
62
|
text: false,
|
|
63
63
|
});
|
|
@@ -73,7 +73,7 @@ await runHook(async () => {
|
|
|
73
73
|
parts.push("Search this project's full history with the memory MCP tools before " +
|
|
74
74
|
'asking the user to re-explain prior decisions.');
|
|
75
75
|
}
|
|
76
|
-
const scanned = (await ctx.store.scannedFiles(project)).length > 0;
|
|
76
|
+
const scanned = (await ctx.store.scannedFiles(project, 1)).length > 0;
|
|
77
77
|
if (!scanned)
|
|
78
78
|
parts.push(SCAN_HINT);
|
|
79
79
|
const update = ctx.config.updates === 'off' ? null : updateNotice();
|
|
@@ -24,7 +24,7 @@ export async function legacyBlock(ctx, project, restored) {
|
|
|
24
24
|
for (const obs of open.slice(0, 3))
|
|
25
25
|
lines.push(`- ${obs.title}`);
|
|
26
26
|
}
|
|
27
|
-
const rules = (await ctx.store.list({ project, kind: 'preference', limit: 100, newest: true }))
|
|
27
|
+
const rules = (await ctx.store.list({ project, kind: 'preference', limit: 100, newest: true, bodies: false }))
|
|
28
28
|
.filter((obs) => obs.status !== 'replaced' && !restored.has(obs.id))
|
|
29
29
|
.sort((a, b) => {
|
|
30
30
|
const manual = Number(b.sessionId === 'manual') - Number(a.sessionId === 'manual');
|
package/dist/store/adapter.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CodeEdge, CodeSymbol, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../types.js';
|
|
1
|
+
import type { CodeEdge, CodeSymbol, CountFilter, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../types.js';
|
|
2
2
|
export interface ProjectSummary {
|
|
3
3
|
project: string;
|
|
4
4
|
observations: number;
|
|
@@ -19,8 +19,10 @@ export interface MemoryStore {
|
|
|
19
19
|
insertObservations(observations: Observation[]): Promise<void>;
|
|
20
20
|
getObservations(ids: string[]): Promise<Observation[]>;
|
|
21
21
|
migrateVectorDims?(dims: number): Promise<boolean>;
|
|
22
|
+
storedVectorDims?(): number | null;
|
|
22
23
|
remove(filter: RemoveFilter): Promise<number>;
|
|
23
24
|
list(filter: ListFilter): Promise<Observation[]>;
|
|
25
|
+
count(filter: CountFilter): Promise<number>;
|
|
24
26
|
listProjects(): Promise<ProjectSummary[]>;
|
|
25
27
|
inventory(): Promise<string[]>;
|
|
26
28
|
searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
@@ -34,8 +36,9 @@ export interface MemoryStore {
|
|
|
34
36
|
files: ScannedFile[];
|
|
35
37
|
}): Promise<void>;
|
|
36
38
|
findSymbols(filter: SymbolFilter): Promise<CodeSymbol[]>;
|
|
39
|
+
symbolNames(project: string, limit: number): Promise<string[]>;
|
|
37
40
|
findEdges(filter: EdgeFilter): Promise<CodeEdge[]>;
|
|
38
|
-
scannedFiles(project: string): Promise<ScannedFile[]>;
|
|
41
|
+
scannedFiles(project: string, limit?: number): Promise<ScannedFile[]>;
|
|
39
42
|
removeGraph(project: string, files?: string[]): Promise<number>;
|
|
40
43
|
}
|
|
41
44
|
export declare function isWholeScope(filter: RemoveFilter): boolean;
|
package/dist/store/each.d.ts
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
1
|
import type { RecallContext } from '../context.js';
|
|
2
|
-
import type { Observation } from '../types.js';
|
|
3
|
-
export declare function eachObservation(ctx: RecallContext, filter:
|
|
4
|
-
project?: string;
|
|
5
|
-
}, visit: (batch: Observation[]) => Promise<void> | void): Promise<number>;
|
|
2
|
+
import type { ListFilter, Observation } from '../types.js';
|
|
3
|
+
export declare function eachObservation(ctx: RecallContext, filter: Pick<ListFilter, 'project' | 'vectorStale'>, visit: (batch: Observation[]) => Promise<void> | void): Promise<number>;
|
package/dist/store/each.js
CHANGED
|
@@ -7,6 +7,7 @@ export declare function upsertGraph(symbols: Collection<SymbolDoc>, edges: Colle
|
|
|
7
7
|
files: ScannedFile[];
|
|
8
8
|
}): Promise<void>;
|
|
9
9
|
export declare function findSymbols(symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, filter: SymbolFilter): Promise<CodeSymbol[]>;
|
|
10
|
+
export declare function symbolNames(symbols: Collection<SymbolDoc>, project: string, limit: number): Promise<string[]>;
|
|
10
11
|
export declare function findEdges(symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, filter: EdgeFilter): Promise<CodeEdge[]>;
|
|
11
|
-
export declare function scannedFiles(symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, project: string): Promise<ScannedFile[]>;
|
|
12
|
+
export declare function scannedFiles(symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, project: string, limit?: number): Promise<ScannedFile[]>;
|
|
12
13
|
export declare function removeGraph(symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, project: string, files?: string[]): Promise<number>;
|
|
@@ -24,6 +24,14 @@ export async function findSymbols(symbols, edges, scanned, filter) {
|
|
|
24
24
|
.toArray();
|
|
25
25
|
return docs.map(toSymbol);
|
|
26
26
|
}
|
|
27
|
+
export async function symbolNames(symbols, project, limit) {
|
|
28
|
+
const docs = await symbols
|
|
29
|
+
.find({ project }, { projection: { name: 1 } })
|
|
30
|
+
.sort({ file: 1, line: 1 })
|
|
31
|
+
.limit(limit)
|
|
32
|
+
.toArray();
|
|
33
|
+
return docs.map((doc) => doc.name);
|
|
34
|
+
}
|
|
27
35
|
export async function findEdges(symbols, edges, scanned, filter) {
|
|
28
36
|
const query = { project: filter.project };
|
|
29
37
|
const alternatives = [];
|
|
@@ -33,6 +41,12 @@ export async function findEdges(symbols, edges, scanned, filter) {
|
|
|
33
41
|
if (filter.dstIds && filter.dstIds.length > 0) {
|
|
34
42
|
alternatives.push({ dstId: { $in: filter.dstIds } });
|
|
35
43
|
}
|
|
44
|
+
if (filter.unresolvedSrcNames && filter.unresolvedSrcNames.length > 0) {
|
|
45
|
+
alternatives.push({ srcId: { $in: ['', null] }, srcName: { $in: filter.unresolvedSrcNames } });
|
|
46
|
+
}
|
|
47
|
+
if (filter.unresolvedDstNames && filter.unresolvedDstNames.length > 0) {
|
|
48
|
+
alternatives.push({ dstId: { $in: ['', null] }, dstName: { $in: filter.unresolvedDstNames } });
|
|
49
|
+
}
|
|
36
50
|
if (alternatives.length > 0)
|
|
37
51
|
query['$or'] = alternatives;
|
|
38
52
|
if (filter.dstName !== undefined) {
|
|
@@ -47,8 +61,9 @@ export async function findEdges(symbols, edges, scanned, filter) {
|
|
|
47
61
|
.toArray();
|
|
48
62
|
return docs.map(toEdge);
|
|
49
63
|
}
|
|
50
|
-
export async function scannedFiles(symbols, edges, scanned, project) {
|
|
51
|
-
const
|
|
64
|
+
export async function scannedFiles(symbols, edges, scanned, project, limit) {
|
|
65
|
+
const cursor = scanned.find({ project });
|
|
66
|
+
const docs = await (limit === undefined ? cursor : cursor.limit(limit)).toArray();
|
|
52
67
|
return docs.map((doc) => ({
|
|
53
68
|
project: doc.project,
|
|
54
69
|
path: doc.path,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CodeEdge, CodeSymbol, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
|
|
1
|
+
import type { CodeEdge, CodeSymbol, CountFilter, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
|
|
2
2
|
import type { MemoryStore, ProjectSummary } from '../adapter.js';
|
|
3
3
|
export declare class MongoStore implements MemoryStore {
|
|
4
4
|
private readonly client;
|
|
@@ -27,6 +27,7 @@ export declare class MongoStore implements MemoryStore {
|
|
|
27
27
|
getObservations(ids: string[]): Promise<Observation[]>;
|
|
28
28
|
remove(filter: RemoveFilter): Promise<number>;
|
|
29
29
|
list(filter: ListFilter): Promise<Observation[]>;
|
|
30
|
+
count(filter: CountFilter): Promise<number>;
|
|
30
31
|
listProjects(): Promise<ProjectSummary[]>;
|
|
31
32
|
inventory(): Promise<string[]>;
|
|
32
33
|
searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
@@ -41,7 +42,8 @@ export declare class MongoStore implements MemoryStore {
|
|
|
41
42
|
files: ScannedFile[];
|
|
42
43
|
}): Promise<void>;
|
|
43
44
|
findSymbols(filter: SymbolFilter): Promise<CodeSymbol[]>;
|
|
45
|
+
symbolNames(project: string, limit: number): Promise<string[]>;
|
|
44
46
|
findEdges(filter: EdgeFilter): Promise<CodeEdge[]>;
|
|
45
|
-
scannedFiles(project: string): Promise<ScannedFile[]>;
|
|
47
|
+
scannedFiles(project: string, limit?: number): Promise<ScannedFile[]>;
|
|
46
48
|
removeGraph(project: string, files?: string[]): Promise<number>;
|
|
47
49
|
}
|
|
@@ -89,6 +89,9 @@ export class MongoStore {
|
|
|
89
89
|
async list(filter) {
|
|
90
90
|
return observationsOps.list(this.observations, this.sessions, this.symbols, this.edges, this.scanned, filter);
|
|
91
91
|
}
|
|
92
|
+
async count(filter) {
|
|
93
|
+
return observationsOps.count(this.observations, filter);
|
|
94
|
+
}
|
|
92
95
|
async listProjects() {
|
|
93
96
|
return observationsOps.listProjects(this.observations);
|
|
94
97
|
}
|
|
@@ -121,11 +124,14 @@ export class MongoStore {
|
|
|
121
124
|
async findSymbols(filter) {
|
|
122
125
|
return graphOps.findSymbols(this.symbols, this.edges, this.scanned, filter);
|
|
123
126
|
}
|
|
127
|
+
async symbolNames(project, limit) {
|
|
128
|
+
return graphOps.symbolNames(this.symbols, project, limit);
|
|
129
|
+
}
|
|
124
130
|
async findEdges(filter) {
|
|
125
131
|
return graphOps.findEdges(this.symbols, this.edges, this.scanned, filter);
|
|
126
132
|
}
|
|
127
|
-
async scannedFiles(project) {
|
|
128
|
-
return graphOps.scannedFiles(this.symbols, this.edges, this.scanned, project);
|
|
133
|
+
async scannedFiles(project, limit) {
|
|
134
|
+
return graphOps.scannedFiles(this.symbols, this.edges, this.scanned, project, limit);
|
|
129
135
|
}
|
|
130
136
|
async removeGraph(project, files) {
|
|
131
137
|
return graphOps.removeGraph(this.symbols, this.edges, this.scanned, project, files);
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import type { Collection } from './driver.js';
|
|
2
|
-
import type { ListFilter, Observation, RemoveFilter } from '../../types.js';
|
|
2
|
+
import type { CountFilter, ListFilter, Observation, RemoveFilter } from '../../types.js';
|
|
3
3
|
import type { ProjectSummary } from '../adapter.js';
|
|
4
4
|
import type { EdgeDoc, ObservationDoc, ScannedFileDoc, SessionDoc, SymbolDoc } from './docs.js';
|
|
5
5
|
export declare function insertObservations(collection: Collection<ObservationDoc>, sessions: Collection<SessionDoc>, symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, rows: Observation[]): Promise<void>;
|
|
6
6
|
export declare function getObservations(observations: Collection<ObservationDoc>, sessions: Collection<SessionDoc>, symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, ids: string[]): Promise<Observation[]>;
|
|
7
7
|
export declare function remove(observations: Collection<ObservationDoc>, sessions: Collection<SessionDoc>, symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, filter: RemoveFilter): Promise<number>;
|
|
8
|
+
export declare function count(observations: Collection<ObservationDoc>, filter: CountFilter): Promise<number>;
|
|
8
9
|
export declare function list(observations: Collection<ObservationDoc>, sessions: Collection<SessionDoc>, symbols: Collection<SymbolDoc>, edges: Collection<EdgeDoc>, scanned: Collection<ScannedFileDoc>, filter: ListFilter): Promise<Observation[]>;
|
|
9
10
|
export declare function listProjects(observations: Collection<ObservationDoc>): Promise<ProjectSummary[]>;
|
|
10
11
|
export declare function closeObservations(observations: Collection<ObservationDoc>, ids: string[]): Promise<number>;
|
|
@@ -60,9 +60,7 @@ export async function remove(observations, sessions, symbols, edges, scanned, fi
|
|
|
60
60
|
}
|
|
61
61
|
return count;
|
|
62
62
|
}
|
|
63
|
-
|
|
64
|
-
if (noProjects(filter.project))
|
|
65
|
-
return [];
|
|
63
|
+
function scopeQuery(filter) {
|
|
66
64
|
const query = {};
|
|
67
65
|
Object.assign(query, projectMatch(filter.project));
|
|
68
66
|
Object.assign(query, sessionMatch(filter.sessionId, filter.excludeSessions));
|
|
@@ -70,6 +68,19 @@ export async function list(observations, sessions, symbols, edges, scanned, filt
|
|
|
70
68
|
query['kind'] = filter.kind;
|
|
71
69
|
if (filter.status)
|
|
72
70
|
query['status'] = filter.status;
|
|
71
|
+
if (filter.tag)
|
|
72
|
+
query['tags'] = filter.tag;
|
|
73
|
+
return query;
|
|
74
|
+
}
|
|
75
|
+
export async function count(observations, filter) {
|
|
76
|
+
if (noProjects(filter.project))
|
|
77
|
+
return 0;
|
|
78
|
+
return observations.countDocuments(scopeQuery(filter));
|
|
79
|
+
}
|
|
80
|
+
export async function list(observations, sessions, symbols, edges, scanned, filter) {
|
|
81
|
+
if (noProjects(filter.project))
|
|
82
|
+
return [];
|
|
83
|
+
const query = scopeQuery(filter);
|
|
73
84
|
if (filter.after !== undefined) {
|
|
74
85
|
if (filter.afterId === undefined)
|
|
75
86
|
query['createdAt'] = { $gt: filter.after };
|
|
@@ -79,6 +90,17 @@ export async function list(observations, sessions, symbols, edges, scanned, filt
|
|
|
79
90
|
{ createdAt: filter.after, _id: { $gt: filter.afterId } },
|
|
80
91
|
];
|
|
81
92
|
}
|
|
93
|
+
if (filter.vectorStale) {
|
|
94
|
+
const { embedder } = filter.vectorStale;
|
|
95
|
+
query['$and'] = [
|
|
96
|
+
{
|
|
97
|
+
$or: [
|
|
98
|
+
{ embedding: null },
|
|
99
|
+
embedder === undefined ? { embedder: null } : { embedder: { $ne: embedder } },
|
|
100
|
+
],
|
|
101
|
+
},
|
|
102
|
+
];
|
|
103
|
+
}
|
|
82
104
|
const direction = filter.newest ? -1 : 1;
|
|
83
105
|
const docs = await observations
|
|
84
106
|
.find(query)
|
|
@@ -6,6 +6,7 @@ export declare function upsertGraph(pool: Pool, scan: {
|
|
|
6
6
|
files: ScannedFile[];
|
|
7
7
|
}): Promise<void>;
|
|
8
8
|
export declare function findSymbols(pool: Pool, filter: SymbolFilter): Promise<CodeSymbol[]>;
|
|
9
|
+
export declare function symbolNames(pool: Pool, project: string, limit: number): Promise<string[]>;
|
|
9
10
|
export declare function findEdges(pool: Pool, filter: EdgeFilter): Promise<CodeEdge[]>;
|
|
10
|
-
export declare function scannedFiles(pool: Pool, project: string): Promise<ScannedFile[]>;
|
|
11
|
+
export declare function scannedFiles(pool: Pool, project: string, limit?: number): Promise<ScannedFile[]>;
|
|
11
12
|
export declare function removeGraph(pool: Pool, project: string, files?: string[]): Promise<number>;
|
|
@@ -68,6 +68,8 @@ export async function upsertGraph(pool, scan) {
|
|
|
68
68
|
client.release();
|
|
69
69
|
}
|
|
70
70
|
}
|
|
71
|
+
const SYMBOL_COLUMNS = 'id, name, kind, file, line, lang, signature';
|
|
72
|
+
const EDGE_COLUMNS = 'id, src_id, src_name, dst_id, dst_name, relation, confidence, score, file, line';
|
|
71
73
|
export async function findSymbols(pool, filter) {
|
|
72
74
|
const values = [filter.project];
|
|
73
75
|
const conditions = ['project = $1'];
|
|
@@ -80,9 +82,13 @@ export async function findSymbols(pool, filter) {
|
|
|
80
82
|
conditions.push(`file = $${values.length}`);
|
|
81
83
|
}
|
|
82
84
|
values.push(filter.limit ?? 500);
|
|
83
|
-
const res = await pool.query(`SELECT
|
|
85
|
+
const res = await pool.query(`SELECT ${SYMBOL_COLUMNS} FROM symbols WHERE ${conditions.join(' AND ')}
|
|
84
86
|
ORDER BY file, line LIMIT $${values.length}`, values);
|
|
85
|
-
return res.rows.map(toSymbol);
|
|
87
|
+
return res.rows.map((row) => toSymbol({ ...row, project: filter.project }));
|
|
88
|
+
}
|
|
89
|
+
export async function symbolNames(pool, project, limit) {
|
|
90
|
+
const res = await pool.query('SELECT name FROM symbols WHERE project = $1 ORDER BY file, line LIMIT $2', [project, limit]);
|
|
91
|
+
return res.rows.map((row) => row['name']);
|
|
86
92
|
}
|
|
87
93
|
export async function findEdges(pool, filter) {
|
|
88
94
|
const values = [filter.project];
|
|
@@ -96,20 +102,29 @@ export async function findEdges(pool, filter) {
|
|
|
96
102
|
values.push(ids);
|
|
97
103
|
sides.push(`${column} = ANY($${values.length}::text[])`);
|
|
98
104
|
}
|
|
105
|
+
for (const [side, names] of [
|
|
106
|
+
['src', filter.unresolvedSrcNames],
|
|
107
|
+
['dst', filter.unresolvedDstNames],
|
|
108
|
+
]) {
|
|
109
|
+
if (!names || names.length === 0)
|
|
110
|
+
continue;
|
|
111
|
+
values.push(names);
|
|
112
|
+
sides.push(`(${side}_id = '' AND ${side}_name = ANY($${values.length}::text[]))`);
|
|
113
|
+
}
|
|
99
114
|
let where = sides.length > 0 ? ` AND (${sides.join(' OR ')})` : '';
|
|
100
115
|
if (filter.dstName !== undefined) {
|
|
101
116
|
values.push(filter.dstName, `%.${filter.dstName}`);
|
|
102
117
|
where += ` AND (dst_name = $${values.length - 1} OR dst_name LIKE $${values.length})`;
|
|
103
118
|
}
|
|
104
119
|
values.push(filter.limit ?? 2000);
|
|
105
|
-
const res = await pool.query(`SELECT
|
|
120
|
+
const res = await pool.query(`SELECT ${EDGE_COLUMNS} FROM symbol_edges WHERE project = $1${where}
|
|
106
121
|
ORDER BY file, line, id LIMIT $${values.length}`, values);
|
|
107
|
-
return res.rows.map(toEdge);
|
|
122
|
+
return res.rows.map((row) => toEdge({ ...row, project: filter.project }));
|
|
108
123
|
}
|
|
109
|
-
export async function scannedFiles(pool, project) {
|
|
110
|
-
const res = await pool.query(
|
|
124
|
+
export async function scannedFiles(pool, project, limit) {
|
|
125
|
+
const res = await pool.query(`SELECT path, hash, scanned_at FROM scanned_files WHERE project = $1${limit === undefined ? '' : ' LIMIT $2'}`, limit === undefined ? [project] : [project, limit]);
|
|
111
126
|
return res.rows.map((row) => ({
|
|
112
|
-
project
|
|
127
|
+
project,
|
|
113
128
|
path: row['path'],
|
|
114
129
|
hash: row['hash'],
|
|
115
130
|
scannedAt: Number(row['scanned_at']),
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CodeEdge, CodeSymbol, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
|
|
1
|
+
import type { CodeEdge, CodeSymbol, CountFilter, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
|
|
2
2
|
import type { MemoryStore, ProjectSummary } from '../adapter.js';
|
|
3
3
|
export declare class PostgresStore implements MemoryStore {
|
|
4
4
|
private readonly pool;
|
|
@@ -18,6 +18,7 @@ export declare class PostgresStore implements MemoryStore {
|
|
|
18
18
|
ensureTagsIndexed(): Promise<void>;
|
|
19
19
|
readVectorDims(): Promise<number | null>;
|
|
20
20
|
migrateVectorDims(dims: number): Promise<boolean>;
|
|
21
|
+
storedVectorDims(): number | null;
|
|
21
22
|
private ensureVectorColumn;
|
|
22
23
|
close(): Promise<void>;
|
|
23
24
|
ping(): Promise<boolean>;
|
|
@@ -32,6 +33,7 @@ export declare class PostgresStore implements MemoryStore {
|
|
|
32
33
|
getObservations(ids: string[]): Promise<Observation[]>;
|
|
33
34
|
remove(filter: RemoveFilter): Promise<number>;
|
|
34
35
|
list(filter: ListFilter): Promise<Observation[]>;
|
|
36
|
+
count(filter: CountFilter): Promise<number>;
|
|
35
37
|
listProjects(): Promise<ProjectSummary[]>;
|
|
36
38
|
inventory(): Promise<string[]>;
|
|
37
39
|
searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
@@ -46,7 +48,8 @@ export declare class PostgresStore implements MemoryStore {
|
|
|
46
48
|
files: ScannedFile[];
|
|
47
49
|
}): Promise<void>;
|
|
48
50
|
findSymbols(filter: SymbolFilter): Promise<CodeSymbol[]>;
|
|
51
|
+
symbolNames(project: string, limit: number): Promise<string[]>;
|
|
49
52
|
findEdges(filter: EdgeFilter): Promise<CodeEdge[]>;
|
|
50
|
-
scannedFiles(project: string): Promise<ScannedFile[]>;
|
|
53
|
+
scannedFiles(project: string, limit?: number): Promise<ScannedFile[]>;
|
|
51
54
|
removeGraph(project: string, files?: string[]): Promise<number>;
|
|
52
55
|
}
|
|
@@ -58,13 +58,15 @@ export class PostgresStore {
|
|
|
58
58
|
await this.pool.query('ALTER TABLE observations DROP COLUMN IF EXISTS embedding');
|
|
59
59
|
await this.pool.query(`ALTER TABLE observations ADD COLUMN embedding vector(${dims})`);
|
|
60
60
|
await this.pool.query('UPDATE observations SET embedder = NULL');
|
|
61
|
-
await this.pool
|
|
62
|
-
ON observations USING hnsw (embedding vector_cosine_ops)`);
|
|
61
|
+
await metaOps.indexEmbeddings(this.pool);
|
|
63
62
|
this.vectorDims = await this.readVectorDims();
|
|
64
63
|
this.warnedDims = false;
|
|
65
64
|
await this.writeMeta();
|
|
66
65
|
return true;
|
|
67
66
|
}
|
|
67
|
+
storedVectorDims() {
|
|
68
|
+
return this.vectorEnabled ? this.vectorDims : null;
|
|
69
|
+
}
|
|
68
70
|
async ensureVectorColumn(dims) {
|
|
69
71
|
if (!this.vectorEnabled)
|
|
70
72
|
return false;
|
|
@@ -72,8 +74,7 @@ export class PostgresStore {
|
|
|
72
74
|
return false;
|
|
73
75
|
if (this.vectorDims === null) {
|
|
74
76
|
await this.pool.query(`ALTER TABLE observations ADD COLUMN IF NOT EXISTS embedding vector(${dims})`);
|
|
75
|
-
await this.pool
|
|
76
|
-
ON observations USING hnsw (embedding vector_cosine_ops)`);
|
|
77
|
+
await metaOps.indexEmbeddings(this.pool);
|
|
77
78
|
this.vectorDims = await this.readVectorDims();
|
|
78
79
|
await this.writeMeta();
|
|
79
80
|
}
|
|
@@ -122,7 +123,10 @@ export class PostgresStore {
|
|
|
122
123
|
return observationsOps.remove(this.pool, filter);
|
|
123
124
|
}
|
|
124
125
|
async list(filter) {
|
|
125
|
-
return observationsOps.list(this.pool, filter);
|
|
126
|
+
return observationsOps.list(this.pool, filter, this.vectorDims !== null);
|
|
127
|
+
}
|
|
128
|
+
async count(filter) {
|
|
129
|
+
return observationsOps.count(this.pool, filter);
|
|
126
130
|
}
|
|
127
131
|
async listProjects() {
|
|
128
132
|
return observationsOps.listProjects(this.pool);
|
|
@@ -154,11 +158,14 @@ export class PostgresStore {
|
|
|
154
158
|
async findSymbols(filter) {
|
|
155
159
|
return graphOps.findSymbols(this.pool, filter);
|
|
156
160
|
}
|
|
161
|
+
async symbolNames(project, limit) {
|
|
162
|
+
return graphOps.symbolNames(this.pool, project, limit);
|
|
163
|
+
}
|
|
157
164
|
async findEdges(filter) {
|
|
158
165
|
return graphOps.findEdges(this.pool, filter);
|
|
159
166
|
}
|
|
160
|
-
async scannedFiles(project) {
|
|
161
|
-
return graphOps.scannedFiles(this.pool, project);
|
|
167
|
+
async scannedFiles(project, limit) {
|
|
168
|
+
return graphOps.scannedFiles(this.pool, project, limit);
|
|
162
169
|
}
|
|
163
170
|
async removeGraph(project, files) {
|
|
164
171
|
return graphOps.removeGraph(this.pool, project, files);
|