claude-db 0.8.5 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -0
- package/dist/capture/flush.js +3 -1
- package/dist/capture/progress.js +41 -9
- package/dist/capture/transcript.js +8 -2
- package/dist/capture/turn-extractor.js +2 -2
- package/dist/cli/commands/graph.js +12 -3
- package/dist/cli/commands/sync.js +9 -3
- package/dist/cli/commands/transfer.js +60 -10
- package/dist/cli/commands/view.js +1 -1
- package/dist/graph/languages/basic.js +5 -0
- package/dist/graph/query/lookup.js +7 -3
- package/dist/graph/scan/extract.d.ts +2 -0
- package/dist/graph/scan/extract.js +40 -20
- package/dist/graph/scan/files.d.ts +2 -1
- package/dist/graph/scan/files.js +4 -3
- package/dist/graph/scan/index.d.ts +1 -0
- package/dist/graph/scan/index.js +11 -1
- package/dist/graph/scan/resolve.js +17 -6
- package/dist/hooks/grep-symbols.js +1 -1
- package/dist/hooks/relevance.d.ts +1 -1
- package/dist/hooks/relevance.js +3 -1
- package/dist/hooks/session-start.js +1 -1
- package/dist/hooks/user-prompt.js +3 -2
- package/dist/store/adapter.d.ts +1 -0
- package/dist/store/mongo/graph.js +7 -0
- package/dist/store/mongo/index.d.ts +1 -0
- package/dist/store/mongo/index.js +3 -0
- package/dist/store/mongo/observations.js +11 -3
- package/dist/store/mongo/search.js +7 -5
- package/dist/store/mongo/sessions.d.ts +1 -0
- package/dist/store/mongo/sessions.js +8 -5
- package/dist/store/mongo/vectorSearch.js +1 -1
- package/dist/store/postgres/graph.js +7 -2
- package/dist/store/postgres/index.d.ts +1 -0
- package/dist/store/postgres/index.js +4 -1
- package/dist/store/postgres/insert.d.ts +1 -1
- package/dist/store/postgres/insert.js +2 -1
- package/dist/store/postgres/meta.js +16 -4
- package/dist/store/postgres/observations.js +10 -3
- package/dist/store/postgres/sessions.d.ts +1 -0
- package/dist/store/postgres/sessions.js +4 -0
- package/dist/store/sqlite/graph.js +6 -2
- package/dist/store/sqlite/index.d.ts +1 -0
- package/dist/store/sqlite/index.js +3 -0
- package/dist/store/sqlite/observations.js +10 -3
- package/dist/store/sqlite/sessions.d.ts +1 -0
- package/dist/store/sqlite/sessions.js +4 -0
- package/dist/types.d.ts +3 -0
- package/dist/usages/find.js +10 -6
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,92 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.9.0
|
|
4
|
+
|
|
5
|
+
A correctness release. Everything below was found by reviewing the whole
|
|
6
|
+
codebase rather than reported, so most of it had been failing quietly.
|
|
7
|
+
|
|
8
|
+
**Stored graphs rebuild on the first scan after upgrading.** Extraction and
|
|
9
|
+
edge ids both changed, so `SCAN_VERSION` is bumped and the old rows are
|
|
10
|
+
discarded rather than left to disagree with the new ones.
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
|
|
14
|
+
- **`claude-db merge` could permanently delete observations.** `list()` ordered
|
|
15
|
+
by `created_at` alone while `after` was a strict `>`, so any row sharing a
|
|
16
|
+
timestamp with a batch boundary was stepped over by every caller that pages
|
|
17
|
+
through the store. `merge` copies page by page and then removes the source
|
|
18
|
+
project, deleting exactly the rows its own loop had skipped. Git-seeded
|
|
19
|
+
observations take the commit date, which has one-second resolution, so ties
|
|
20
|
+
are routine. Paging is now a keyset walk over `(created_at, id)`.
|
|
21
|
+
`export` and `sync` were dropping the same rows without deleting them.
|
|
22
|
+
|
|
23
|
+
- **A session's last turn was lost when the prompt was still being written.**
|
|
24
|
+
The flush cursor advanced to the raw end of the transcript even when the file
|
|
25
|
+
ended mid-line, so the next read began inside the JSON, the entry was dropped,
|
|
26
|
+
and the prompt with all its work never appeared again. The cursor now stops at
|
|
27
|
+
the last complete line.
|
|
28
|
+
|
|
29
|
+
- **`find_usages` hid the real caller of any method.** Inbound edges were
|
|
30
|
+
matched on the destination name, but a member call is stored under its
|
|
31
|
+
qualified name, so the actual `repo.save()` caller was fetched and then
|
|
32
|
+
discarded, leaving only the class that declares it.
|
|
33
|
+
|
|
34
|
+
- **A repeated name in one file pointed its edge at the wrong declaration.**
|
|
35
|
+
0.8.5 gave each declaration its own id, but the resolver still matched by
|
|
36
|
+
name and took the first one in the file, so the second `run()` got an edge to
|
|
37
|
+
the first, stamped at the confidence tier that means "read from the syntax".
|
|
38
|
+
|
|
39
|
+
- **`implements` never produced an edge, and neither did `extends` in `.js` or
|
|
40
|
+
`.jsx`.** Those rules capture a whole clause, whose text includes the keyword,
|
|
41
|
+
and the whitespace guard dropped it.
|
|
42
|
+
|
|
43
|
+
- **An incremental `scan` broke every cross-file edge it touched**, rewriting it
|
|
44
|
+
with no destination at full confidence, until someone ran `--force`.
|
|
45
|
+
|
|
46
|
+
- **Two edges on one line could overwrite each other**, because the edge id
|
|
47
|
+
ignored the relation and the call it came from. `trim(value.trim())` lost the
|
|
48
|
+
direct call entirely.
|
|
49
|
+
|
|
50
|
+
- **A file over 1 MB or containing a NUL made the graph reparse the whole repo
|
|
51
|
+
forever**, and kept its stale symbols indefinitely.
|
|
52
|
+
|
|
53
|
+
- **A standing rule stopped being injected** once a project had more than 100
|
|
54
|
+
preferences, and `view` showed months-old rows as the current state past 500
|
|
55
|
+
observations. Both asked for the oldest N and sorted afterwards.
|
|
56
|
+
|
|
57
|
+
- **Committed work was announced as "not committed yet".** Landed work was
|
|
58
|
+
closed before the new rows were inserted, so a session's final turn was always
|
|
59
|
+
left open. Renamed files were also mis-parsed out of `git status`, closing
|
|
60
|
+
work whose rename had not landed.
|
|
61
|
+
|
|
62
|
+
- **A memory dropped by the prompt budget was suppressed for the rest of the
|
|
63
|
+
session**, having been marked as shown without being rendered.
|
|
64
|
+
|
|
65
|
+
- **An unclosed `<private>` tag was not redacted**, so a pasted credential was
|
|
66
|
+
stored and injected back into context. The README promises it is stripped.
|
|
67
|
+
|
|
68
|
+
- **The grep hook ignored `grep --recursive <symbol>`**, because its
|
|
69
|
+
invert-flag guard also matched `--recursive` and `--devices`.
|
|
70
|
+
|
|
71
|
+
- **`export` and `import` carried no sessions**, so a restored database reported
|
|
72
|
+
"none yet for this project" against a full memory. One truncated line also
|
|
73
|
+
aborted `import` after earlier batches had committed, with no report of what
|
|
74
|
+
had landed.
|
|
75
|
+
|
|
76
|
+
- **Postgres and Mongo diverged from SQLite in four places**: Mongo read a
|
|
77
|
+
zero-row timeline request as unlimited, scored Atlas vector hits on a
|
|
78
|
+
different scale so the relevance floor passed everything, and moved a
|
|
79
|
+
session's start time to when it ended; Postgres kept a stale vector when a row
|
|
80
|
+
was re-inserted without one, and could leave keyword search permanently broken
|
|
81
|
+
if it crashed while rebuilding the `tsv` column.
|
|
82
|
+
|
|
83
|
+
- **`usages --mode path X --target Y` searched for a target named `--target`.**
|
|
84
|
+
|
|
85
|
+
### Added
|
|
86
|
+
|
|
87
|
+
- `ListFilter` takes `newest` and `afterId`; `EdgeFilter` takes `dstName`;
|
|
88
|
+
stores expose `sessionProjects()`.
|
|
89
|
+
|
|
3
90
|
## 0.8.5
|
|
4
91
|
|
|
5
92
|
### Added
|
package/dist/capture/flush.js
CHANGED
|
@@ -8,18 +8,20 @@ export async function flushSession(ctx, sessionId, project, transcriptPath, rebu
|
|
|
8
8
|
const path = transcriptPath ?? transcriptPathFor(project, sessionId);
|
|
9
9
|
const cursor = readCursor(sessionId);
|
|
10
10
|
const { turns, nextOffset } = readTranscript(path, cursor);
|
|
11
|
-
await closeLandedWork(ctx.store, project);
|
|
12
11
|
if (turns.length === 0) {
|
|
12
|
+
await closeLandedWork(ctx.store, project);
|
|
13
13
|
writeCursor(sessionId, nextOffset);
|
|
14
14
|
return { observations: 0, summary: null };
|
|
15
15
|
}
|
|
16
16
|
const observations = observationsFromTurns(turns, sessionId, project, ctx.config);
|
|
17
17
|
if (observations.length === 0) {
|
|
18
|
+
await closeLandedWork(ctx.store, project);
|
|
18
19
|
writeCursor(sessionId, nextOffset);
|
|
19
20
|
return { observations: 0, summary: null };
|
|
20
21
|
}
|
|
21
22
|
await embedObservations(ctx, observations);
|
|
22
23
|
await ctx.store.insertObservations(observations);
|
|
24
|
+
await closeLandedWork(ctx.store, project);
|
|
23
25
|
const previous = rebuild ? null : await ctx.store.getSession(sessionId);
|
|
24
26
|
const summary = summarize(observations, previous?.summary);
|
|
25
27
|
await ctx.store.upsertSession({
|
package/dist/capture/progress.js
CHANGED
|
@@ -8,10 +8,21 @@ function uncommittedFiles(project) {
|
|
|
8
8
|
encoding: 'utf8',
|
|
9
9
|
maxBuffer: MAX_BUFFER,
|
|
10
10
|
});
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
const files = new Set();
|
|
12
|
+
const fields = raw.split('\0');
|
|
13
|
+
for (let i = 0; i < fields.length; i += 1) {
|
|
14
|
+
const entry = fields[i];
|
|
15
|
+
if (entry === undefined || entry.length <= 3)
|
|
16
|
+
continue;
|
|
17
|
+
files.add(join(project, entry.slice(3)));
|
|
18
|
+
if (/^[RC]/.test(entry) || /^.[RC]/.test(entry)) {
|
|
19
|
+
const origin = fields[i + 1];
|
|
20
|
+
if (origin !== undefined && origin.length > 0)
|
|
21
|
+
files.add(join(project, origin));
|
|
22
|
+
i += 1;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return files;
|
|
15
26
|
}
|
|
16
27
|
catch {
|
|
17
28
|
return null;
|
|
@@ -21,13 +32,34 @@ export async function closeLandedWork(store, project) {
|
|
|
21
32
|
const dirty = uncommittedFiles(project);
|
|
22
33
|
if (!dirty)
|
|
23
34
|
return 0;
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
35
|
+
const landed = [];
|
|
36
|
+
let after;
|
|
37
|
+
let afterId;
|
|
38
|
+
for (;;) {
|
|
39
|
+
const batch = await store.list({
|
|
40
|
+
project,
|
|
41
|
+
status: 'open',
|
|
42
|
+
...(after === undefined ? {} : { after }),
|
|
43
|
+
...(afterId === undefined ? {} : { afterId }),
|
|
44
|
+
limit: OPEN_LIMIT,
|
|
45
|
+
});
|
|
46
|
+
if (batch.length === 0)
|
|
47
|
+
break;
|
|
48
|
+
for (const obs of batch) {
|
|
49
|
+
if (obs.files.every((file) => !dirty.has(file)))
|
|
50
|
+
landed.push(obs.id);
|
|
51
|
+
}
|
|
52
|
+
const last = batch[batch.length - 1];
|
|
53
|
+
if (!last)
|
|
54
|
+
break;
|
|
55
|
+
after = last.createdAt;
|
|
56
|
+
afterId = last.id;
|
|
57
|
+
if (batch.length < OPEN_LIMIT)
|
|
58
|
+
break;
|
|
59
|
+
}
|
|
28
60
|
return store.closeObservations(landed);
|
|
29
61
|
}
|
|
30
62
|
export async function openWork(store, project) {
|
|
31
|
-
const open = await store.list({ project, status: 'open', limit: OPEN_LIMIT });
|
|
63
|
+
const open = await store.list({ project, status: 'open', limit: OPEN_LIMIT, newest: true });
|
|
32
64
|
return open.sort((a, b) => b.createdAt - a.createdAt);
|
|
33
65
|
}
|
|
@@ -31,8 +31,10 @@ export function readTranscript(path, fromOffset = 0) {
|
|
|
31
31
|
}
|
|
32
32
|
catch { }
|
|
33
33
|
}
|
|
34
|
+
const lastNewline = raw.lastIndexOf('\n');
|
|
35
|
+
const complete = lastNewline < 0 ? start : start + Buffer.byteLength(raw.slice(0, lastNewline + 1), 'utf8');
|
|
34
36
|
const turns = groupIntoTurns(entries);
|
|
35
|
-
const nextOffset = turns.length > 0 ? (turns[turns.length - 1]?.offset ??
|
|
37
|
+
const nextOffset = turns.length > 0 ? (turns[turns.length - 1]?.offset ?? complete) : complete;
|
|
36
38
|
return { turns, nextOffset };
|
|
37
39
|
}
|
|
38
40
|
finally {
|
|
@@ -42,8 +44,12 @@ export function readTranscript(path, fromOffset = 0) {
|
|
|
42
44
|
function groupIntoTurns(entries) {
|
|
43
45
|
const turns = [];
|
|
44
46
|
let current = null;
|
|
47
|
+
let lastKnown = 0;
|
|
45
48
|
for (const { entry, offset } of entries) {
|
|
46
|
-
const
|
|
49
|
+
const parsed = Date.parse(entry.timestamp ?? '');
|
|
50
|
+
const timestamp = Number.isNaN(parsed) ? lastKnown : parsed;
|
|
51
|
+
if (!Number.isNaN(parsed))
|
|
52
|
+
lastKnown = parsed;
|
|
47
53
|
if (entry.type === 'user') {
|
|
48
54
|
const text = extractText(entry.message?.content);
|
|
49
55
|
if (!text || entry.isCompactSummary || isSyntheticPrompt(text))
|
|
@@ -8,7 +8,7 @@ export function observationsFromTurns(turns, sessionId, project, config) {
|
|
|
8
8
|
.map((turn) => buildObservation(turn, sessionId, project, config));
|
|
9
9
|
}
|
|
10
10
|
function withoutPrivate(turn) {
|
|
11
|
-
const strip = (text) => text.replace(/<private>[\s\S]
|
|
11
|
+
const strip = (text) => text.replace(/<private>[\s\S]*?(?:<\/private>|$)/gi, '[private]');
|
|
12
12
|
return { ...turn, prompt: strip(turn.prompt), reasoning: strip(turn.reasoning) };
|
|
13
13
|
}
|
|
14
14
|
function isSubstantive(turn, config) {
|
|
@@ -165,7 +165,7 @@ function shortPath(path) {
|
|
|
165
165
|
function isExcluded(file, patterns) {
|
|
166
166
|
return patterns.some((pattern) => file.includes(pattern));
|
|
167
167
|
}
|
|
168
|
-
const PRIVATE_BLOCK = /<private>[\s\S]
|
|
168
|
+
const PRIVATE_BLOCK = /<private>[\s\S]*?(?:<\/private>|$)/gi;
|
|
169
169
|
const PRIVATE_KEY = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g;
|
|
170
170
|
export function redact(text) {
|
|
171
171
|
return text
|
|
@@ -13,8 +13,15 @@ export async function cmdScan(argv) {
|
|
|
13
13
|
const known = force
|
|
14
14
|
? new Map()
|
|
15
15
|
: new Map((await ctx.store.scannedFiles(project)).map((f) => [f.path, f.hash]));
|
|
16
|
+
const existing = force ? [] : await ctx.store.findSymbols({ project, limit: 200000 });
|
|
16
17
|
const started = Date.now();
|
|
17
|
-
const scan = scanRepository({
|
|
18
|
+
const scan = scanRepository({
|
|
19
|
+
root,
|
|
20
|
+
project,
|
|
21
|
+
known,
|
|
22
|
+
...(force ? { force } : {}),
|
|
23
|
+
...(existing.length > 0 ? { existing } : {}),
|
|
24
|
+
});
|
|
18
25
|
if (force)
|
|
19
26
|
await ctx.store.removeGraph(project);
|
|
20
27
|
else if (scan.changed.length > 0)
|
|
@@ -43,7 +50,8 @@ export async function cmdUsages(argv) {
|
|
|
43
50
|
const context = Number(valueOf(argv, '--context') ?? 0);
|
|
44
51
|
const limit = Number(valueOf(argv, '--limit') ?? 100);
|
|
45
52
|
const mode = valueOf(argv, '--mode') ?? 'text';
|
|
46
|
-
const
|
|
53
|
+
const targetFlag = valueOf(argv, '--target');
|
|
54
|
+
const words = withoutFlags(argv, ['--path', '--context', '--limit', '--mode', '--target'])
|
|
47
55
|
.filter((arg) => arg !== '--regex')
|
|
48
56
|
.join(' ')
|
|
49
57
|
.trim();
|
|
@@ -69,7 +77,8 @@ export async function cmdUsages(argv) {
|
|
|
69
77
|
console.error(`Unknown --mode "${mode}". Use text, usages, explain or path.`);
|
|
70
78
|
process.exit(1);
|
|
71
79
|
}
|
|
72
|
-
const [symbol,
|
|
80
|
+
const [symbol, positional] = words.split(/\s+/);
|
|
81
|
+
const target = targetFlag ?? positional;
|
|
73
82
|
if (mode === 'path' && !target) {
|
|
74
83
|
console.error('--mode path needs two symbols: claude-db usages --mode path <from> <to>');
|
|
75
84
|
process.exit(1);
|
|
@@ -79,16 +79,22 @@ async function syncSessions(local, remote) {
|
|
|
79
79
|
return moved;
|
|
80
80
|
}
|
|
81
81
|
async function eachRemoteObservation(store, visit) {
|
|
82
|
-
let after
|
|
82
|
+
let after;
|
|
83
|
+
let afterId;
|
|
83
84
|
for (;;) {
|
|
84
|
-
const batch = await store.list({
|
|
85
|
+
const batch = await store.list({
|
|
86
|
+
...(after === undefined ? {} : { after }),
|
|
87
|
+
...(afterId === undefined ? {} : { afterId }),
|
|
88
|
+
limit: BATCH,
|
|
89
|
+
});
|
|
85
90
|
if (batch.length === 0)
|
|
86
91
|
return;
|
|
87
92
|
await visit(batch);
|
|
88
93
|
const last = batch[batch.length - 1];
|
|
89
94
|
if (!last)
|
|
90
95
|
return;
|
|
91
|
-
after = last.createdAt
|
|
96
|
+
after = last.createdAt;
|
|
97
|
+
afterId = last.id;
|
|
92
98
|
if (batch.length < BATCH)
|
|
93
99
|
return;
|
|
94
100
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { BATCH } from '../constants.js';
|
|
2
|
+
const SESSION_LIMIT = 10000;
|
|
2
3
|
import { createContext } from '../../context.js';
|
|
3
4
|
import { embedObservations } from '../../capture/index.js';
|
|
4
5
|
import { loadConfig } from '../../config/index.js';
|
|
@@ -6,10 +7,16 @@ import { readFileSync } from 'node:fs';
|
|
|
6
7
|
import { resolveProject } from '../../util/project.js';
|
|
7
8
|
import { valueOf } from '../args.js';
|
|
8
9
|
export async function eachObservation(ctx, filter, visit) {
|
|
9
|
-
let after
|
|
10
|
+
let after;
|
|
11
|
+
let afterId;
|
|
10
12
|
let total = 0;
|
|
11
13
|
for (;;) {
|
|
12
|
-
const batch = await ctx.store.list({
|
|
14
|
+
const batch = await ctx.store.list({
|
|
15
|
+
...filter,
|
|
16
|
+
...(after === undefined ? {} : { after }),
|
|
17
|
+
...(afterId === undefined ? {} : { afterId }),
|
|
18
|
+
limit: BATCH,
|
|
19
|
+
});
|
|
13
20
|
if (batch.length === 0)
|
|
14
21
|
return total;
|
|
15
22
|
await visit(batch);
|
|
@@ -17,7 +24,8 @@ export async function eachObservation(ctx, filter, visit) {
|
|
|
17
24
|
const last = batch[batch.length - 1];
|
|
18
25
|
if (!last)
|
|
19
26
|
return total;
|
|
20
|
-
after = last.createdAt
|
|
27
|
+
after = last.createdAt;
|
|
28
|
+
afterId = last.id;
|
|
21
29
|
if (batch.length < BATCH)
|
|
22
30
|
return total;
|
|
23
31
|
}
|
|
@@ -30,7 +38,22 @@ export async function cmdExport(argv) {
|
|
|
30
38
|
for (const obs of batch)
|
|
31
39
|
process.stdout.write(`${JSON.stringify(obs)}\n`);
|
|
32
40
|
});
|
|
33
|
-
|
|
41
|
+
const projects = all
|
|
42
|
+
? [
|
|
43
|
+
...new Set([
|
|
44
|
+
...(await ctx.store.listProjects()).map((entry) => entry.project),
|
|
45
|
+
...(await ctx.store.sessionProjects()),
|
|
46
|
+
]),
|
|
47
|
+
]
|
|
48
|
+
: [resolveProject(undefined)];
|
|
49
|
+
let sessions = 0;
|
|
50
|
+
for (const project of projects) {
|
|
51
|
+
for (const session of await ctx.store.recentSessions(project, SESSION_LIMIT)) {
|
|
52
|
+
process.stdout.write(`${JSON.stringify({ ...session, _type: 'session' })}\n`);
|
|
53
|
+
sessions += 1;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
process.stderr.write(`${count} observation(s) and ${sessions} session(s) exported.\n`);
|
|
34
57
|
}
|
|
35
58
|
finally {
|
|
36
59
|
await ctx.close();
|
|
@@ -46,20 +69,47 @@ export async function cmdImport(path) {
|
|
|
46
69
|
.filter((line) => line.trim().length > 0);
|
|
47
70
|
const ctx = await createContext();
|
|
48
71
|
let imported = 0;
|
|
72
|
+
let skipped = 0;
|
|
73
|
+
let restored = 0;
|
|
49
74
|
try {
|
|
50
75
|
for (let i = 0; i < lines.length; i += BATCH) {
|
|
51
|
-
const batch =
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
76
|
+
const batch = [];
|
|
77
|
+
for (const line of lines.slice(i, i + BATCH)) {
|
|
78
|
+
let parsed;
|
|
79
|
+
try {
|
|
80
|
+
parsed = JSON.parse(line);
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
skipped += 1;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
if (parsed._type === 'session') {
|
|
87
|
+
const { _type, ...session } = parsed;
|
|
88
|
+
void _type;
|
|
89
|
+
if (typeof session.id === 'string' && typeof session.project === 'string') {
|
|
90
|
+
await ctx.store.upsertSession(session);
|
|
91
|
+
restored += 1;
|
|
92
|
+
}
|
|
93
|
+
else
|
|
94
|
+
skipped += 1;
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
if (typeof parsed.id === 'string' && typeof parsed.project === 'string')
|
|
98
|
+
batch.push(parsed);
|
|
99
|
+
else
|
|
100
|
+
skipped += 1;
|
|
101
|
+
}
|
|
102
|
+
if (batch.length > 0)
|
|
103
|
+
await ctx.store.insertObservations(batch);
|
|
56
104
|
imported += batch.length;
|
|
57
105
|
}
|
|
58
106
|
}
|
|
59
107
|
finally {
|
|
60
108
|
await ctx.close();
|
|
61
109
|
}
|
|
62
|
-
console.log(`Imported ${imported} observation(s)
|
|
110
|
+
console.log(`Imported ${imported} observation(s)` +
|
|
111
|
+
`${restored > 0 ? `, ${restored} session(s)` : ''}` +
|
|
112
|
+
`${skipped > 0 ? `, skipped ${skipped} unreadable line(s)` : ''}.`);
|
|
63
113
|
}
|
|
64
114
|
export async function cmdPrune(argv) {
|
|
65
115
|
const days = Number(valueOf(argv, '--older-than') ?? NaN);
|
|
@@ -7,7 +7,7 @@ import { resolveProject } from '../../util/project.js';
|
|
|
7
7
|
import { toShortId } from '../../util/shortid.js';
|
|
8
8
|
import { renderPage } from './view-page.js';
|
|
9
9
|
async function collect(ctx, project) {
|
|
10
|
-
const all = (await ctx.store.list({ project, limit: 500 })).sort((a, b) => b.createdAt - a.createdAt);
|
|
10
|
+
const all = (await ctx.store.list({ project, limit: 500, newest: true })).sort((a, b) => b.createdAt - a.createdAt);
|
|
11
11
|
const kinds = {};
|
|
12
12
|
for (const obs of all)
|
|
13
13
|
kinds[obs.kind] = (kinds[obs.kind] ?? 0) + 1;
|
|
@@ -104,6 +104,11 @@ export const BASIC_FINGERPRINT = [
|
|
|
104
104
|
CALLABLE.source,
|
|
105
105
|
CALL.source,
|
|
106
106
|
Object.keys(LABELS).join(','),
|
|
107
|
+
Object.entries(KEYWORDS)
|
|
108
|
+
.map(([word, kind]) => `${word}:${kind}`)
|
|
109
|
+
.sort()
|
|
110
|
+
.join(','),
|
|
111
|
+
[...NOT_A_CALL].sort().join(','),
|
|
107
112
|
].join('|');
|
|
108
113
|
export const BASIC_LANGUAGES = Object.entries(LABELS).map(([label, extensions]) => ({
|
|
109
114
|
id: 'basic',
|
|
@@ -29,11 +29,15 @@ export async function queryGraph(store, project, query) {
|
|
|
29
29
|
const ids = answer.definitions.map((symbol) => symbol.id);
|
|
30
30
|
const edges = await store.findEdges({
|
|
31
31
|
project,
|
|
32
|
-
...(ids.length > 0 ? { srcIds: ids, dstIds: ids } : {}),
|
|
32
|
+
...(ids.length > 0 ? { srcIds: ids, dstIds: ids } : { dstName: query.symbol }),
|
|
33
33
|
limit: Math.max(query.limit * 10, 500),
|
|
34
34
|
});
|
|
35
|
-
|
|
36
|
-
|
|
35
|
+
const idSet = new Set(ids);
|
|
36
|
+
const pointsAtSymbol = (edge) => edge.dstName === query.symbol ||
|
|
37
|
+
edge.dstName.endsWith(`.${query.symbol}`) ||
|
|
38
|
+
(edge.dstId !== '' && idSet.has(edge.dstId));
|
|
39
|
+
answer.inbound = edges.filter(pointsAtSymbol);
|
|
40
|
+
answer.outbound = edges.filter((edge) => idSet.has(edge.srcId) && !pointsAtSymbol(edge));
|
|
37
41
|
answer.empty = answer.definitions.length === 0 && answer.inbound.length === 0;
|
|
38
42
|
if (answer.empty)
|
|
39
43
|
answer.suggestions = await suggestFor(store, project, query.symbol);
|
|
@@ -87,6 +87,18 @@ function enclosing(line, spans) {
|
|
|
87
87
|
}
|
|
88
88
|
return (best ?? fallback)?.symbol ?? null;
|
|
89
89
|
}
|
|
90
|
+
function namedTargets(node) {
|
|
91
|
+
if (!/\s/.test(node.text().trim()))
|
|
92
|
+
return [node];
|
|
93
|
+
const inner = [];
|
|
94
|
+
for (const kind of ['type_identifier', 'identifier']) {
|
|
95
|
+
try {
|
|
96
|
+
inner.push(...node.findAll({ rule: { kind } }));
|
|
97
|
+
}
|
|
98
|
+
catch { }
|
|
99
|
+
}
|
|
100
|
+
return inner.length > 0 ? inner : [node];
|
|
101
|
+
}
|
|
90
102
|
export function extractFile(file, project) {
|
|
91
103
|
if (file.spec.basic)
|
|
92
104
|
return extractByPattern(file, project);
|
|
@@ -132,31 +144,39 @@ export function extractFile(file, project) {
|
|
|
132
144
|
relation: 'defines',
|
|
133
145
|
line: span.start,
|
|
134
146
|
from: owner,
|
|
147
|
+
to: span.symbol.id,
|
|
135
148
|
});
|
|
136
149
|
}
|
|
137
150
|
for (const rule of file.spec.references) {
|
|
138
151
|
for (const node of root.findAll({ rule: { kind: rule.kind } })) {
|
|
139
|
-
const
|
|
140
|
-
|
|
141
|
-
if (!target || !raw)
|
|
142
|
-
continue;
|
|
143
|
-
const name = unquote(raw.trim());
|
|
144
|
-
if (!name || /\s/.test(name))
|
|
145
|
-
continue;
|
|
146
|
-
if (rule.relation === 'references' && name.includes('.'))
|
|
147
|
-
continue;
|
|
148
|
-
if (rule.namePattern && !rule.namePattern.test(name))
|
|
152
|
+
const matched = rule.field.length === 0 ? node : resolveField(node, rule.field);
|
|
153
|
+
if (!matched)
|
|
149
154
|
continue;
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
name
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
155
|
+
const targets = rule.field.length === 0 ? namedTargets(matched) : [matched];
|
|
156
|
+
for (const target of targets) {
|
|
157
|
+
const raw = target.text();
|
|
158
|
+
if (!raw)
|
|
159
|
+
continue;
|
|
160
|
+
const name = unquote(raw.trim());
|
|
161
|
+
if (!name || /\s/.test(name))
|
|
162
|
+
continue;
|
|
163
|
+
if (rule.relation === 'references' && name.includes('.'))
|
|
164
|
+
continue;
|
|
165
|
+
if (rule.namePattern && !rule.namePattern.test(name))
|
|
166
|
+
continue;
|
|
167
|
+
if (rule.excludeParents?.includes(node.parent()?.kind() ?? ''))
|
|
168
|
+
continue;
|
|
169
|
+
const line = target.range().start.line + 1;
|
|
170
|
+
const span = node.range();
|
|
171
|
+
references.push({
|
|
172
|
+
file: file.path,
|
|
173
|
+
name,
|
|
174
|
+
relation: rule.relation,
|
|
175
|
+
line,
|
|
176
|
+
from: enclosing(line, spans),
|
|
177
|
+
origin: `${span.start.line}:${span.start.column}-${span.end.line}:${span.end.column}`,
|
|
178
|
+
});
|
|
179
|
+
}
|
|
160
180
|
}
|
|
161
181
|
}
|
|
162
182
|
return { symbols, references };
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import type { LanguageSpec } from '../languages/index.js';
|
|
2
2
|
export interface SourceFile {
|
|
3
|
+
unreadable?: boolean;
|
|
3
4
|
path: string;
|
|
4
5
|
spec: LanguageSpec;
|
|
5
6
|
source: string;
|
|
6
7
|
hash: string;
|
|
7
8
|
}
|
|
8
|
-
export declare const SCAN_VERSION =
|
|
9
|
+
export declare const SCAN_VERSION = 5;
|
|
9
10
|
export declare function hashOf(bytes: Buffer): string;
|
|
10
11
|
export declare function listFiles(root: string): string[];
|
|
11
12
|
export declare function sourceFiles(root: string): string[];
|
package/dist/graph/scan/files.js
CHANGED
|
@@ -5,7 +5,7 @@ import { join } from 'node:path';
|
|
|
5
5
|
import { BASIC_FINGERPRINT, LANGUAGES, languageFor } from '../languages/index.js';
|
|
6
6
|
const MAX_BUFFER = 64 * 1024 * 1024;
|
|
7
7
|
const MAX_FILE_BYTES = 1024 * 1024;
|
|
8
|
-
export const SCAN_VERSION =
|
|
8
|
+
export const SCAN_VERSION = 5;
|
|
9
9
|
const RULES_FINGERPRINT = createHash('sha256')
|
|
10
10
|
.update(JSON.stringify(LANGUAGES, (_key, value) => (value instanceof RegExp ? String(value) : value)))
|
|
11
11
|
.update(BASIC_FINGERPRINT)
|
|
@@ -40,8 +40,9 @@ export function readSource(root, path) {
|
|
|
40
40
|
catch {
|
|
41
41
|
return null;
|
|
42
42
|
}
|
|
43
|
-
if (bytes.length > MAX_FILE_BYTES || bytes.includes(0))
|
|
44
|
-
return
|
|
43
|
+
if (bytes.length > MAX_FILE_BYTES || bytes.includes(0)) {
|
|
44
|
+
return { path, spec, source: '', hash: hashOf(bytes), unreadable: true };
|
|
45
|
+
}
|
|
45
46
|
return { path, spec, source: bytes.toString('utf8'), hash: hashOf(bytes) };
|
|
46
47
|
}
|
|
47
48
|
export function currentHashes(root) {
|
package/dist/graph/scan/index.js
CHANGED
|
@@ -26,6 +26,12 @@ export function scanRepository(options) {
|
|
|
26
26
|
result.skipped += 1;
|
|
27
27
|
continue;
|
|
28
28
|
}
|
|
29
|
+
if (file.unreadable) {
|
|
30
|
+
result.unsupported += 1;
|
|
31
|
+
result.changed.push(path);
|
|
32
|
+
result.files.push({ project, path, hash: file.hash, scannedAt });
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
29
35
|
let extracted;
|
|
30
36
|
try {
|
|
31
37
|
extracted = extractFile(file, project);
|
|
@@ -39,6 +45,10 @@ export function scanRepository(options) {
|
|
|
39
45
|
result.changed.push(path);
|
|
40
46
|
result.files.push({ project, path, hash: file.hash, scannedAt });
|
|
41
47
|
}
|
|
42
|
-
|
|
48
|
+
const rescanned = new Set(result.changed);
|
|
49
|
+
const resolvable = options.existing === undefined
|
|
50
|
+
? result.symbols
|
|
51
|
+
: [...result.symbols, ...options.existing.filter((s) => !rescanned.has(s.file))];
|
|
52
|
+
result.edges = resolveEdges(project, resolvable, references);
|
|
43
53
|
return result;
|
|
44
54
|
}
|
|
@@ -2,8 +2,8 @@ import { observationId } from '../../capture/identity.js';
|
|
|
2
2
|
const SAME_FILE = 1;
|
|
3
3
|
const SINGLE_MATCH = 0.95;
|
|
4
4
|
const AMBIGUOUS_MATCH = 0.85;
|
|
5
|
-
function edgeId(project, file, line, from, to) {
|
|
6
|
-
return observationId('graph', 0, `${project}\0${file}\0${line}\0${from}\0${to}`);
|
|
5
|
+
function edgeId(project, file, line, from, to, relation, origin) {
|
|
6
|
+
return observationId('graph', 0, `${project}\0${file}\0${line}\0${from}\0${to}\0${relation}\0${origin}`);
|
|
7
7
|
}
|
|
8
8
|
export function resolveEdges(project, symbols, references) {
|
|
9
9
|
const byName = new Map();
|
|
@@ -15,9 +15,13 @@ export function resolveEdges(project, symbols, references) {
|
|
|
15
15
|
byName.set(symbol.name, [symbol]);
|
|
16
16
|
}
|
|
17
17
|
const edges = [];
|
|
18
|
+
const origins = new Map();
|
|
18
19
|
for (const reference of references) {
|
|
19
20
|
const candidates = byName.get(reference.name) ?? [];
|
|
20
|
-
const
|
|
21
|
+
const exact = reference.to
|
|
22
|
+
? candidates.find((candidate) => candidate.id === reference.to)
|
|
23
|
+
: undefined;
|
|
24
|
+
const local = exact ?? candidates.find((candidate) => candidate.file === reference.file);
|
|
21
25
|
const target = local ?? candidates[0];
|
|
22
26
|
if (!target && reference.weak)
|
|
23
27
|
continue;
|
|
@@ -33,7 +37,7 @@ export function resolveEdges(project, symbols, references) {
|
|
|
33
37
|
}
|
|
34
38
|
const fromName = reference.from?.name ?? reference.file;
|
|
35
39
|
edges.push({
|
|
36
|
-
id: edgeId(project, reference.file, reference.line, fromName, reference.name),
|
|
40
|
+
id: edgeId(project, reference.file, reference.line, fromName, reference.name, reference.relation, reference.origin ?? ''),
|
|
37
41
|
project,
|
|
38
42
|
srcId: reference.from?.id ?? '',
|
|
39
43
|
srcName: fromName,
|
|
@@ -45,10 +49,13 @@ export function resolveEdges(project, symbols, references) {
|
|
|
45
49
|
file: reference.file,
|
|
46
50
|
line: reference.line,
|
|
47
51
|
});
|
|
52
|
+
const added = edges[edges.length - 1];
|
|
53
|
+
if (added && reference.origin !== undefined)
|
|
54
|
+
origins.set(added.id, reference.origin);
|
|
48
55
|
}
|
|
49
|
-
return mergeMemberCalls(edges);
|
|
56
|
+
return mergeMemberCalls(edges, origins);
|
|
50
57
|
}
|
|
51
|
-
function mergeMemberCalls(edges) {
|
|
58
|
+
function mergeMemberCalls(edges, origins) {
|
|
52
59
|
const byLine = new Map();
|
|
53
60
|
for (const edge of edges) {
|
|
54
61
|
const key = `${edge.file}\0${edge.line}`;
|
|
@@ -70,6 +77,10 @@ function mergeMemberCalls(edges) {
|
|
|
70
77
|
continue;
|
|
71
78
|
if (bare.dstName !== tail || bare.relation !== qualified.relation)
|
|
72
79
|
continue;
|
|
80
|
+
const from = origins.get(qualified.id);
|
|
81
|
+
const to = origins.get(bare.id);
|
|
82
|
+
if (from === undefined || to === undefined || from !== to)
|
|
83
|
+
continue;
|
|
73
84
|
if (!qualified.dstId && bare.dstId) {
|
|
74
85
|
qualified.dstId = bare.dstId;
|
|
75
86
|
qualified.confidence = bare.confidence;
|
|
@@ -16,7 +16,7 @@ export function symbolsGreppedIn(command) {
|
|
|
16
16
|
const rest = match[5] ?? '';
|
|
17
17
|
if (!SYMBOL.test(pattern) && !WORD.test(pattern))
|
|
18
18
|
continue;
|
|
19
|
-
if (
|
|
19
|
+
if (/(?:^|\s)-[A-Za-z]*v|--invert-match/.test(flags))
|
|
20
20
|
continue;
|
|
21
21
|
const invocation = match[0];
|
|
22
22
|
const piped = /^\s*\|/.test(invocation);
|
|
@@ -3,4 +3,4 @@ export declare function overlapCount(prompt: string, entry: {
|
|
|
3
3
|
title: string;
|
|
4
4
|
snippet?: string | undefined;
|
|
5
5
|
}): number;
|
|
6
|
-
export declare function renderPromptContext(entries: ObservationIndexEntry[], maxChars: number, expanded?: Observation[], expandMaxChars?: number): string | null;
|
|
6
|
+
export declare function renderPromptContext(entries: ObservationIndexEntry[], maxChars: number, expanded?: Observation[], expandMaxChars?: number, shown?: string[]): string | null;
|
package/dist/hooks/relevance.js
CHANGED
|
@@ -9,7 +9,7 @@ export function overlapCount(prompt, entry) {
|
|
|
9
9
|
shared++;
|
|
10
10
|
return shared;
|
|
11
11
|
}
|
|
12
|
-
export function renderPromptContext(entries, maxChars, expanded = [], expandMaxChars = 900) {
|
|
12
|
+
export function renderPromptContext(entries, maxChars, expanded = [], expandMaxChars = 900, shown = []) {
|
|
13
13
|
if (entries.length === 0)
|
|
14
14
|
return null;
|
|
15
15
|
const expandedIds = new Set(expanded.map((obs) => obs.id));
|
|
@@ -17,6 +17,7 @@ export function renderPromptContext(entries, maxChars, expanded = [], expandMaxC
|
|
|
17
17
|
for (const obs of expanded) {
|
|
18
18
|
const date = new Date(obs.createdAt).toISOString().slice(0, 10);
|
|
19
19
|
sections.push(`${toShortId(obs.id)} ${obs.kind} ${date} ${obs.title}\n` + clip(obs.body, expandMaxChars));
|
|
20
|
+
shown.push(obs.id);
|
|
20
21
|
}
|
|
21
22
|
const lines = [];
|
|
22
23
|
let budget = maxChars;
|
|
@@ -30,6 +31,7 @@ export function renderPromptContext(entries, maxChars, expanded = [], expandMaxC
|
|
|
30
31
|
break;
|
|
31
32
|
budget -= line.length + 1;
|
|
32
33
|
lines.push(line);
|
|
34
|
+
shown.push(entry.id);
|
|
33
35
|
}
|
|
34
36
|
if (sections.length === 0 && lines.length === 0)
|
|
35
37
|
return null;
|
|
@@ -78,7 +78,7 @@ await runHook(async () => {
|
|
|
78
78
|
lines.push(`- ${obs.title}`);
|
|
79
79
|
}
|
|
80
80
|
}
|
|
81
|
-
const rules = (await ctx.store.list({ project, kind: 'preference', limit: 100 }))
|
|
81
|
+
const rules = (await ctx.store.list({ project, kind: 'preference', limit: 100, newest: true }))
|
|
82
82
|
.sort((a, b) => {
|
|
83
83
|
const manual = Number(b.sessionId === 'manual') - Number(a.sessionId === 'manual');
|
|
84
84
|
return manual !== 0 ? manual : b.createdAt - a.createdAt;
|
|
@@ -40,10 +40,11 @@ await runHook(async () => {
|
|
|
40
40
|
const expanded = toExpand.length > 0
|
|
41
41
|
? await ctx.search.getObservations(toExpand.map((entry) => entry.id))
|
|
42
42
|
: [];
|
|
43
|
-
const
|
|
43
|
+
const emitted = [];
|
|
44
|
+
const block = renderPromptContext(entries, ctx.config.inject.promptMaxChars, expanded, ctx.config.inject.expandMaxChars, emitted);
|
|
44
45
|
if (!block)
|
|
45
46
|
return;
|
|
46
|
-
markShown(sessionId,
|
|
47
|
+
markShown(sessionId, emitted);
|
|
47
48
|
emitContext(`${block}\n(context ≈ ${Math.round(block.length / 4)} tokens)\n`);
|
|
48
49
|
}
|
|
49
50
|
finally {
|
package/dist/store/adapter.d.ts
CHANGED
|
@@ -12,6 +12,7 @@ export interface MemoryStore {
|
|
|
12
12
|
upsertSession(session: Session): Promise<void>;
|
|
13
13
|
getSession(id: string): Promise<Session | null>;
|
|
14
14
|
recentSessions(project: string, limit: number): Promise<Session[]>;
|
|
15
|
+
sessionProjects(): Promise<string[]>;
|
|
15
16
|
clearSummary(id: string): Promise<boolean>;
|
|
16
17
|
insertObservations(observations: Observation[]): Promise<void>;
|
|
17
18
|
getObservations(ids: string[]): Promise<Observation[]>;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { toEdge, toSymbol, upsertsOf } from './docs.js';
|
|
2
|
+
import { escapeRegex } from './helpers.js';
|
|
2
3
|
export async function upsertGraph(symbols, edges, scanned, scan) {
|
|
3
4
|
if (scan.symbols.length > 0) {
|
|
4
5
|
await symbols.bulkWrite(upsertsOf(scan.symbols, (s) => s.id), { ordered: false });
|
|
@@ -34,8 +35,14 @@ export async function findEdges(symbols, edges, scanned, filter) {
|
|
|
34
35
|
}
|
|
35
36
|
if (alternatives.length > 0)
|
|
36
37
|
query['$or'] = alternatives;
|
|
38
|
+
if (filter.dstName !== undefined) {
|
|
39
|
+
query['dstName'] = {
|
|
40
|
+
$in: [filter.dstName, new RegExp(`\\.${escapeRegex(filter.dstName)}$`)],
|
|
41
|
+
};
|
|
42
|
+
}
|
|
37
43
|
const docs = await edges
|
|
38
44
|
.find(query)
|
|
45
|
+
.sort({ file: 1, line: 1, _id: 1 })
|
|
39
46
|
.limit(filter.limit ?? 2000)
|
|
40
47
|
.toArray();
|
|
41
48
|
return docs.map(toEdge);
|
|
@@ -18,6 +18,7 @@ export declare class MongoStore implements MemoryStore {
|
|
|
18
18
|
upsertSession(session: Session): Promise<void>;
|
|
19
19
|
getSession(id: string): Promise<Session | null>;
|
|
20
20
|
recentSessions(project: string, limit: number): Promise<Session[]>;
|
|
21
|
+
sessionProjects(): Promise<string[]>;
|
|
21
22
|
clearSummary(id: string): Promise<boolean>;
|
|
22
23
|
insertObservations(observations: Observation[]): Promise<void>;
|
|
23
24
|
getObservations(ids: string[]): Promise<Observation[]>;
|
|
@@ -60,6 +60,9 @@ export class MongoStore {
|
|
|
60
60
|
async recentSessions(project, limit) {
|
|
61
61
|
return sessionsOps.recentSessions(this.sessions, project, limit);
|
|
62
62
|
}
|
|
63
|
+
async sessionProjects() {
|
|
64
|
+
return sessionsOps.sessionProjects(this.sessions);
|
|
65
|
+
}
|
|
63
66
|
async clearSummary(id) {
|
|
64
67
|
return sessionsOps.clearSummary(this.sessions, id);
|
|
65
68
|
}
|
|
@@ -65,11 +65,19 @@ export async function list(observations, sessions, symbols, edges, scanned, filt
|
|
|
65
65
|
query['kind'] = filter.kind;
|
|
66
66
|
if (filter.status)
|
|
67
67
|
query['status'] = filter.status;
|
|
68
|
-
if (filter.after !== undefined)
|
|
69
|
-
|
|
68
|
+
if (filter.after !== undefined) {
|
|
69
|
+
if (filter.afterId === undefined)
|
|
70
|
+
query['createdAt'] = { $gt: filter.after };
|
|
71
|
+
else
|
|
72
|
+
query['$or'] = [
|
|
73
|
+
{ createdAt: { $gt: filter.after } },
|
|
74
|
+
{ createdAt: filter.after, _id: { $gt: filter.afterId } },
|
|
75
|
+
];
|
|
76
|
+
}
|
|
77
|
+
const direction = filter.newest ? -1 : 1;
|
|
70
78
|
const docs = await observations
|
|
71
79
|
.find(query)
|
|
72
|
-
.sort({ createdAt:
|
|
80
|
+
.sort({ createdAt: direction, _id: direction })
|
|
73
81
|
.limit(filter.limit ?? 1000)
|
|
74
82
|
.toArray();
|
|
75
83
|
return docs.map(toObservation);
|
|
@@ -32,11 +32,13 @@ export async function timeline(observations, query) {
|
|
|
32
32
|
.sort({ createdAt: -1 })
|
|
33
33
|
.limit(query.before + 1)
|
|
34
34
|
.toArray(),
|
|
35
|
-
|
|
36
|
-
.
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
35
|
+
query.after <= 0
|
|
36
|
+
? Promise.resolve([])
|
|
37
|
+
: observations
|
|
38
|
+
.find({ project: anchor.project, createdAt: { $gt: anchor.createdAt } }, { projection })
|
|
39
|
+
.sort({ createdAt: 1 })
|
|
40
|
+
.limit(query.after)
|
|
41
|
+
.toArray(),
|
|
40
42
|
]);
|
|
41
43
|
return [...before.reverse(), ...after].map((doc) => toIndexEntry(doc, 0));
|
|
42
44
|
}
|
|
@@ -5,3 +5,4 @@ export declare function upsertSession(sessions: Collection<SessionDoc>, session:
|
|
|
5
5
|
export declare function getSession(sessions: Collection<SessionDoc>, id: string): Promise<Session | null>;
|
|
6
6
|
export declare function clearSummary(sessions: Collection<SessionDoc>, id: string): Promise<boolean>;
|
|
7
7
|
export declare function recentSessions(sessions: Collection<SessionDoc>, project: string, limit: number): Promise<Session[]>;
|
|
8
|
+
export declare function sessionProjects(sessions: Collection<SessionDoc>): Promise<string[]>;
|
|
@@ -1,14 +1,11 @@
|
|
|
1
1
|
import { toSession } from './docs.js';
|
|
2
2
|
export async function upsertSession(sessions, session) {
|
|
3
|
-
const set = {
|
|
4
|
-
project: session.project,
|
|
5
|
-
startedAt: session.startedAt,
|
|
6
|
-
};
|
|
3
|
+
const set = { project: session.project };
|
|
7
4
|
if (session.endedAt !== undefined)
|
|
8
5
|
set.endedAt = session.endedAt;
|
|
9
6
|
if (session.summary !== undefined)
|
|
10
7
|
set.summary = session.summary;
|
|
11
|
-
await sessions.updateOne({ _id: session.id }, { $set: set }, { upsert: true });
|
|
8
|
+
await sessions.updateOne({ _id: session.id }, { $set: set, $setOnInsert: { startedAt: session.startedAt } }, { upsert: true });
|
|
12
9
|
}
|
|
13
10
|
export async function getSession(sessions, id) {
|
|
14
11
|
const doc = await sessions.findOne({ _id: id });
|
|
@@ -26,3 +23,9 @@ export async function recentSessions(sessions, project, limit) {
|
|
|
26
23
|
.toArray();
|
|
27
24
|
return docs.map(toSession);
|
|
28
25
|
}
|
|
26
|
+
export async function sessionProjects(sessions) {
|
|
27
|
+
const rows = await sessions
|
|
28
|
+
.aggregate([{ $group: { _id: '$project' } }])
|
|
29
|
+
.toArray();
|
|
30
|
+
return rows.map((row) => String(row._id ?? '')).filter((project) => project.length > 0);
|
|
31
|
+
}
|
|
@@ -37,7 +37,7 @@ export async function searchVector(observations, cache, vector, query) {
|
|
|
37
37
|
},
|
|
38
38
|
])
|
|
39
39
|
.toArray();
|
|
40
|
-
return docs.map((doc) => toIndexEntry(doc, doc.score));
|
|
40
|
+
return docs.map((doc) => toIndexEntry(doc, Number(doc.score) * 2 - 1));
|
|
41
41
|
}
|
|
42
42
|
const docs = await observations
|
|
43
43
|
.find({
|
|
@@ -96,9 +96,14 @@ export async function findEdges(pool, filter) {
|
|
|
96
96
|
values.push(ids);
|
|
97
97
|
sides.push(`${column} = ANY($${values.length}::text[])`);
|
|
98
98
|
}
|
|
99
|
-
|
|
99
|
+
let where = sides.length > 0 ? ` AND (${sides.join(' OR ')})` : '';
|
|
100
|
+
if (filter.dstName !== undefined) {
|
|
101
|
+
values.push(filter.dstName, `%.${filter.dstName}`);
|
|
102
|
+
where += ` AND (dst_name = $${values.length - 1} OR dst_name LIKE $${values.length})`;
|
|
103
|
+
}
|
|
100
104
|
values.push(filter.limit ?? 2000);
|
|
101
|
-
const res = await pool.query(`SELECT * FROM symbol_edges WHERE project = $1${where}
|
|
105
|
+
const res = await pool.query(`SELECT * FROM symbol_edges WHERE project = $1${where}
|
|
106
|
+
ORDER BY file, line, id LIMIT $${values.length}`, values);
|
|
102
107
|
return res.rows.map(toEdge);
|
|
103
108
|
}
|
|
104
109
|
export async function scannedFiles(pool, project) {
|
|
@@ -23,6 +23,7 @@ export declare class PostgresStore implements MemoryStore {
|
|
|
23
23
|
ping(): Promise<boolean>;
|
|
24
24
|
upsertSession(session: Session): Promise<void>;
|
|
25
25
|
getSession(id: string): Promise<Session | null>;
|
|
26
|
+
sessionProjects(): Promise<string[]>;
|
|
26
27
|
recentSessions(project: string, limit: number): Promise<Session[]>;
|
|
27
28
|
clearSummary(id: string): Promise<boolean>;
|
|
28
29
|
insertObservations(observations: Observation[]): Promise<void>;
|
|
@@ -96,6 +96,9 @@ export class PostgresStore {
|
|
|
96
96
|
async getSession(id) {
|
|
97
97
|
return sessionsOps.getSession(this.pool, id);
|
|
98
98
|
}
|
|
99
|
+
async sessionProjects() {
|
|
100
|
+
return sessionsOps.sessionProjects(this.pool);
|
|
101
|
+
}
|
|
99
102
|
async recentSessions(project, limit) {
|
|
100
103
|
return sessionsOps.recentSessions(this.pool, project, limit);
|
|
101
104
|
}
|
|
@@ -103,7 +106,7 @@ export class PostgresStore {
|
|
|
103
106
|
return sessionsOps.clearSummary(this.pool, id);
|
|
104
107
|
}
|
|
105
108
|
async insertObservations(observations) {
|
|
106
|
-
return insertOps.insertObservations(this.pool, (dims) => this.ensureVectorColumn(dims), observations);
|
|
109
|
+
return insertOps.insertObservations(this.pool, (dims) => this.ensureVectorColumn(dims), observations, this.vectorDims !== null);
|
|
107
110
|
}
|
|
108
111
|
async getObservations(ids) {
|
|
109
112
|
return observationsOps.getObservations(this.pool, ids);
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { Pool } from './driver.js';
|
|
2
2
|
import type { Observation } from '../../types.js';
|
|
3
|
-
export declare function insertObservations(pool: Pool, ensureVectorColumn: (dims: number) => Promise<boolean>, observations: Observation[]): Promise<void>;
|
|
3
|
+
export declare function insertObservations(pool: Pool, ensureVectorColumn: (dims: number) => Promise<boolean>, observations: Observation[], hasVectorColumn?: boolean): Promise<void>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export async function insertObservations(pool, ensureVectorColumn, observations) {
|
|
1
|
+
export async function insertObservations(pool, ensureVectorColumn, observations, hasVectorColumn = false) {
|
|
2
2
|
if (observations.length === 0)
|
|
3
3
|
return;
|
|
4
4
|
const width = observations.find((obs) => obs.embedding?.length)?.embedding?.length;
|
|
@@ -44,6 +44,7 @@ export async function insertObservations(pool, ensureVectorColumn, observations)
|
|
|
44
44
|
const updates = columns
|
|
45
45
|
.slice(1)
|
|
46
46
|
.map((column) => `${column} = EXCLUDED.${column}`)
|
|
47
|
+
.concat(!vectorStored && (hasVectorColumn || vectors) ? ['embedding = NULL'] : [])
|
|
47
48
|
.join(', ');
|
|
48
49
|
await client.query(`INSERT INTO observations (${columns.join(', ')}) VALUES (${holes})
|
|
49
50
|
ON CONFLICT (id) DO UPDATE SET ${updates}`, values);
|
|
@@ -21,12 +21,24 @@ export async function ensureTagsIndexed(pool) {
|
|
|
21
21
|
JOIN pg_attribute a ON a.attrelid = d.adrelid AND a.attnum = d.adnum
|
|
22
22
|
WHERE d.adrelid = 'observations'::regclass AND a.attname = 'tsv'`);
|
|
23
23
|
const expr = String(res.rows[0]?.['expr'] ?? '');
|
|
24
|
-
if (expr.
|
|
24
|
+
if (expr.includes('tags'))
|
|
25
25
|
return;
|
|
26
|
-
await pool.
|
|
27
|
-
|
|
26
|
+
const client = await pool.connect();
|
|
27
|
+
try {
|
|
28
|
+
await client.query('BEGIN');
|
|
29
|
+
await client.query('ALTER TABLE observations DROP COLUMN IF EXISTS tsv');
|
|
30
|
+
await client.query(`ALTER TABLE observations
|
|
28
31
|
ADD COLUMN tsv TSVECTOR GENERATED ALWAYS AS (${TSV_EXPRESSION}) STORED`);
|
|
29
|
-
|
|
32
|
+
await client.query('CREATE INDEX IF NOT EXISTS idx_obs_tsv ON observations USING GIN(tsv)');
|
|
33
|
+
await client.query('COMMIT');
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
await client.query('ROLLBACK');
|
|
37
|
+
throw error;
|
|
38
|
+
}
|
|
39
|
+
finally {
|
|
40
|
+
client.release();
|
|
41
|
+
}
|
|
30
42
|
}
|
|
31
43
|
export async function readVectorDims(pool) {
|
|
32
44
|
const res = await pool.query(`SELECT atttypmod AS dims FROM pg_attribute
|
|
@@ -88,13 +88,20 @@ export async function list(pool, filter) {
|
|
|
88
88
|
conditions.push(`status = $${values.length}`);
|
|
89
89
|
}
|
|
90
90
|
if (filter.after !== undefined) {
|
|
91
|
-
|
|
92
|
-
|
|
91
|
+
if (filter.afterId === undefined) {
|
|
92
|
+
values.push(filter.after);
|
|
93
|
+
conditions.push(`created_at > $${values.length}`);
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
values.push(filter.after, filter.after, filter.afterId);
|
|
97
|
+
conditions.push(`(created_at > $${values.length - 2} OR (created_at = $${values.length - 1} AND id > $${values.length}))`);
|
|
98
|
+
}
|
|
93
99
|
}
|
|
94
100
|
values.push(filter.limit ?? 1000);
|
|
101
|
+
const direction = filter.newest ? 'DESC' : 'ASC';
|
|
95
102
|
const res = await pool.query(`SELECT * FROM observations
|
|
96
103
|
${conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''}
|
|
97
|
-
ORDER BY created_at
|
|
104
|
+
ORDER BY created_at ${direction}, id ${direction} LIMIT $${values.length}`, values);
|
|
98
105
|
return res.rows.map(toObservation);
|
|
99
106
|
}
|
|
100
107
|
export async function listProjects(pool) {
|
|
@@ -4,3 +4,4 @@ export declare function upsertSession(pool: Pool, session: Session): Promise<voi
|
|
|
4
4
|
export declare function getSession(pool: Pool, id: string): Promise<Session | null>;
|
|
5
5
|
export declare function clearSummary(pool: Pool, id: string): Promise<boolean>;
|
|
6
6
|
export declare function recentSessions(pool: Pool, project: string, limit: number): Promise<Session[]>;
|
|
7
|
+
export declare function sessionProjects(pool: Pool): Promise<string[]>;
|
|
@@ -28,3 +28,7 @@ export async function recentSessions(pool, project, limit) {
|
|
|
28
28
|
ORDER BY started_at DESC LIMIT $2`, [project, limit]);
|
|
29
29
|
return res.rows.map(toSession);
|
|
30
30
|
}
|
|
31
|
+
export async function sessionProjects(pool) {
|
|
32
|
+
const res = await pool.query('SELECT DISTINCT project FROM sessions');
|
|
33
|
+
return res.rows.map((row) => String(row['project'] ?? '')).filter((p) => p.length > 0);
|
|
34
|
+
}
|
|
@@ -57,10 +57,14 @@ export async function findEdges(db, filter) {
|
|
|
57
57
|
sides.push(`${column} IN (${ids.map(() => '?').join(',')})`);
|
|
58
58
|
params.push(...ids);
|
|
59
59
|
}
|
|
60
|
-
|
|
60
|
+
let where = sides.length > 0 ? ` AND (${sides.join(' OR ')})` : '';
|
|
61
|
+
if (filter.dstName !== undefined) {
|
|
62
|
+
where += ' AND (dst_name = ? OR dst_name LIKE ?)';
|
|
63
|
+
params.push(filter.dstName, `%.${filter.dstName}`);
|
|
64
|
+
}
|
|
61
65
|
params.push(filter.limit ?? 2000);
|
|
62
66
|
const rows = db
|
|
63
|
-
.prepare(`SELECT * FROM symbol_edges WHERE project = ?${where} LIMIT ?`)
|
|
67
|
+
.prepare(`SELECT * FROM symbol_edges WHERE project = ?${where} ORDER BY file, line, id LIMIT ?`)
|
|
64
68
|
.all(...params);
|
|
65
69
|
return rows.map(toEdge);
|
|
66
70
|
}
|
|
@@ -18,6 +18,7 @@ export declare class SqliteStore implements MemoryStore {
|
|
|
18
18
|
remove(filter: RemoveFilter): Promise<number>;
|
|
19
19
|
list(filter: ListFilter): Promise<Observation[]>;
|
|
20
20
|
listProjects(): Promise<ProjectSummary[]>;
|
|
21
|
+
sessionProjects(): Promise<string[]>;
|
|
21
22
|
inventory(): Promise<string[]>;
|
|
22
23
|
searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
23
24
|
searchVector(vector: number[], query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
@@ -74,6 +74,9 @@ export class SqliteStore {
|
|
|
74
74
|
async listProjects() {
|
|
75
75
|
return observationsOps.listProjects(this.db);
|
|
76
76
|
}
|
|
77
|
+
async sessionProjects() {
|
|
78
|
+
return sessionsOps.sessionProjects(this.db);
|
|
79
|
+
}
|
|
77
80
|
async inventory() {
|
|
78
81
|
return observationsOps.inventory(this.db);
|
|
79
82
|
}
|
|
@@ -87,14 +87,21 @@ export async function list(db, filter) {
|
|
|
87
87
|
params.push(filter.status);
|
|
88
88
|
}
|
|
89
89
|
if (filter.after !== undefined) {
|
|
90
|
-
|
|
91
|
-
|
|
90
|
+
if (filter.afterId === undefined) {
|
|
91
|
+
conditions.push('created_at > ?');
|
|
92
|
+
params.push(filter.after);
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
conditions.push('(created_at > ? OR (created_at = ? AND id > ?))');
|
|
96
|
+
params.push(filter.after, filter.after, filter.afterId);
|
|
97
|
+
}
|
|
92
98
|
}
|
|
93
99
|
params.push(filter.limit ?? 1000);
|
|
100
|
+
const direction = filter.newest ? 'DESC' : 'ASC';
|
|
94
101
|
const rows = db
|
|
95
102
|
.prepare(`SELECT * FROM observations
|
|
96
103
|
${conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''}
|
|
97
|
-
ORDER BY created_at
|
|
104
|
+
ORDER BY created_at ${direction}, id ${direction} LIMIT ?`)
|
|
98
105
|
.all(...params);
|
|
99
106
|
return rows.map(toObservation);
|
|
100
107
|
}
|
|
@@ -4,3 +4,4 @@ export declare function upsertSession(db: DatabaseSync, session: Session): Promi
|
|
|
4
4
|
export declare function clearSummary(db: DatabaseSync, id: string): Promise<boolean>;
|
|
5
5
|
export declare function getSession(db: DatabaseSync, id: string): Promise<Session | null>;
|
|
6
6
|
export declare function recentSessions(db: DatabaseSync, project: string, limit: number): Promise<Session[]>;
|
|
7
|
+
export declare function sessionProjects(db: DatabaseSync): Promise<string[]>;
|
|
@@ -25,3 +25,7 @@ export async function recentSessions(db, project, limit) {
|
|
|
25
25
|
.all(project, limit);
|
|
26
26
|
return rows.map(toSession);
|
|
27
27
|
}
|
|
28
|
+
export async function sessionProjects(db) {
|
|
29
|
+
const rows = db.prepare('SELECT DISTINCT project FROM sessions').all();
|
|
30
|
+
return rows.map((row) => String(row['project'] ?? '')).filter((project) => project.length > 0);
|
|
31
|
+
}
|
package/dist/types.d.ts
CHANGED
|
@@ -53,7 +53,9 @@ export interface ListFilter {
|
|
|
53
53
|
kind?: ObservationKind;
|
|
54
54
|
status?: ObservationStatus;
|
|
55
55
|
after?: number;
|
|
56
|
+
afterId?: string;
|
|
56
57
|
limit?: number;
|
|
58
|
+
newest?: boolean;
|
|
57
59
|
}
|
|
58
60
|
export interface TimelineQuery {
|
|
59
61
|
observationId: string;
|
|
@@ -96,6 +98,7 @@ export interface EdgeFilter {
|
|
|
96
98
|
project: string;
|
|
97
99
|
srcIds?: string[];
|
|
98
100
|
dstIds?: string[];
|
|
101
|
+
dstName?: string;
|
|
99
102
|
limit?: number;
|
|
100
103
|
}
|
|
101
104
|
export interface ScannedFile {
|
package/dist/usages/find.js
CHANGED
|
@@ -59,8 +59,7 @@ function buildArgs({ symbol, regex }) {
|
|
|
59
59
|
function withContext(root, matches, context) {
|
|
60
60
|
const fileLines = new Map();
|
|
61
61
|
const out = [];
|
|
62
|
-
|
|
63
|
-
let lastLineShown = 0;
|
|
62
|
+
const emitted = new Map();
|
|
64
63
|
for (const m of matches) {
|
|
65
64
|
let lines = fileLines.get(m.file);
|
|
66
65
|
if (lines === undefined) {
|
|
@@ -78,8 +77,14 @@ function withContext(root, matches, context) {
|
|
|
78
77
|
}
|
|
79
78
|
const from = Math.max(1, m.line - context);
|
|
80
79
|
const to = Math.min(lines.length, m.line + context);
|
|
81
|
-
|
|
82
|
-
|
|
80
|
+
for (let n = from; n <= to; n += 1) {
|
|
81
|
+
const key = `${m.file}\0${n}`;
|
|
82
|
+
const at = emitted.get(key);
|
|
83
|
+
if (at !== undefined) {
|
|
84
|
+
if (n === m.line)
|
|
85
|
+
out[at] = m;
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
83
88
|
out.push(n === m.line
|
|
84
89
|
? m
|
|
85
90
|
: {
|
|
@@ -89,9 +94,8 @@ function withContext(root, matches, context) {
|
|
|
89
94
|
isDefinition: false,
|
|
90
95
|
isMatch: false,
|
|
91
96
|
});
|
|
97
|
+
emitted.set(key, out.length - 1);
|
|
92
98
|
}
|
|
93
|
-
lastFile = m.file;
|
|
94
|
-
lastLineShown = to;
|
|
95
99
|
}
|
|
96
100
|
return out;
|
|
97
101
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-db",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Persistent memory for Claude Code. Captures why decisions were made, not just which files changed. Bring your own database: SQLite, MongoDB or Postgres. No cloud, no subscription.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|