claude-db 0.12.2 → 0.12.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/dist/capture/flush.js +41 -4
- package/dist/capture/index.d.ts +1 -0
- package/dist/capture/index.js +1 -0
- package/dist/capture/progress.js +8 -1
- package/dist/capture/vectors.d.ts +7 -0
- package/dist/capture/vectors.js +38 -0
- package/dist/catchup/gather.js +1 -0
- package/dist/cli/commands/adoption.js +41 -29
- package/dist/cli/commands/languages.js +10 -6
- package/dist/cli/commands/sync.js +5 -4
- package/dist/cli/commands/transfer.js +28 -8
- package/dist/facts/claude-memory.js +7 -2
- package/dist/facts/distill.js +4 -1
- package/dist/facts/handoff.js +6 -1
- package/dist/facts/health.js +2 -1
- package/dist/facts/known.js +2 -0
- package/dist/facts/start.js +1 -1
- package/dist/graph/query/answer.js +1 -1
- package/dist/graph/query/lookup.d.ts +1 -0
- package/dist/graph/query/lookup.js +2 -1
- package/dist/graph/query/path.js +69 -37
- package/dist/graph/query/suggest.js +1 -2
- package/dist/hooks/background.js +5 -1
- package/dist/hooks/compact.js +7 -1
- package/dist/hooks/detached.d.ts +1 -0
- package/dist/hooks/detached.js +37 -2
- package/dist/hooks/prefer-usages.js +1 -1
- package/dist/hooks/session-start.js +1 -1
- package/dist/hooks/start-legacy.js +1 -1
- package/dist/store/adapter.d.ts +5 -2
- package/dist/store/each.d.ts +2 -4
- package/dist/store/each.js +1 -0
- package/dist/store/mongo/graph.d.ts +2 -1
- package/dist/store/mongo/graph.js +17 -2
- package/dist/store/mongo/index.d.ts +4 -2
- package/dist/store/mongo/index.js +8 -2
- package/dist/store/mongo/observations.d.ts +2 -1
- package/dist/store/mongo/observations.js +25 -3
- package/dist/store/postgres/graph.d.ts +2 -1
- package/dist/store/postgres/graph.js +22 -7
- package/dist/store/postgres/index.d.ts +5 -2
- package/dist/store/postgres/index.js +14 -7
- package/dist/store/postgres/insert.js +3 -1
- package/dist/store/postgres/meta.d.ts +1 -0
- package/dist/store/postgres/meta.js +4 -0
- package/dist/store/postgres/observations.d.ts +3 -2
- package/dist/store/postgres/observations.js +40 -6
- package/dist/store/postgres/rows.js +1 -1
- package/dist/store/scoped-store.js +4 -0
- package/dist/store/sqlite/graph.d.ts +2 -1
- package/dist/store/sqlite/graph.js +19 -2
- package/dist/store/sqlite/index.d.ts +4 -2
- package/dist/store/sqlite/index.js +8 -2
- package/dist/store/sqlite/observations.d.ts +2 -1
- package/dist/store/sqlite/observations.js +32 -5
- package/dist/types.d.ts +9 -0
- package/package.json +6 -8
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,47 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.12.4
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
|
|
7
|
+
- **`find_usages` finds a path between two symbols anywhere in the code graph.** The search used to load only
|
|
8
|
+
the first 20,000 links of a project, so on a large repository it often said there was no path when there was
|
|
9
|
+
one. It now walks out from both ends and asks the database only for the links it needs.
|
|
10
|
+
- **Each request's memories are saved once.** The last request of a chat is read again on the next save, and it
|
|
11
|
+
was embedded and written again even when nothing in it had changed. It is now skipped.
|
|
12
|
+
|
|
13
|
+
### Changed
|
|
14
|
+
|
|
15
|
+
- **claude-db downloads far less from a remote database.** Memories are read without their vectors unless a copy
|
|
16
|
+
needs them, and without their text where only titles are shown. Counting facts, finding imported Claude
|
|
17
|
+
memories and finding rows that need a vector now happen in the database instead of reading every row. This
|
|
18
|
+
matters most on a hosted Postgres that bills for data sent out, such as Neon.
|
|
19
|
+
- **`graphology` and `graphology-shortest-path` are no longer dependencies.**
|
|
20
|
+
|
|
21
|
+
By @alok-debnath (#36).
|
|
22
|
+
|
|
23
|
+
## 0.12.3
|
|
24
|
+
|
|
25
|
+
### Fixed
|
|
26
|
+
|
|
27
|
+
- **`distill` no longer stops on Postgres.** A Claude memory file with no `modified` time made it fail with
|
|
28
|
+
`invalid input syntax for type bigint`, so chats waited for days. Times are now whole milliseconds.
|
|
29
|
+
- **Rows saved without a vector get one later.** A row is saved as text only when the embedding model is still
|
|
30
|
+
loading, or when its width differs from the Postgres column. The next chat start fills such rows in the
|
|
31
|
+
background, at most every 6 hours, once an embedder of the right width loads. It never replaces a vector from
|
|
32
|
+
another embedder.
|
|
33
|
+
- **Background jobs keep their output.** It goes to `~/.claude-memory/logs/background.log`, which rolls over at
|
|
34
|
+
1 MB.
|
|
35
|
+
- **The waiting-chats warning names `claude-db distill --backfill`**, which works through them.
|
|
36
|
+
- **`claude-db languages` lists TypeScript, TSX and JavaScript** as built in.
|
|
37
|
+
|
|
38
|
+
### Changed
|
|
39
|
+
|
|
40
|
+
- **A grep for a symbol that is not in the graph is faster.** The hook no longer runs a "did you mean" scan it
|
|
41
|
+
never shows.
|
|
42
|
+
- **`claude-db adoption` is faster.** It looks each symbol up once, several at a time, and shows a counter on a
|
|
43
|
+
terminal.
|
|
44
|
+
|
|
3
45
|
## 0.12.2
|
|
4
46
|
|
|
5
47
|
### Fixed
|
package/dist/capture/flush.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
1
2
|
import { mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
2
3
|
import { dirname, join } from 'node:path';
|
|
3
4
|
import { CONFIG_DIR } from '../config/index.js';
|
|
4
5
|
import { readTranscript, sessionIdsOnDisk, transcriptPathFor } from './transcript.js';
|
|
5
6
|
import { observationsFromTurns } from './turn-extractor.js';
|
|
6
7
|
import { closeLandedWork } from './progress.js';
|
|
8
|
+
import { noteMissingVectors } from './vectors.js';
|
|
7
9
|
export async function flushSession(ctx, sessionId, project, transcriptPath, rebuild = false, finalReply = '') {
|
|
8
10
|
const path = transcriptPath ?? transcriptPathFor(project, sessionId);
|
|
9
11
|
const cursor = readCursor(sessionId);
|
|
@@ -22,8 +24,12 @@ export async function flushSession(ctx, sessionId, project, transcriptPath, rebu
|
|
|
22
24
|
writeCursor(sessionId, nextOffset);
|
|
23
25
|
return { observations: 0, summary: null, lastReply };
|
|
24
26
|
}
|
|
25
|
-
|
|
26
|
-
|
|
27
|
+
const written = rebuild ? new Map() : readWritten(sessionId);
|
|
28
|
+
const prints = new Map(observations.map((obs) => [obs.id, printOf(obs)]));
|
|
29
|
+
const fresh = observations.filter((obs) => written.get(obs.id) !== prints.get(obs.id));
|
|
30
|
+
await embedObservations(ctx, fresh);
|
|
31
|
+
await ctx.store.insertObservations(fresh);
|
|
32
|
+
writeWritten(sessionId, prints);
|
|
27
33
|
await closeLandedWork(ctx.store, project);
|
|
28
34
|
const previous = rebuild ? null : await ctx.store.getSession(sessionId);
|
|
29
35
|
const summary = summarize(observations, previous?.summary);
|
|
@@ -37,10 +43,15 @@ export async function flushSession(ctx, sessionId, project, transcriptPath, rebu
|
|
|
37
43
|
return { observations: observations.length, summary, lastReply };
|
|
38
44
|
}
|
|
39
45
|
export async function embedObservations(ctx, observations) {
|
|
46
|
+
if (observations.length === 0)
|
|
47
|
+
return;
|
|
40
48
|
try {
|
|
41
49
|
const embedder = await ctx.embedder();
|
|
42
|
-
if (embedder.dimensions === 0)
|
|
50
|
+
if (embedder.dimensions === 0) {
|
|
51
|
+
if (ctx.config.embeddings.provider !== 'none')
|
|
52
|
+
noteMissingVectors(observations);
|
|
43
53
|
return;
|
|
54
|
+
}
|
|
44
55
|
const size = ctx.config.embeddings.batchSize;
|
|
45
56
|
for (let start = 0; start < observations.length; start += size) {
|
|
46
57
|
const chunk = observations.slice(start, start + size);
|
|
@@ -53,8 +64,11 @@ export async function embedObservations(ctx, observations) {
|
|
|
53
64
|
}
|
|
54
65
|
});
|
|
55
66
|
}
|
|
67
|
+
const stored = ctx.store.storedVectorDims?.() ?? null;
|
|
68
|
+
noteMissingVectors(observations, stored !== null && stored !== embedder.dimensions);
|
|
56
69
|
}
|
|
57
70
|
catch (error) {
|
|
71
|
+
noteMissingVectors(observations);
|
|
58
72
|
process.stderr.write(`[claude-db] embedding skipped: ` +
|
|
59
73
|
`${error instanceof Error ? error.message : String(error)}\n`);
|
|
60
74
|
}
|
|
@@ -91,6 +105,27 @@ function cursorName(sessionId) {
|
|
|
91
105
|
function cursorPath(sessionId) {
|
|
92
106
|
return join(CONFIG_DIR, 'cursors', cursorName(sessionId));
|
|
93
107
|
}
|
|
108
|
+
const writtenPath = (sessionId) => cursorPath(sessionId).replace(/\.offset$/, '.written');
|
|
109
|
+
const printOf = (obs) => createHash('sha256').update(JSON.stringify(obs)).digest('base64url');
|
|
110
|
+
function readWritten(sessionId) {
|
|
111
|
+
try {
|
|
112
|
+
const parsed = JSON.parse(readFileSync(writtenPath(sessionId), 'utf8'));
|
|
113
|
+
if (!parsed || typeof parsed !== 'object')
|
|
114
|
+
return new Map();
|
|
115
|
+
return new Map(Object.entries(parsed).filter((entry) => typeof entry[1] === 'string'));
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return new Map();
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
function writeWritten(sessionId, prints) {
|
|
122
|
+
const path = writtenPath(sessionId);
|
|
123
|
+
try {
|
|
124
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
125
|
+
writeFileSync(path, JSON.stringify(Object.fromEntries(prints)), 'utf8');
|
|
126
|
+
}
|
|
127
|
+
catch { }
|
|
128
|
+
}
|
|
94
129
|
function readCursor(sessionId) {
|
|
95
130
|
try {
|
|
96
131
|
const value = Number.parseInt(readFileSync(cursorPath(sessionId), 'utf8'), 10);
|
|
@@ -110,10 +145,12 @@ function writeCursor(sessionId, offset) {
|
|
|
110
145
|
}
|
|
111
146
|
export function resetCursor(sessionId) {
|
|
112
147
|
writeCursor(sessionId, 0);
|
|
148
|
+
rmSync(writtenPath(sessionId), { force: true });
|
|
113
149
|
}
|
|
114
150
|
export function clearCursor(sessionId) {
|
|
115
151
|
rmSync(cursorPath(sessionId), { force: true });
|
|
116
152
|
rmSync(cursorPath(sessionId).replace(/\.offset$/, '.shown'), { force: true });
|
|
153
|
+
rmSync(writtenPath(sessionId), { force: true });
|
|
117
154
|
}
|
|
118
155
|
export function sweepCursors() {
|
|
119
156
|
const dir = join(CONFIG_DIR, 'cursors');
|
|
@@ -127,7 +164,7 @@ export function sweepCursors() {
|
|
|
127
164
|
const live = new Set(sessionIdsOnDisk().map((id) => id.replace(/[^\w-]/g, '_')));
|
|
128
165
|
let removed = 0;
|
|
129
166
|
for (const file of files) {
|
|
130
|
-
const match = /^(.+)\.(offset|shown)$/.exec(file);
|
|
167
|
+
const match = /^(.+)\.(offset|shown|written)$/.exec(file);
|
|
131
168
|
if (!match?.[1] || live.has(match[1]))
|
|
132
169
|
continue;
|
|
133
170
|
rmSync(join(dir, file), { force: true });
|
package/dist/capture/index.d.ts
CHANGED
|
@@ -10,6 +10,7 @@ export { redact } from './redact.js';
|
|
|
10
10
|
export { claimScrub, finishScrub, releaseScrub, scrubSecrets } from './scrub.js';
|
|
11
11
|
export { flushSession, resetCursor, clearCursor, sweepCursors, summarize, embedObservations, } from './flush.js';
|
|
12
12
|
export type { FlushResult } from './flush.js';
|
|
13
|
+
export { claimReembed, finishReembed, releaseReembed, startReembed, vectorsMissing, } from './vectors.js';
|
|
13
14
|
export { closeLandedWork, openWork } from './progress.js';
|
|
14
15
|
export { REINGEST_VERSION, claimReingest, finishReingest, reingestDone, reingestProject, rememberedTranscripts, reingestSession, releaseReingest, } from './reingest.js';
|
|
15
16
|
export type { ProjectReingest, SessionReingest } from './reingest.js';
|
package/dist/capture/index.js
CHANGED
|
@@ -7,6 +7,7 @@ export { observationsFromTurns } from './turn-extractor.js';
|
|
|
7
7
|
export { redact } from './redact.js';
|
|
8
8
|
export { claimScrub, finishScrub, releaseScrub, scrubSecrets } from './scrub.js';
|
|
9
9
|
export { flushSession, resetCursor, clearCursor, sweepCursors, summarize, embedObservations, } from './flush.js';
|
|
10
|
+
export { claimReembed, finishReembed, releaseReembed, startReembed, vectorsMissing, } from './vectors.js';
|
|
10
11
|
export { closeLandedWork, openWork } from './progress.js';
|
|
11
12
|
export { REINGEST_VERSION, claimReingest, finishReingest, reingestDone, reingestProject, rememberedTranscripts, reingestSession, releaseReingest, } from './reingest.js';
|
|
12
13
|
export { aiSummary, buildSummaryPrompt, validateSummary } from './summarize.js';
|
package/dist/capture/progress.js
CHANGED
|
@@ -43,6 +43,7 @@ export async function closeLandedWork(store, project) {
|
|
|
43
43
|
...(after === undefined ? {} : { after }),
|
|
44
44
|
...(afterId === undefined ? {} : { afterId }),
|
|
45
45
|
limit: OPEN_LIMIT,
|
|
46
|
+
bodies: false,
|
|
46
47
|
});
|
|
47
48
|
if (batch.length === 0)
|
|
48
49
|
break;
|
|
@@ -61,6 +62,12 @@ export async function closeLandedWork(store, project) {
|
|
|
61
62
|
return store.closeObservations(landed);
|
|
62
63
|
}
|
|
63
64
|
export async function openWork(store, project) {
|
|
64
|
-
const open = await store.list({
|
|
65
|
+
const open = await store.list({
|
|
66
|
+
project,
|
|
67
|
+
status: 'open',
|
|
68
|
+
limit: OPEN_LIMIT,
|
|
69
|
+
newest: true,
|
|
70
|
+
bodies: false,
|
|
71
|
+
});
|
|
65
72
|
return open.sort((a, b) => b.createdAt - a.createdAt);
|
|
66
73
|
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { Observation } from '../types.js';
|
|
2
|
+
export declare function noteMissingVectors(observations: Observation[], everyRow?: boolean): void;
|
|
3
|
+
export declare const vectorsMissing: (project: string) => boolean;
|
|
4
|
+
export declare function claimReembed(project: string, now?: number): boolean;
|
|
5
|
+
export declare function startReembed(project: string): void;
|
|
6
|
+
export declare function finishReembed(project: string, now?: number): void;
|
|
7
|
+
export declare function releaseReembed(project: string): void;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { CONFIG_DIR } from '../config/index.js';
|
|
4
|
+
import { claimJob, finishJob, jobMark, releaseJob } from '../util/job-lock.js';
|
|
5
|
+
import { scopeToken } from '../util/scope.js';
|
|
6
|
+
const JOB = 'reembed';
|
|
7
|
+
const FLAGS = 'vectors-missing';
|
|
8
|
+
const REEMBED_EVERY_MS = 6 * 60 * 60 * 1000;
|
|
9
|
+
const flagPath = (project) => join(CONFIG_DIR, FLAGS, `${scopeToken(project)}.flag`);
|
|
10
|
+
export function noteMissingVectors(observations, everyRow = false) {
|
|
11
|
+
const projects = new Set(observations.filter((obs) => everyRow || !obs.embedding?.length).map((obs) => obs.project));
|
|
12
|
+
for (const project of projects) {
|
|
13
|
+
try {
|
|
14
|
+
mkdirSync(join(CONFIG_DIR, FLAGS), { recursive: true });
|
|
15
|
+
writeFileSync(flagPath(project), '');
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
continue;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
export const vectorsMissing = (project) => existsSync(flagPath(project));
|
|
23
|
+
export function claimReembed(project, now = Date.now()) {
|
|
24
|
+
if (!vectorsMissing(project))
|
|
25
|
+
return false;
|
|
26
|
+
if (now - jobMark(JOB, project) < REEMBED_EVERY_MS)
|
|
27
|
+
return false;
|
|
28
|
+
return claimJob(JOB, project, now);
|
|
29
|
+
}
|
|
30
|
+
export function startReembed(project) {
|
|
31
|
+
rmSync(flagPath(project), { force: true });
|
|
32
|
+
}
|
|
33
|
+
export function finishReembed(project, now = Date.now()) {
|
|
34
|
+
finishJob(JOB, project, now);
|
|
35
|
+
}
|
|
36
|
+
export function releaseReembed(project) {
|
|
37
|
+
releaseJob(JOB, project);
|
|
38
|
+
}
|
package/dist/catchup/gather.js
CHANGED
|
@@ -18,6 +18,7 @@ export async function gatherCatchup(ctx, project, current, now = Date.now()) {
|
|
|
18
18
|
sessionId: FACT_SESSION,
|
|
19
19
|
newest: true,
|
|
20
20
|
limit: FACTS_SCANNED,
|
|
21
|
+
bodies: false,
|
|
21
22
|
});
|
|
22
23
|
const kept = onePerKey(facts.filter((obs) => obs.status !== 'replaced'));
|
|
23
24
|
const todos = kept
|
|
@@ -4,6 +4,8 @@ import { queryGraph } from '../../graph/index.js';
|
|
|
4
4
|
import { transcriptsFor } from '../../capture/transcript.js';
|
|
5
5
|
import { resolveProject } from '../../util/project.js';
|
|
6
6
|
import { DECLARED, isSymbol, symbolsGreppedIn } from '../../hooks/grep-symbols.js';
|
|
7
|
+
const MAX_SYMBOLS = 2;
|
|
8
|
+
const LOOKUPS_IN_FLIGHT = 8;
|
|
7
9
|
export async function cmdAdoption() {
|
|
8
10
|
const project = resolveProject(undefined);
|
|
9
11
|
const files = transcriptsFor(project);
|
|
@@ -45,35 +47,8 @@ export async function cmdAdoption() {
|
|
|
45
47
|
}
|
|
46
48
|
const greps = bash.filter((command) => /\b(?:grep|rg)\b/.test(command));
|
|
47
49
|
const candidates = greps.map((command) => symbolsGreppedIn(command)).filter((s) => s.length > 0);
|
|
48
|
-
const
|
|
49
|
-
|
|
50
|
-
const cache = new Map();
|
|
51
|
-
try {
|
|
52
|
-
for (const symbols of candidates) {
|
|
53
|
-
let hit = false;
|
|
54
|
-
for (const symbol of symbols.slice(0, 2)) {
|
|
55
|
-
let fired = cache.get(symbol);
|
|
56
|
-
if (fired === undefined) {
|
|
57
|
-
const answer = await queryGraph(ctx.store, project, {
|
|
58
|
-
mode: 'usages',
|
|
59
|
-
symbol,
|
|
60
|
-
limit: 20,
|
|
61
|
-
});
|
|
62
|
-
fired =
|
|
63
|
-
!answer.empty &&
|
|
64
|
-
(isSymbol(symbol) || answer.definitions.some((d) => DECLARED.has(d.kind)));
|
|
65
|
-
cache.set(symbol, fired);
|
|
66
|
-
}
|
|
67
|
-
if (fired)
|
|
68
|
-
hit = true;
|
|
69
|
-
}
|
|
70
|
-
if (hit)
|
|
71
|
-
fires++;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
finally {
|
|
75
|
-
await ctx.close();
|
|
76
|
-
}
|
|
50
|
+
const firing = await symbolsThatFire(project, candidates.flatMap((symbols) => symbols.slice(0, MAX_SYMBOLS)));
|
|
51
|
+
const fires = candidates.filter((symbols) => symbols.slice(0, MAX_SYMBOLS).some((symbol) => firing.has(symbol))).length;
|
|
77
52
|
const memoryTotal = [...memory.values()].reduce((a, b) => a + b, 0);
|
|
78
53
|
const pct = (n, of) => (of === 0 ? '0.0%' : `${((n / of) * 100).toFixed(1)}%`);
|
|
79
54
|
console.log(`Adoption for ${project} (${files.length} transcripts)\n`);
|
|
@@ -92,3 +67,40 @@ export async function cmdAdoption() {
|
|
|
92
67
|
console.log(`\n ${ratio}`);
|
|
93
68
|
}
|
|
94
69
|
}
|
|
70
|
+
async function symbolsThatFire(project, symbols) {
|
|
71
|
+
const unique = [...new Set(symbols)];
|
|
72
|
+
const firing = new Set();
|
|
73
|
+
if (unique.length === 0)
|
|
74
|
+
return firing;
|
|
75
|
+
const ctx = await createContext();
|
|
76
|
+
const tty = process.stderr.isTTY === true;
|
|
77
|
+
let next = 0;
|
|
78
|
+
let done = 0;
|
|
79
|
+
const lookup = async () => {
|
|
80
|
+
while (next < unique.length) {
|
|
81
|
+
const symbol = unique[next++];
|
|
82
|
+
const answer = await queryGraph(ctx.store, project, {
|
|
83
|
+
mode: 'usages',
|
|
84
|
+
symbol,
|
|
85
|
+
limit: 20,
|
|
86
|
+
suggest: false,
|
|
87
|
+
});
|
|
88
|
+
if (!answer.empty &&
|
|
89
|
+
(isSymbol(symbol) || answer.definitions.some((d) => DECLARED.has(d.kind)))) {
|
|
90
|
+
firing.add(symbol);
|
|
91
|
+
}
|
|
92
|
+
done += 1;
|
|
93
|
+
if (tty)
|
|
94
|
+
process.stderr.write(`\rchecking symbols ${done}/${unique.length}`);
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
try {
|
|
98
|
+
await Promise.all(Array.from({ length: Math.min(LOOKUPS_IN_FLIGHT, unique.length) }, () => lookup()));
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
if (tty)
|
|
102
|
+
process.stderr.write('\r\x1b[K');
|
|
103
|
+
await ctx.close();
|
|
104
|
+
}
|
|
105
|
+
return firing;
|
|
106
|
+
}
|
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import { GRAMMARS, PLATFORM, hasGrammar } from '../../graph/grammars.js';
|
|
2
|
-
import { languageNames } from '../../graph/index.js';
|
|
2
|
+
import { LANGUAGES, languageNames } from '../../graph/index.js';
|
|
3
3
|
const SHIPPED = 'Grammars ship with claude-db for each platform, so there is nothing to add or remove.';
|
|
4
|
-
const stateOf = (name) =>
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
const stateOf = (name) => {
|
|
5
|
+
if (!GRAMMARS.includes(name))
|
|
6
|
+
return 'read with real syntax, built in';
|
|
7
|
+
return hasGrammar(name)
|
|
8
|
+
? 'read with real syntax'
|
|
9
|
+
: `read by pattern: no grammar for ${PLATFORM} in this install`;
|
|
10
|
+
};
|
|
7
11
|
function listLanguages() {
|
|
8
12
|
console.log(`read with real syntax: ${languageNames()}`);
|
|
9
|
-
for (const
|
|
10
|
-
console.log(` ${
|
|
13
|
+
for (const { id, label } of LANGUAGES)
|
|
14
|
+
console.log(` ${label.padEnd(10)} ${stateOf(id)}`);
|
|
11
15
|
if (GRAMMARS.some((name) => !hasGrammar(name))) {
|
|
12
16
|
console.log('Reinstall claude-db to add the grammars for this platform.');
|
|
13
17
|
}
|
|
@@ -18,13 +18,13 @@ export async function cmdSync(argv) {
|
|
|
18
18
|
try {
|
|
19
19
|
await remote.init();
|
|
20
20
|
const localStatus = new Map();
|
|
21
|
-
await eachRemoteObservation(local.store, (batch) => {
|
|
21
|
+
await eachRemoteObservation(local.store, false, (batch) => {
|
|
22
22
|
for (const obs of batch)
|
|
23
23
|
localStatus.set(obs.id, obs.status ?? 'done');
|
|
24
24
|
});
|
|
25
25
|
const remoteStatus = new Map();
|
|
26
26
|
let pulled = 0;
|
|
27
|
-
await eachRemoteObservation(remote, async (batch) => {
|
|
27
|
+
await eachRemoteObservation(remote, true, async (batch) => {
|
|
28
28
|
for (const obs of batch)
|
|
29
29
|
remoteStatus.set(obs.id, obs.status ?? 'done');
|
|
30
30
|
const fresh = batch.filter((obs) => !localStatus.has(obs.id));
|
|
@@ -35,7 +35,7 @@ export async function cmdSync(argv) {
|
|
|
35
35
|
await local.store.insertObservations(fresh);
|
|
36
36
|
});
|
|
37
37
|
let pushed = 0;
|
|
38
|
-
await eachRemoteObservation(local.store, async (batch) => {
|
|
38
|
+
await eachRemoteObservation(local.store, true, async (batch) => {
|
|
39
39
|
const fresh = batch.filter((obs) => !remoteStatus.has(obs.id));
|
|
40
40
|
if (fresh.length === 0)
|
|
41
41
|
return;
|
|
@@ -139,7 +139,7 @@ async function reconcileSession(session, from, to) {
|
|
|
139
139
|
}
|
|
140
140
|
return false;
|
|
141
141
|
}
|
|
142
|
-
async function eachRemoteObservation(store, visit) {
|
|
142
|
+
async function eachRemoteObservation(store, vectors, visit) {
|
|
143
143
|
let after;
|
|
144
144
|
let afterId;
|
|
145
145
|
for (;;) {
|
|
@@ -147,6 +147,7 @@ async function eachRemoteObservation(store, visit) {
|
|
|
147
147
|
...(after === undefined ? {} : { after }),
|
|
148
148
|
...(afterId === undefined ? {} : { afterId }),
|
|
149
149
|
limit: BATCH,
|
|
150
|
+
vectors,
|
|
150
151
|
});
|
|
151
152
|
if (batch.length === 0)
|
|
152
153
|
return;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { BATCH } from '../constants.js';
|
|
2
2
|
const SESSION_LIMIT = 10000;
|
|
3
3
|
import { createContext } from '../../context.js';
|
|
4
|
-
import { embedObservations } from '../../capture/index.js';
|
|
4
|
+
import { embedObservations, finishReembed, releaseReembed, startReembed, } from '../../capture/index.js';
|
|
5
5
|
import { loadConfig } from '../../config/index.js';
|
|
6
6
|
import { readFileSync } from 'node:fs';
|
|
7
7
|
import { resolveProject } from '../../util/project.js';
|
|
@@ -127,39 +127,59 @@ export async function cmdPrune(argv) {
|
|
|
127
127
|
}
|
|
128
128
|
export async function cmdReembed(argv = []) {
|
|
129
129
|
const scoped = argv.includes('--project') || argv.includes('-p');
|
|
130
|
+
const background = argv.includes('--background');
|
|
131
|
+
const project = resolveProject(undefined);
|
|
130
132
|
const base = loadConfig();
|
|
131
133
|
const ctx = await createContext({ embeddings: { ...base.embeddings, timeoutMs: 0 } });
|
|
134
|
+
if (background)
|
|
135
|
+
startReembed(project);
|
|
136
|
+
let finished = false;
|
|
132
137
|
try {
|
|
133
138
|
const embedder = await ctx.embedder();
|
|
134
139
|
if (embedder.dimensions === 0) {
|
|
135
140
|
console.error('No embedder available; nothing to do.');
|
|
136
141
|
process.exit(1);
|
|
137
142
|
}
|
|
143
|
+
const stored = ctx.store.storedVectorDims?.() ?? null;
|
|
144
|
+
if (background && stored !== null && stored !== embedder.dimensions) {
|
|
145
|
+
finished = true;
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
138
148
|
let migrated = false;
|
|
139
149
|
if (!scoped && ctx.store.migrateVectorDims) {
|
|
140
150
|
migrated = await ctx.store.migrateVectorDims(embedder.dimensions);
|
|
141
151
|
}
|
|
142
152
|
let updated = 0;
|
|
143
|
-
|
|
144
|
-
const
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
153
|
+
const filter = scoped ? { project } : {};
|
|
154
|
+
const scanned = await ctx.store.count(filter);
|
|
155
|
+
const vectorStale = background ? {} : { embedder: embedder.id };
|
|
156
|
+
await eachObservation(ctx, { ...filter, vectorStale }, async (batch) => {
|
|
157
|
+
const stale = batch.filter((obs) => !obs.embedding?.length || (!background && obs.embedder !== embedder.id));
|
|
148
158
|
if (stale.length === 0)
|
|
149
159
|
return;
|
|
150
160
|
await embedObservations(ctx, stale);
|
|
151
161
|
await ctx.store.insertObservations(stale);
|
|
152
162
|
updated += stale.length;
|
|
153
|
-
|
|
163
|
+
if (!background)
|
|
164
|
+
process.stderr.write(`\r${updated} re-embedded...`);
|
|
154
165
|
});
|
|
155
|
-
|
|
166
|
+
if (!background)
|
|
167
|
+
process.stderr.write('\r');
|
|
168
|
+
const skipped = scanned - updated;
|
|
156
169
|
if (migrated) {
|
|
157
170
|
console.log(`Rebuilt vector storage at ${embedder.dimensions}d.`);
|
|
158
171
|
}
|
|
159
172
|
console.log(`Scanned ${scanned}, re-embedded ${updated} with ${embedder.id}` +
|
|
160
173
|
`${skipped > 0 ? `, ${skipped} already current` : ''}.`);
|
|
174
|
+
finished = true;
|
|
161
175
|
}
|
|
162
176
|
finally {
|
|
177
|
+
if (background) {
|
|
178
|
+
if (finished)
|
|
179
|
+
finishReembed(project);
|
|
180
|
+
else
|
|
181
|
+
releaseReembed(project);
|
|
182
|
+
}
|
|
163
183
|
await ctx.close();
|
|
164
184
|
}
|
|
165
185
|
}
|
|
@@ -76,7 +76,7 @@ function memoryFiles(dir) {
|
|
|
76
76
|
for (const name of names) {
|
|
77
77
|
try {
|
|
78
78
|
const path = join(dir, name);
|
|
79
|
-
const parsed = parseMemoryFile(readFileSync(path, 'utf8'), statSync(path).mtimeMs);
|
|
79
|
+
const parsed = parseMemoryFile(readFileSync(path, 'utf8'), Math.trunc(statSync(path).mtimeMs));
|
|
80
80
|
if (parsed)
|
|
81
81
|
files.push(parsed);
|
|
82
82
|
}
|
|
@@ -88,7 +88,12 @@ async function importedFacts(ctx, project) {
|
|
|
88
88
|
const mine = [machineTag(), originTag(project)];
|
|
89
89
|
const rows = [];
|
|
90
90
|
for (const owner of [project, youScope()]) {
|
|
91
|
-
const batch = await ctx.store.list({
|
|
91
|
+
const batch = await ctx.store.list({
|
|
92
|
+
project: owner,
|
|
93
|
+
sessionId: FACT_SESSION,
|
|
94
|
+
tag: CLAUDE_MEMORY_TAG,
|
|
95
|
+
limit: 5000,
|
|
96
|
+
});
|
|
92
97
|
rows.push(...batch.filter((obs) => obs.tags.includes(CLAUDE_MEMORY_TAG) && mine.every((tag) => obs.tags.includes(tag))));
|
|
93
98
|
}
|
|
94
99
|
return rows;
|
package/dist/facts/distill.js
CHANGED
|
@@ -128,6 +128,9 @@ function clip(text, max) {
|
|
|
128
128
|
return text.length <= max ? text : `${text.slice(0, max)}…`;
|
|
129
129
|
}
|
|
130
130
|
export async function factCounts(ctx, project) {
|
|
131
|
-
const count = async (owner) =>
|
|
131
|
+
const count = async (owner) => {
|
|
132
|
+
const scope = { project: owner, sessionId: FACT_SESSION };
|
|
133
|
+
return ((await ctx.store.count(scope)) - (await ctx.store.count({ ...scope, status: 'replaced' })));
|
|
134
|
+
};
|
|
132
135
|
return { project: await count(project), you: await count(youScope()) };
|
|
133
136
|
}
|
package/dist/facts/handoff.js
CHANGED
|
@@ -19,11 +19,16 @@ export async function newestHandoff(ctx, project, now = Date.now()) {
|
|
|
19
19
|
sessionId: MANUAL_SESSION,
|
|
20
20
|
newest: true,
|
|
21
21
|
limit: MANUAL_NOTES_SCANNED,
|
|
22
|
+
bodies: false,
|
|
22
23
|
});
|
|
23
24
|
const current = (obs) => obs.status !== 'replaced' &&
|
|
24
25
|
obs.tags.includes(HANDOFF_TAG) &&
|
|
25
26
|
now - obs.createdAt <= HANDOFF_MAX_AGE_MS;
|
|
26
|
-
|
|
27
|
+
const found = notes.find(current);
|
|
28
|
+
if (!found)
|
|
29
|
+
return null;
|
|
30
|
+
const [full] = await ctx.store.getObservations([found.id]);
|
|
31
|
+
return full ?? null;
|
|
27
32
|
}
|
|
28
33
|
function clipLine(line) {
|
|
29
34
|
return line.length > HANDOFF_LINE_CHARS
|
package/dist/facts/health.js
CHANGED
|
@@ -6,5 +6,6 @@ export function waitingWarning(waiting, oldestAt, usage, now = Date.now()) {
|
|
|
6
6
|
return null;
|
|
7
7
|
if (usage.used > 0 || usage.pausedUntil > now)
|
|
8
8
|
return null;
|
|
9
|
-
return `warning : ${waiting} chat(s) have waited over a day and no facts were tried today.
|
|
9
|
+
return (`warning : ${waiting} chat(s) have waited over a day and no facts were tried today. ` +
|
|
10
|
+
'Run claude-db distill --backfill to work through them now, or claude-db doctor to check Haiku');
|
|
10
11
|
}
|
package/dist/facts/known.js
CHANGED
|
@@ -14,6 +14,7 @@ export async function existingFacts(ctx, project) {
|
|
|
14
14
|
sessionId: FACT_SESSION,
|
|
15
15
|
newest: true,
|
|
16
16
|
limit: EXISTING_LIMIT,
|
|
17
|
+
bodies: false,
|
|
17
18
|
});
|
|
18
19
|
for (const obs of onePerKey(rows.filter((row) => row.status !== 'replaced'))) {
|
|
19
20
|
const key = factKey(obs);
|
|
@@ -31,6 +32,7 @@ export async function notedByUser(ctx, project) {
|
|
|
31
32
|
sessionId: MANUAL_SESSION,
|
|
32
33
|
newest: true,
|
|
33
34
|
limit: EXISTING_LIMIT,
|
|
35
|
+
bodies: false,
|
|
34
36
|
});
|
|
35
37
|
return rows
|
|
36
38
|
.filter((row) => row.status !== 'replaced' && !row.tags.includes(HANDOFF_TAG))
|
package/dist/facts/start.js
CHANGED
|
@@ -13,7 +13,7 @@ const UNCOMMITTED = 3;
|
|
|
13
13
|
const ORDER = ['rule', 'decision', 'deadend', 'fact'];
|
|
14
14
|
export async function startFacts(ctx, project, hidden = new Set(), current) {
|
|
15
15
|
const visible = (obs) => obs.status !== 'replaced' && !hidden.has(obs.id) && !seenByClaude(obs, project);
|
|
16
|
-
const list = async (owner, sessionId, limit) => (await ctx.store.list({ project: owner, sessionId, newest: true, limit })).filter(visible);
|
|
16
|
+
const list = async (owner, sessionId, limit) => (await ctx.store.list({ project: owner, sessionId, newest: true, limit, bodies: false })).filter(visible);
|
|
17
17
|
const you = onePerKey(await list(youScope(), FACT_SESSION, 50));
|
|
18
18
|
const facts = onePerKey(await list(project, FACT_SESSION, 300));
|
|
19
19
|
const handoff = await newestHandoff(ctx, project);
|
|
@@ -10,7 +10,7 @@ export function addText(answer, root) {
|
|
|
10
10
|
return;
|
|
11
11
|
answer.text = unlinkedText(root, answer.symbol, answer.definitions, answer.inbound);
|
|
12
12
|
}
|
|
13
|
-
export const hasGraph = async (store, project) => (await store.scannedFiles(project)).length > 0;
|
|
13
|
+
export const hasGraph = async (store, project) => (await store.scannedFiles(project, 1)).length > 0;
|
|
14
14
|
export async function answerQuery(request) {
|
|
15
15
|
const { store, root, project, query } = request;
|
|
16
16
|
const refreshed = request.refresh ? await refreshGraph(store, root, project) : [];
|
|
@@ -56,7 +56,8 @@ export async function queryGraph(store, project, query) {
|
|
|
56
56
|
answer.inbound = [...direct, ...(await throughAliases(store, project, direct, edgeLimit))];
|
|
57
57
|
answer.outbound = edges.filter((edge) => idSet.has(edge.srcId) && !pointsAtSymbol(edge));
|
|
58
58
|
answer.empty = answer.definitions.length === 0 && answer.inbound.length === 0;
|
|
59
|
-
if (answer.empty)
|
|
59
|
+
if (answer.empty && query.suggest !== false) {
|
|
60
60
|
answer.suggestions = await suggestFor(store, project, query.symbol);
|
|
61
|
+
}
|
|
61
62
|
return answer;
|
|
62
63
|
}
|