claude-db 0.12.4 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/dist/capture/cursors.d.ts +8 -0
- package/dist/capture/cursors.js +99 -0
- package/dist/capture/flush-lock.d.ts +11 -0
- package/dist/capture/flush-lock.js +108 -0
- package/dist/capture/flush.d.ts +4 -4
- package/dist/capture/flush.js +19 -80
- package/dist/capture/index.d.ts +3 -3
- package/dist/capture/index.js +3 -3
- package/dist/capture/reingest.d.ts +1 -0
- package/dist/capture/reingest.js +4 -1
- package/dist/capture/scrub.d.ts +1 -0
- package/dist/capture/scrub.js +4 -1
- package/dist/capture/vectors.js +2 -1
- package/dist/cli/commands/daemon.d.ts +1 -0
- package/dist/cli/commands/daemon.js +29 -0
- package/dist/cli/commands/distill.js +2 -1
- package/dist/cli/commands/graph.js +2 -1
- package/dist/cli/commands/maintain.js +3 -1
- package/dist/cli/index.js +4 -0
- package/dist/cli/usage.js +1 -0
- package/dist/config/schema.d.ts +4 -0
- package/dist/config/schema.js +6 -0
- package/dist/context.d.ts +1 -0
- package/dist/context.js +21 -6
- package/dist/daemon/client.d.ts +12 -0
- package/dist/daemon/client.js +88 -0
- package/dist/daemon/paths.d.ts +16 -0
- package/dist/daemon/paths.js +19 -0
- package/dist/daemon/server.d.ts +2 -0
- package/dist/daemon/server.js +225 -0
- package/dist/facts/budget.d.ts +1 -0
- package/dist/facts/budget.js +4 -0
- package/dist/facts/claude-memory.js +1 -1
- package/dist/facts/distill.js +2 -2
- package/dist/facts/known.js +1 -1
- package/dist/facts/model.d.ts +1 -1
- package/dist/facts/model.js +18 -7
- package/dist/facts/start.js +1 -1
- package/dist/graph/query/refresh.js +5 -1
- package/dist/graph/scan/cache.d.ts +1 -0
- package/dist/graph/scan/cache.js +1 -1
- package/dist/graph/scan/persist.js +12 -16
- package/dist/hooks/compact.d.ts +2 -1
- package/dist/hooks/compact.js +2 -2
- package/dist/hooks/detached.d.ts +1 -0
- package/dist/hooks/detached.js +6 -0
- package/dist/hooks/graph-presence.d.ts +3 -0
- package/dist/hooks/graph-presence.js +42 -0
- package/dist/hooks/handlers/prefer-usages.d.ts +2 -0
- package/dist/hooks/handlers/prefer-usages.js +113 -0
- package/dist/hooks/handlers/session-end.d.ts +2 -0
- package/dist/hooks/handlers/session-end.js +62 -0
- package/dist/hooks/handlers/session-start.d.ts +2 -0
- package/dist/hooks/handlers/session-start.js +104 -0
- package/dist/hooks/handlers/turn-end.d.ts +2 -0
- package/dist/hooks/handlers/turn-end.js +31 -0
- package/dist/hooks/handlers/user-prompt.d.ts +2 -0
- package/dist/hooks/handlers/user-prompt.js +55 -0
- package/dist/hooks/local.d.ts +2 -0
- package/dist/hooks/local.js +23 -0
- package/dist/hooks/payload.d.ts +9 -4
- package/dist/hooks/payload.js +26 -17
- package/dist/hooks/prefer-usages.js +2 -118
- package/dist/hooks/route.d.ts +2 -0
- package/dist/hooks/route.js +28 -0
- package/dist/hooks/run.d.ts +14 -0
- package/dist/hooks/run.js +12 -0
- package/dist/hooks/save-turn.d.ts +2 -0
- package/dist/hooks/save-turn.js +11 -0
- package/dist/hooks/session-end.js +2 -63
- package/dist/hooks/session-start.js +2 -90
- package/dist/hooks/turn-end.js +2 -25
- package/dist/hooks/turn-save.js +10 -15
- package/dist/hooks/user-prompt.js +2 -58
- package/dist/search/service.d.ts +1 -0
- package/dist/search/service.js +16 -6
- package/dist/store/adapter.d.ts +7 -5
- package/dist/store/mongo/index.d.ts +3 -6
- package/dist/store/mongo/index.js +5 -0
- package/dist/store/mongo/observations.js +13 -1
- package/dist/store/postgres/driver.d.ts +10 -3
- package/dist/store/postgres/driver.js +12 -0
- package/dist/store/postgres/graph.d.ts +3 -5
- package/dist/store/postgres/graph.js +56 -50
- package/dist/store/postgres/index.d.ts +3 -6
- package/dist/store/postgres/index.js +5 -4
- package/dist/store/postgres/insert.js +98 -50
- package/dist/store/project-resolver.d.ts +3 -0
- package/dist/store/project-resolver.js +34 -23
- package/dist/store/scoped-store.js +1 -1
- package/dist/store/sqlite/graph.d.ts +3 -5
- package/dist/store/sqlite/graph.js +42 -33
- package/dist/store/sqlite/index.d.ts +3 -6
- package/dist/store/sqlite/index.js +3 -0
- package/dist/store/sqlite/observations.js +6 -2
- package/dist/util/job-lock.d.ts +1 -0
- package/dist/util/job-lock.js +50 -10
- package/docs/setup-guide.md +5 -0
- package/package.json +6 -6
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,42 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.13.0
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- **An optional daemon keeps one process warm for the hooks.** Each hook starts node, opens a database
|
|
8
|
+
connection and loads the embedding model before it does any work. `claude-db daemon on` keeps one process
|
|
9
|
+
that the hooks hand their work to, so they skip that cost, and `claude-db daemon off` turns it off. It stops
|
|
10
|
+
after `daemon.idleMinutes` (15) with nothing to do and starts again with the next hook. It is off by default.
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
|
|
14
|
+
- **Hooks send fewer queries to a remote database.** A folder's project scope is read once per process, a batch
|
|
15
|
+
of memories is saved in one statement on Postgres, the session row is written once per prompt, and the prompt
|
|
16
|
+
is embedded while the keyword search runs.
|
|
17
|
+
- **A grep in a repository with no code graph no longer asks the database.** The hook remembers for ten minutes
|
|
18
|
+
that the project has no graph.
|
|
19
|
+
- **A chat's read position is kept when it ends**, so a resumed chat no longer saves its earlier turns again.
|
|
20
|
+
Read positions of chats that are gone are cleared once a day.
|
|
21
|
+
- **Session end no longer starts distill when the day's budget is used up or paused.**
|
|
22
|
+
- **`find_usages` and background graph refreshes no longer read every file when nothing changed since the
|
|
23
|
+
scan.**
|
|
24
|
+
|
|
25
|
+
### Fixed
|
|
26
|
+
|
|
27
|
+
- **A dropped connection to a remote Postgres no longer stops memory.** An idle connection closed by the server,
|
|
28
|
+
as Neon does, could end the MCP server for the rest of the chat.
|
|
29
|
+
- **One slow load of the embedding model no longer turns vectors off for good.** The model is used as soon as it
|
|
30
|
+
loads, and all calls in a process share one wait of `embeddings.timeoutMs`.
|
|
31
|
+
- **Only one save of a chat runs at a time.** Two hooks that saved the same chat at once embedded and wrote the
|
|
32
|
+
same turns twice.
|
|
33
|
+
- **A background job never runs twice at once.** A stale lock is taken over by one process only, and long jobs
|
|
34
|
+
keep their lock fresh.
|
|
35
|
+
- **Code graph changes are saved in one transaction**, so a reader never sees a changed file with no symbols.
|
|
36
|
+
- **Closed work stays closed when its row is saved again.**
|
|
37
|
+
|
|
38
|
+
By @alok-debnath (#38, #39, #41).
|
|
39
|
+
|
|
3
40
|
## 0.12.4
|
|
4
41
|
|
|
5
42
|
### Fixed
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare function readWritten(sessionId: string): Map<string, string>;
|
|
2
|
+
export declare function writeWritten(sessionId: string, prints: ReadonlyMap<string, string>): void;
|
|
3
|
+
export declare function readCursor(sessionId: string): number;
|
|
4
|
+
export declare function advanceCursor(sessionId: string, from: number, to: number): void;
|
|
5
|
+
export declare function resetCursor(sessionId: string): void;
|
|
6
|
+
export declare function clearCursor(sessionId: string): void;
|
|
7
|
+
export declare function sweepCursorsDaily(now?: number): number;
|
|
8
|
+
export declare function sweepCursors(): number;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
import { CONFIG_DIR } from '../config/index.js';
|
|
4
|
+
import { sessionIdsOnDisk } from './transcript.js';
|
|
5
|
+
const SWEEP_EVERY_MS = 24 * 60 * 60 * 1000;
|
|
6
|
+
function cursorName(sessionId) {
|
|
7
|
+
return `${sessionId.replace(/[^\w-]/g, '_')}.offset`;
|
|
8
|
+
}
|
|
9
|
+
function cursorPath(sessionId) {
|
|
10
|
+
return join(CONFIG_DIR, 'cursors', cursorName(sessionId));
|
|
11
|
+
}
|
|
12
|
+
const writtenPath = (sessionId) => cursorPath(sessionId).replace(/\.offset$/, '.written');
|
|
13
|
+
export function readWritten(sessionId) {
|
|
14
|
+
try {
|
|
15
|
+
const parsed = JSON.parse(readFileSync(writtenPath(sessionId), 'utf8'));
|
|
16
|
+
if (!parsed || typeof parsed !== 'object')
|
|
17
|
+
return new Map();
|
|
18
|
+
return new Map(Object.entries(parsed).filter((entry) => typeof entry[1] === 'string'));
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return new Map();
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
export function writeWritten(sessionId, prints) {
|
|
25
|
+
const path = writtenPath(sessionId);
|
|
26
|
+
try {
|
|
27
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
28
|
+
writeFileSync(path, JSON.stringify(Object.fromEntries(prints)), 'utf8');
|
|
29
|
+
}
|
|
30
|
+
catch { }
|
|
31
|
+
}
|
|
32
|
+
export function readCursor(sessionId) {
|
|
33
|
+
try {
|
|
34
|
+
const value = Number.parseInt(readFileSync(cursorPath(sessionId), 'utf8'), 10);
|
|
35
|
+
return Number.isFinite(value) && value >= 0 ? value : 0;
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return 0;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
export function advanceCursor(sessionId, from, to) {
|
|
42
|
+
const now = readCursor(sessionId);
|
|
43
|
+
if (now !== from && now > to)
|
|
44
|
+
return;
|
|
45
|
+
writeCursor(sessionId, to);
|
|
46
|
+
}
|
|
47
|
+
function writeCursor(sessionId, offset) {
|
|
48
|
+
const path = cursorPath(sessionId);
|
|
49
|
+
try {
|
|
50
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
51
|
+
writeFileSync(path, String(offset), 'utf8');
|
|
52
|
+
}
|
|
53
|
+
catch { }
|
|
54
|
+
}
|
|
55
|
+
export function resetCursor(sessionId) {
|
|
56
|
+
writeCursor(sessionId, 0);
|
|
57
|
+
rmSync(writtenPath(sessionId), { force: true });
|
|
58
|
+
}
|
|
59
|
+
export function clearCursor(sessionId) {
|
|
60
|
+
rmSync(cursorPath(sessionId), { force: true });
|
|
61
|
+
rmSync(cursorPath(sessionId).replace(/\.offset$/, '.shown'), { force: true });
|
|
62
|
+
rmSync(writtenPath(sessionId), { force: true });
|
|
63
|
+
}
|
|
64
|
+
export function sweepCursorsDaily(now = Date.now()) {
|
|
65
|
+
const marker = join(CONFIG_DIR, 'cursors', '.swept');
|
|
66
|
+
try {
|
|
67
|
+
if (now - statSync(marker).mtimeMs < SWEEP_EVERY_MS)
|
|
68
|
+
return 0;
|
|
69
|
+
}
|
|
70
|
+
catch { }
|
|
71
|
+
try {
|
|
72
|
+
mkdirSync(dirname(marker), { recursive: true });
|
|
73
|
+
writeFileSync(marker, String(now), 'utf8');
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
return 0;
|
|
77
|
+
}
|
|
78
|
+
return sweepCursors();
|
|
79
|
+
}
|
|
80
|
+
export function sweepCursors() {
|
|
81
|
+
const dir = join(CONFIG_DIR, 'cursors');
|
|
82
|
+
let files;
|
|
83
|
+
try {
|
|
84
|
+
files = readdirSync(dir);
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
return 0;
|
|
88
|
+
}
|
|
89
|
+
const live = new Set(sessionIdsOnDisk().map((id) => id.replace(/[^\w-]/g, '_')));
|
|
90
|
+
let removed = 0;
|
|
91
|
+
for (const file of files) {
|
|
92
|
+
const match = /^(.+)\.(offset|shown|written)$/.exec(file);
|
|
93
|
+
if (!match?.[1] || live.has(match[1]))
|
|
94
|
+
continue;
|
|
95
|
+
rmSync(join(dir, file), { force: true });
|
|
96
|
+
removed += 1;
|
|
97
|
+
}
|
|
98
|
+
return removed;
|
|
99
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface FlushLockOptions {
|
|
2
|
+
waitMs?: number;
|
|
3
|
+
onTimeout?: 'run' | 'skip';
|
|
4
|
+
}
|
|
5
|
+
export type Locked<T> = {
|
|
6
|
+
ran: true;
|
|
7
|
+
value: T;
|
|
8
|
+
} | {
|
|
9
|
+
ran: false;
|
|
10
|
+
};
|
|
11
|
+
export declare function withFlushLock<T>(sessionId: string, work: () => Promise<T>, { waitMs, onTimeout }?: FlushLockOptions): Promise<Locked<T>>;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { closeSync, fstatSync, mkdirSync, openSync, readFileSync, rmSync, writeFileSync, } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { CONFIG_DIR } from '../config/index.js';
|
|
5
|
+
import { safeName } from '../util/json-file.js';
|
|
6
|
+
const WAIT_MS = 5_000;
|
|
7
|
+
const POLL_MS = 100;
|
|
8
|
+
const STALE_MS = 2 * 60_000;
|
|
9
|
+
const UNREADABLE_GRACE_MS = 5_000;
|
|
10
|
+
const lockOf = (sessionId) => join(CONFIG_DIR, 'cursors', `${safeName(sessionId)}.lock`);
|
|
11
|
+
function alive(pid) {
|
|
12
|
+
try {
|
|
13
|
+
process.kill(pid, 0);
|
|
14
|
+
return true;
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
return error.code === 'EPERM';
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
function readOpen(path) {
|
|
21
|
+
let fd;
|
|
22
|
+
try {
|
|
23
|
+
fd = openSync(path, 'r');
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
try {
|
|
29
|
+
return { text: readFileSync(fd, 'utf8'), mtimeMs: fstatSync(fd).mtimeMs };
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
finally {
|
|
35
|
+
closeSync(fd);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
function abandoned(path, now) {
|
|
39
|
+
const held = readOpen(path);
|
|
40
|
+
if (!held)
|
|
41
|
+
return false;
|
|
42
|
+
const age = now - held.mtimeMs;
|
|
43
|
+
if (age > STALE_MS)
|
|
44
|
+
return true;
|
|
45
|
+
const pid = Number(held.text.split(' ')[0]);
|
|
46
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
47
|
+
return age > UNREADABLE_GRACE_MS;
|
|
48
|
+
return !alive(pid);
|
|
49
|
+
}
|
|
50
|
+
function tryLock(path, owner) {
|
|
51
|
+
try {
|
|
52
|
+
writeFileSync(path, owner, { flag: 'wx' });
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function release(path, owner) {
|
|
60
|
+
if (readOpen(path)?.text === owner)
|
|
61
|
+
rmSync(path, { force: true });
|
|
62
|
+
}
|
|
63
|
+
function takeOver(lock, owner, now) {
|
|
64
|
+
const breaker = `${lock}.break`;
|
|
65
|
+
if (!tryLock(breaker, owner)) {
|
|
66
|
+
if (!abandoned(breaker, now))
|
|
67
|
+
return false;
|
|
68
|
+
rmSync(breaker, { force: true });
|
|
69
|
+
if (!tryLock(breaker, owner))
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
try {
|
|
73
|
+
if (!abandoned(lock, now))
|
|
74
|
+
return false;
|
|
75
|
+
rmSync(lock, { force: true });
|
|
76
|
+
return tryLock(lock, owner);
|
|
77
|
+
}
|
|
78
|
+
finally {
|
|
79
|
+
release(breaker, owner);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
async function acquire(lock, owner, waitMs) {
|
|
83
|
+
mkdirSync(join(CONFIG_DIR, 'cursors'), { recursive: true });
|
|
84
|
+
const until = Date.now() + waitMs;
|
|
85
|
+
for (;;) {
|
|
86
|
+
if (tryLock(lock, owner))
|
|
87
|
+
return true;
|
|
88
|
+
if (abandoned(lock, Date.now()) && takeOver(lock, owner, Date.now()))
|
|
89
|
+
return true;
|
|
90
|
+
if (Date.now() >= until)
|
|
91
|
+
return false;
|
|
92
|
+
await new Promise((settle) => setTimeout(settle, POLL_MS));
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
export async function withFlushLock(sessionId, work, { waitMs = WAIT_MS, onTimeout = 'run' } = {}) {
|
|
96
|
+
const lock = lockOf(sessionId);
|
|
97
|
+
const owner = `${process.pid} ${randomUUID()}`;
|
|
98
|
+
const held = await acquire(lock, owner, waitMs).catch(() => false);
|
|
99
|
+
if (!held && onTimeout === 'skip')
|
|
100
|
+
return { ran: false };
|
|
101
|
+
try {
|
|
102
|
+
return { ran: true, value: await work() };
|
|
103
|
+
}
|
|
104
|
+
finally {
|
|
105
|
+
if (held)
|
|
106
|
+
release(lock, owner);
|
|
107
|
+
}
|
|
108
|
+
}
|
package/dist/capture/flush.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import type { RecallContext } from '../context.js';
|
|
2
2
|
import type { Observation } from '../types.js';
|
|
3
|
+
import type { FlushLockOptions } from './flush-lock.js';
|
|
3
4
|
export interface FlushResult {
|
|
4
5
|
observations: number;
|
|
5
6
|
summary: string | null;
|
|
6
7
|
lastReply: string;
|
|
8
|
+
skipped?: boolean;
|
|
7
9
|
}
|
|
8
|
-
export declare function flushSession(ctx: RecallContext, sessionId: string, project: string, transcriptPath?: string, rebuild?: boolean, finalReply?: string): Promise<FlushResult>;
|
|
10
|
+
export declare function flushSession(ctx: RecallContext, sessionId: string, project: string, transcriptPath?: string, rebuild?: boolean, finalReply?: string, lock?: FlushLockOptions): Promise<FlushResult>;
|
|
9
11
|
export declare function embedObservations(ctx: RecallContext, observations: Observation[]): Promise<void>;
|
|
10
12
|
export declare function summarize(observations: Observation[], previous?: string): string;
|
|
11
|
-
export
|
|
12
|
-
export declare function clearCursor(sessionId: string): void;
|
|
13
|
-
export declare function sweepCursors(): number;
|
|
13
|
+
export { clearCursor, resetCursor, sweepCursors, sweepCursorsDaily } from './cursors.js';
|
package/dist/capture/flush.js
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
-
import {
|
|
3
|
-
import { dirname, join } from 'node:path';
|
|
4
|
-
import { CONFIG_DIR } from '../config/index.js';
|
|
5
|
-
import { readTranscript, sessionIdsOnDisk, transcriptPathFor } from './transcript.js';
|
|
2
|
+
import { readTranscript, transcriptPathFor } from './transcript.js';
|
|
6
3
|
import { observationsFromTurns } from './turn-extractor.js';
|
|
7
4
|
import { closeLandedWork } from './progress.js';
|
|
8
5
|
import { noteMissingVectors } from './vectors.js';
|
|
9
|
-
|
|
6
|
+
import { advanceCursor, readCursor, readWritten, writeWritten } from './cursors.js';
|
|
7
|
+
import { withFlushLock } from './flush-lock.js';
|
|
8
|
+
export async function flushSession(ctx, sessionId, project, transcriptPath, rebuild = false, finalReply = '', lock = {}) {
|
|
9
|
+
const result = await withFlushLock(sessionId, () => flushUnlocked(ctx, sessionId, project, transcriptPath, rebuild, finalReply), lock);
|
|
10
|
+
if (result.ran)
|
|
11
|
+
return result.value;
|
|
12
|
+
return { observations: 0, summary: null, lastReply: '', skipped: true };
|
|
13
|
+
}
|
|
14
|
+
async function flushUnlocked(ctx, sessionId, project, transcriptPath, rebuild, finalReply) {
|
|
10
15
|
const path = transcriptPath ?? transcriptPathFor(project, sessionId);
|
|
11
16
|
const cursor = readCursor(sessionId);
|
|
12
17
|
const read = readTranscript(path, cursor);
|
|
@@ -15,18 +20,23 @@ export async function flushSession(ctx, sessionId, project, transcriptPath, rebu
|
|
|
15
20
|
const lastReply = turns.at(-1)?.reasoning ?? '';
|
|
16
21
|
if (turns.length === 0) {
|
|
17
22
|
await closeLandedWork(ctx.store, project);
|
|
18
|
-
|
|
23
|
+
advanceCursor(sessionId, cursor, nextOffset);
|
|
19
24
|
return { observations: 0, summary: null, lastReply };
|
|
20
25
|
}
|
|
21
26
|
const observations = observationsFromTurns(turns, sessionId, project, ctx.config);
|
|
22
27
|
if (observations.length === 0) {
|
|
23
28
|
await closeLandedWork(ctx.store, project);
|
|
24
|
-
|
|
29
|
+
advanceCursor(sessionId, cursor, nextOffset);
|
|
25
30
|
return { observations: 0, summary: null, lastReply };
|
|
26
31
|
}
|
|
27
32
|
const written = rebuild ? new Map() : readWritten(sessionId);
|
|
28
33
|
const prints = new Map(observations.map((obs) => [obs.id, printOf(obs)]));
|
|
29
34
|
const fresh = observations.filter((obs) => written.get(obs.id) !== prints.get(obs.id));
|
|
35
|
+
if (fresh.length === 0) {
|
|
36
|
+
await closeLandedWork(ctx.store, project);
|
|
37
|
+
advanceCursor(sessionId, cursor, nextOffset);
|
|
38
|
+
return { observations: 0, summary: null, lastReply };
|
|
39
|
+
}
|
|
30
40
|
await embedObservations(ctx, fresh);
|
|
31
41
|
await ctx.store.insertObservations(fresh);
|
|
32
42
|
writeWritten(sessionId, prints);
|
|
@@ -39,7 +49,7 @@ export async function flushSession(ctx, sessionId, project, transcriptPath, rebu
|
|
|
39
49
|
startedAt: observations[0]?.createdAt ?? Date.now(),
|
|
40
50
|
summary,
|
|
41
51
|
});
|
|
42
|
-
|
|
52
|
+
advanceCursor(sessionId, cursor, nextOffset);
|
|
43
53
|
return { observations: observations.length, summary, lastReply };
|
|
44
54
|
}
|
|
45
55
|
export async function embedObservations(ctx, observations) {
|
|
@@ -99,76 +109,5 @@ export function summarize(observations, previous) {
|
|
|
99
109
|
}
|
|
100
110
|
return kept.join(' | ');
|
|
101
111
|
}
|
|
102
|
-
function cursorName(sessionId) {
|
|
103
|
-
return `${sessionId.replace(/[^\w-]/g, '_')}.offset`;
|
|
104
|
-
}
|
|
105
|
-
function cursorPath(sessionId) {
|
|
106
|
-
return join(CONFIG_DIR, 'cursors', cursorName(sessionId));
|
|
107
|
-
}
|
|
108
|
-
const writtenPath = (sessionId) => cursorPath(sessionId).replace(/\.offset$/, '.written');
|
|
109
112
|
const printOf = (obs) => createHash('sha256').update(JSON.stringify(obs)).digest('base64url');
|
|
110
|
-
|
|
111
|
-
try {
|
|
112
|
-
const parsed = JSON.parse(readFileSync(writtenPath(sessionId), 'utf8'));
|
|
113
|
-
if (!parsed || typeof parsed !== 'object')
|
|
114
|
-
return new Map();
|
|
115
|
-
return new Map(Object.entries(parsed).filter((entry) => typeof entry[1] === 'string'));
|
|
116
|
-
}
|
|
117
|
-
catch {
|
|
118
|
-
return new Map();
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
function writeWritten(sessionId, prints) {
|
|
122
|
-
const path = writtenPath(sessionId);
|
|
123
|
-
try {
|
|
124
|
-
mkdirSync(dirname(path), { recursive: true });
|
|
125
|
-
writeFileSync(path, JSON.stringify(Object.fromEntries(prints)), 'utf8');
|
|
126
|
-
}
|
|
127
|
-
catch { }
|
|
128
|
-
}
|
|
129
|
-
function readCursor(sessionId) {
|
|
130
|
-
try {
|
|
131
|
-
const value = Number.parseInt(readFileSync(cursorPath(sessionId), 'utf8'), 10);
|
|
132
|
-
return Number.isFinite(value) && value >= 0 ? value : 0;
|
|
133
|
-
}
|
|
134
|
-
catch {
|
|
135
|
-
return 0;
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
function writeCursor(sessionId, offset) {
|
|
139
|
-
const path = cursorPath(sessionId);
|
|
140
|
-
try {
|
|
141
|
-
mkdirSync(dirname(path), { recursive: true });
|
|
142
|
-
writeFileSync(path, String(offset), 'utf8');
|
|
143
|
-
}
|
|
144
|
-
catch { }
|
|
145
|
-
}
|
|
146
|
-
export function resetCursor(sessionId) {
|
|
147
|
-
writeCursor(sessionId, 0);
|
|
148
|
-
rmSync(writtenPath(sessionId), { force: true });
|
|
149
|
-
}
|
|
150
|
-
export function clearCursor(sessionId) {
|
|
151
|
-
rmSync(cursorPath(sessionId), { force: true });
|
|
152
|
-
rmSync(cursorPath(sessionId).replace(/\.offset$/, '.shown'), { force: true });
|
|
153
|
-
rmSync(writtenPath(sessionId), { force: true });
|
|
154
|
-
}
|
|
155
|
-
export function sweepCursors() {
|
|
156
|
-
const dir = join(CONFIG_DIR, 'cursors');
|
|
157
|
-
let files;
|
|
158
|
-
try {
|
|
159
|
-
files = readdirSync(dir);
|
|
160
|
-
}
|
|
161
|
-
catch {
|
|
162
|
-
return 0;
|
|
163
|
-
}
|
|
164
|
-
const live = new Set(sessionIdsOnDisk().map((id) => id.replace(/[^\w-]/g, '_')));
|
|
165
|
-
let removed = 0;
|
|
166
|
-
for (const file of files) {
|
|
167
|
-
const match = /^(.+)\.(offset|shown|written)$/.exec(file);
|
|
168
|
-
if (!match?.[1] || live.has(match[1]))
|
|
169
|
-
continue;
|
|
170
|
-
rmSync(join(dir, file), { force: true });
|
|
171
|
-
removed += 1;
|
|
172
|
-
}
|
|
173
|
-
return removed;
|
|
174
|
-
}
|
|
113
|
+
export { clearCursor, resetCursor, sweepCursors, sweepCursorsDaily } from './cursors.js';
|
package/dist/capture/index.d.ts
CHANGED
|
@@ -7,11 +7,11 @@ export type { Turn, TranscriptRead } from './transcript.js';
|
|
|
7
7
|
export { observationsFromGit } from './git.js';
|
|
8
8
|
export { observationsFromTurns } from './turn-extractor.js';
|
|
9
9
|
export { redact } from './redact.js';
|
|
10
|
-
export { claimScrub, finishScrub, releaseScrub, scrubSecrets } from './scrub.js';
|
|
11
|
-
export { flushSession, resetCursor, clearCursor, sweepCursors, summarize, embedObservations, } from './flush.js';
|
|
10
|
+
export { claimScrub, finishScrub, keepScrub, releaseScrub, scrubSecrets } from './scrub.js';
|
|
11
|
+
export { flushSession, resetCursor, clearCursor, sweepCursors, sweepCursorsDaily, summarize, embedObservations, } from './flush.js';
|
|
12
12
|
export type { FlushResult } from './flush.js';
|
|
13
13
|
export { claimReembed, finishReembed, releaseReembed, startReembed, vectorsMissing, } from './vectors.js';
|
|
14
14
|
export { closeLandedWork, openWork } from './progress.js';
|
|
15
|
-
export { REINGEST_VERSION, claimReingest, finishReingest, reingestDone, reingestProject, rememberedTranscripts, reingestSession, releaseReingest, } from './reingest.js';
|
|
15
|
+
export { REINGEST_VERSION, claimReingest, finishReingest, keepReingest, reingestDone, reingestProject, rememberedTranscripts, reingestSession, releaseReingest, } from './reingest.js';
|
|
16
16
|
export type { ProjectReingest, SessionReingest } from './reingest.js';
|
|
17
17
|
export { aiSummary, buildSummaryPrompt, validateSummary } from './summarize.js';
|
package/dist/capture/index.js
CHANGED
|
@@ -5,9 +5,9 @@ export { readTranscript, sessionIdsOnDisk, transcriptPathFor, transcriptsFor, }
|
|
|
5
5
|
export { observationsFromGit } from './git.js';
|
|
6
6
|
export { observationsFromTurns } from './turn-extractor.js';
|
|
7
7
|
export { redact } from './redact.js';
|
|
8
|
-
export { claimScrub, finishScrub, releaseScrub, scrubSecrets } from './scrub.js';
|
|
9
|
-
export { flushSession, resetCursor, clearCursor, sweepCursors, summarize, embedObservations, } from './flush.js';
|
|
8
|
+
export { claimScrub, finishScrub, keepScrub, releaseScrub, scrubSecrets } from './scrub.js';
|
|
9
|
+
export { flushSession, resetCursor, clearCursor, sweepCursors, sweepCursorsDaily, summarize, embedObservations, } from './flush.js';
|
|
10
10
|
export { claimReembed, finishReembed, releaseReembed, startReembed, vectorsMissing, } from './vectors.js';
|
|
11
11
|
export { closeLandedWork, openWork } from './progress.js';
|
|
12
|
-
export { REINGEST_VERSION, claimReingest, finishReingest, reingestDone, reingestProject, rememberedTranscripts, reingestSession, releaseReingest, } from './reingest.js';
|
|
12
|
+
export { REINGEST_VERSION, claimReingest, finishReingest, keepReingest, reingestDone, reingestProject, rememberedTranscripts, reingestSession, releaseReingest, } from './reingest.js';
|
|
13
13
|
export { aiSummary, buildSummaryPrompt, validateSummary } from './summarize.js';
|
|
@@ -15,5 +15,6 @@ export declare function rememberedTranscripts(ctx: RecallContext, project: strin
|
|
|
15
15
|
export declare function reingestProject(ctx: RecallContext, project: string, paths?: string[], onSession?: (result: SessionReingest) => void): Promise<ProjectReingest>;
|
|
16
16
|
export declare function reingestDone(project: string): boolean;
|
|
17
17
|
export declare function claimReingest(project: string, now?: number): boolean;
|
|
18
|
+
export declare function keepReingest(project: string): void;
|
|
18
19
|
export declare function finishReingest(project: string): void;
|
|
19
20
|
export declare function releaseReingest(project: string): void;
|
package/dist/capture/reingest.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { basename } from 'node:path';
|
|
2
|
-
import { claimJob, finishJob, jobMark, releaseJob } from '../util/job-lock.js';
|
|
2
|
+
import { claimJob, finishJob, jobMark, keepJob, releaseJob } from '../util/job-lock.js';
|
|
3
3
|
import { flushSession, resetCursor } from './flush.js';
|
|
4
4
|
import { readTranscript, transcriptsFor } from './transcript.js';
|
|
5
5
|
import { observationsFromTurns } from './turn-extractor.js';
|
|
@@ -48,6 +48,9 @@ export function claimReingest(project, now = Date.now()) {
|
|
|
48
48
|
return false;
|
|
49
49
|
return claimJob(JOB, project, now);
|
|
50
50
|
}
|
|
51
|
+
export function keepReingest(project) {
|
|
52
|
+
keepJob(JOB, project);
|
|
53
|
+
}
|
|
51
54
|
export function finishReingest(project) {
|
|
52
55
|
finishJob(JOB, project, REINGEST_VERSION);
|
|
53
56
|
}
|
package/dist/capture/scrub.d.ts
CHANGED
|
@@ -7,5 +7,6 @@ export interface ScrubResult {
|
|
|
7
7
|
export declare function scrubSecrets(ctx: RecallContext, now?: number): Promise<ScrubResult>;
|
|
8
8
|
export declare function scrubDone(database: string): boolean;
|
|
9
9
|
export declare function claimScrub(database: string, now?: number): boolean;
|
|
10
|
+
export declare function keepScrub(database: string): void;
|
|
10
11
|
export declare function finishScrub(database: string): void;
|
|
11
12
|
export declare function releaseScrub(database: string): void;
|
package/dist/capture/scrub.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { eachObservation } from '../store/each.js';
|
|
2
|
-
import { claimJob, finishJob, jobMark, releaseJob } from '../util/job-lock.js';
|
|
2
|
+
import { claimJob, finishJob, jobMark, keepJob, releaseJob } from '../util/job-lock.js';
|
|
3
3
|
import { embedObservations } from './flush.js';
|
|
4
4
|
import { redact } from './redact.js';
|
|
5
5
|
export const REDACT_VERSION = 1;
|
|
@@ -40,6 +40,9 @@ export function scrubDone(database) {
|
|
|
40
40
|
export function claimScrub(database, now = Date.now()) {
|
|
41
41
|
return !scrubDone(database) && claimJob(JOB, database, now);
|
|
42
42
|
}
|
|
43
|
+
export function keepScrub(database) {
|
|
44
|
+
keepJob(JOB, database);
|
|
45
|
+
}
|
|
43
46
|
export function finishScrub(database) {
|
|
44
47
|
finishJob(JOB, database, REDACT_VERSION);
|
|
45
48
|
}
|
package/dist/capture/vectors.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { CONFIG_DIR } from '../config/index.js';
|
|
4
|
-
import { claimJob, finishJob, jobMark, releaseJob } from '../util/job-lock.js';
|
|
4
|
+
import { claimJob, finishJob, jobMark, keepJob, releaseJob } from '../util/job-lock.js';
|
|
5
5
|
import { scopeToken } from '../util/scope.js';
|
|
6
6
|
const JOB = 'reembed';
|
|
7
7
|
const FLAGS = 'vectors-missing';
|
|
@@ -29,6 +29,7 @@ export function claimReembed(project, now = Date.now()) {
|
|
|
29
29
|
}
|
|
30
30
|
export function startReembed(project) {
|
|
31
31
|
rmSync(flagPath(project), { force: true });
|
|
32
|
+
keepJob(JOB, project);
|
|
32
33
|
}
|
|
33
34
|
export function finishReembed(project, now = Date.now()) {
|
|
34
35
|
finishJob(JOB, project, now);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function cmdDaemon(argv: (string | undefined)[]): Promise<void>;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { setConfigValue } from '../../config/index.js';
|
|
2
|
+
import { askDaemon, startDaemon } from '../../daemon/client.js';
|
|
3
|
+
import { buildId } from '../../daemon/paths.js';
|
|
4
|
+
const STOP = '__stop';
|
|
5
|
+
async function running() {
|
|
6
|
+
const asked = await askDaemon({ build: buildId(), hook: '__ping', raw: '', env: {} }, 1000);
|
|
7
|
+
return asked.kind === 'done';
|
|
8
|
+
}
|
|
9
|
+
export async function cmdDaemon(argv) {
|
|
10
|
+
const [action] = argv;
|
|
11
|
+
switch (action) {
|
|
12
|
+
case 'on':
|
|
13
|
+
case 'off':
|
|
14
|
+
setConfigValue('daemon', 'enabled', action === 'on');
|
|
15
|
+
if (action === 'off')
|
|
16
|
+
await askDaemon({ build: buildId(), hook: STOP, raw: '', env: {} }, 2000);
|
|
17
|
+
console.log(`daemon: ${action}`);
|
|
18
|
+
return;
|
|
19
|
+
case 'start':
|
|
20
|
+
console.log(startDaemon() ? 'daemon: starting' : 'daemon: already starting');
|
|
21
|
+
return;
|
|
22
|
+
case 'stop':
|
|
23
|
+
await askDaemon({ build: buildId(), hook: STOP, raw: '', env: {} }, 2000);
|
|
24
|
+
console.log('daemon: stopped');
|
|
25
|
+
return;
|
|
26
|
+
default:
|
|
27
|
+
console.log(`daemon: ${(await running()) ? 'running' : 'not running'}`);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -6,7 +6,7 @@ import { importClaudeMemory } from '../../facts/claude-memory.js';
|
|
|
6
6
|
import { FACTS_JOB, backfill, distillSession, factCounts, pendingSessions, } from '../../facts/distill.js';
|
|
7
7
|
import { runHeadlessResult } from '../../util/claude-cli.js';
|
|
8
8
|
import { describePause } from '../../util/daily-budget.js';
|
|
9
|
-
import { finishJob, releaseJob } from '../../util/job-lock.js';
|
|
9
|
+
import { finishJob, keepJob, releaseJob } from '../../util/job-lock.js';
|
|
10
10
|
import { resolveProject } from '../../util/project.js';
|
|
11
11
|
export async function cmdDistill(argv) {
|
|
12
12
|
const [first] = argv;
|
|
@@ -37,6 +37,7 @@ export async function cmdDistill(argv) {
|
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
39
|
async function runBackfill(project) {
|
|
40
|
+
keepJob(FACTS_JOB, project);
|
|
40
41
|
try {
|
|
41
42
|
const ctx = await createContext();
|
|
42
43
|
try {
|
|
@@ -3,7 +3,7 @@ import { findUsages, formatUsages, repoRootFor } from '../../usages/index.js';
|
|
|
3
3
|
import { NO_GRAPH, NO_GRAPH_FOR_PATH, answerQuery, formatGraph, hasGraph, languageNames, refreshGraph, saveScan, scanRepository, suggestFor, } from '../../graph/index.js';
|
|
4
4
|
import { PLATFORM } from '../../graph/grammars.js';
|
|
5
5
|
import { GRAPH_JOB } from '../../graph/scan/job.js';
|
|
6
|
-
import { releaseJob } from '../../util/job-lock.js';
|
|
6
|
+
import { keepJob, releaseJob } from '../../util/job-lock.js';
|
|
7
7
|
import { resolveProject } from '../../util/project.js';
|
|
8
8
|
import { valueOf, withoutFlags } from '../args.js';
|
|
9
9
|
function printScan(root, scan, elapsedMs) {
|
|
@@ -51,6 +51,7 @@ export async function cmdScan(argv) {
|
|
|
51
51
|
await runScan(root, project, argv.includes('--force'));
|
|
52
52
|
return;
|
|
53
53
|
}
|
|
54
|
+
keepJob(GRAPH_JOB, project);
|
|
54
55
|
try {
|
|
55
56
|
await runRefresh(root, project);
|
|
56
57
|
}
|
|
@@ -2,7 +2,7 @@ import { CONFIG_DIR, loadConfig } from '../../config/index.js';
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { checkForUpdate } from '../../update.js';
|
|
4
4
|
import { createContext } from '../../context.js';
|
|
5
|
-
import { finishReingest, finishScrub, redact, reingestProject, releaseReingest, releaseScrub, rememberedTranscripts, scrubSecrets, sweepCursors, transcriptsFor, } from '../../capture/index.js';
|
|
5
|
+
import { finishReingest, finishScrub, keepReingest, keepScrub, redact, reingestProject, releaseReingest, releaseScrub, rememberedTranscripts, scrubSecrets, sweepCursors, transcriptsFor, } from '../../capture/index.js';
|
|
6
6
|
import { resolveProject } from '../../util/project.js';
|
|
7
7
|
import { rmSync } from 'node:fs';
|
|
8
8
|
export async function cmdReset(argv) {
|
|
@@ -42,6 +42,7 @@ export async function cmdFlush(argv) {
|
|
|
42
42
|
process.exit(1);
|
|
43
43
|
}
|
|
44
44
|
let total;
|
|
45
|
+
keepReingest(project);
|
|
45
46
|
try {
|
|
46
47
|
total = await reingest(project, transcripts, repair);
|
|
47
48
|
}
|
|
@@ -93,6 +94,7 @@ export async function cmdRedact(argv) {
|
|
|
93
94
|
const background = argv.includes('--background');
|
|
94
95
|
const ctx = await createContext();
|
|
95
96
|
const database = ctx.config.database;
|
|
97
|
+
keepScrub(database);
|
|
96
98
|
let result;
|
|
97
99
|
try {
|
|
98
100
|
result = await scrubSecrets(ctx);
|
package/dist/cli/index.js
CHANGED
|
@@ -17,6 +17,7 @@ import { cmdLanguages } from './commands/languages.js';
|
|
|
17
17
|
import { cmdSeed } from './commands/seed.js';
|
|
18
18
|
import { cmdAdoption } from './commands/adoption.js';
|
|
19
19
|
import { cmdView } from './commands/view.js';
|
|
20
|
+
import { cmdDaemon } from './commands/daemon.js';
|
|
20
21
|
import { silenceSqliteWarning } from '../util/warnings.js';
|
|
21
22
|
import { packageVersion } from '../update.js';
|
|
22
23
|
silenceSqliteWarning();
|
|
@@ -117,6 +118,9 @@ async function run() {
|
|
|
117
118
|
case 'sync':
|
|
118
119
|
await cmdSync(args);
|
|
119
120
|
break;
|
|
121
|
+
case 'daemon':
|
|
122
|
+
await cmdDaemon(args);
|
|
123
|
+
break;
|
|
120
124
|
case '--version':
|
|
121
125
|
case '-v':
|
|
122
126
|
console.log(packageVersion());
|
package/dist/cli/usage.js
CHANGED
|
@@ -34,6 +34,7 @@ export function usage() {
|
|
|
34
34
|
export [--all] > out.jsonl Dump memory as JSONL, for backup or migration
|
|
35
35
|
import <file.jsonl> Load a dump back in (safe to repeat)
|
|
36
36
|
sync <url> [--yes] Two-way merge with another database
|
|
37
|
+
daemon [on|off|start|stop] Serve hooks from one warm process; no argument shows status
|
|
37
38
|
update Install a newer compatible release now
|
|
38
39
|
reembed [--project] Re-embed with the current model
|
|
39
40
|
prune --older-than <days> Delete old memory (dry run without --yes)
|
package/dist/config/schema.d.ts
CHANGED
|
@@ -17,6 +17,10 @@ export declare const ConfigSchema: z.ZodObject<{
|
|
|
17
17
|
notify: "notify";
|
|
18
18
|
off: "off";
|
|
19
19
|
}>>;
|
|
20
|
+
daemon: z.ZodPrefault<z.ZodObject<{
|
|
21
|
+
enabled: z.ZodDefault<z.ZodBoolean>;
|
|
22
|
+
idleMinutes: z.ZodDefault<z.ZodNumber>;
|
|
23
|
+
}, z.core.$strip>>;
|
|
20
24
|
project: z.ZodPrefault<z.ZodObject<{
|
|
21
25
|
remote: z.ZodDefault<z.ZodString>;
|
|
22
26
|
}, z.core.$strip>>;
|