clementine-agent 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/assistant.js +12 -0
- package/dist/cli/dashboard.js +3034 -734
- package/dist/cli/static/LICENSE-NOTICES.md +12 -0
- package/dist/cli/static/drawflow.min.css +1 -0
- package/dist/cli/static/drawflow.min.js +1 -0
- package/dist/config.d.ts +11 -0
- package/dist/config.js +16 -0
- package/dist/dashboard/builder/dry-run.d.ts +31 -0
- package/dist/dashboard/builder/dry-run.js +138 -0
- package/dist/dashboard/builder/events.d.ts +23 -0
- package/dist/dashboard/builder/events.js +28 -0
- package/dist/dashboard/builder/mcp-invoke.d.ts +25 -0
- package/dist/dashboard/builder/mcp-invoke.js +143 -0
- package/dist/dashboard/builder/runner.d.ts +68 -0
- package/dist/dashboard/builder/runner.js +418 -0
- package/dist/dashboard/builder/serializer.d.ts +79 -0
- package/dist/dashboard/builder/serializer.js +547 -0
- package/dist/dashboard/builder/snapshots.d.ts +32 -0
- package/dist/dashboard/builder/snapshots.js +138 -0
- package/dist/dashboard/builder/validation.d.ts +26 -0
- package/dist/dashboard/builder/validation.js +183 -0
- package/dist/gateway/router.js +31 -2
- package/dist/index.js +38 -0
- package/dist/memory/chunker.js +13 -2
- package/dist/memory/hot-cache.d.ts +38 -0
- package/dist/memory/hot-cache.js +73 -0
- package/dist/memory/integrity.d.ts +28 -0
- package/dist/memory/integrity.js +119 -0
- package/dist/memory/maintenance.d.ts +23 -2
- package/dist/memory/maintenance.js +140 -3
- package/dist/memory/store.d.ts +259 -2
- package/dist/memory/store.js +751 -21
- package/dist/memory/write-queue.d.ts +96 -0
- package/dist/memory/write-queue.js +165 -0
- package/dist/tools/builder-tools.d.ts +13 -0
- package/dist/tools/builder-tools.js +437 -0
- package/dist/tools/mcp-server.js +2 -0
- package/dist/tools/memory-tools.js +38 -1
- package/dist/types.d.ts +56 -2
- package/package.json +2 -2
- package/vault/00-System/skills/builder-canvas.md +126 -0
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory store integrity probes — self-healing checks that run on the
|
|
3
|
+
* janitor's periodic cycle. Each probe is independent and conservative:
|
|
4
|
+
* - reports what it found,
|
|
5
|
+
* - repairs only when the fix is non-destructive,
|
|
6
|
+
* - never throws (logs and continues).
|
|
7
|
+
*
|
|
8
|
+
* Three checks today (the cheap, high-value ones):
|
|
9
|
+
* 1. FTS5 contentless-table integrity → auto-rebuild on failure
|
|
10
|
+
* 2. derived_from references to deleted chunks → nullify the dangling refs
|
|
11
|
+
* 3. chunks with content but no embedding → return count for backfill
|
|
12
|
+
*
|
|
13
|
+
* Graph reachability is intentionally NOT probed here — it lives in
|
|
14
|
+
* graph-store.ts's own health probe, which auto-restarts FalkorDB.
|
|
15
|
+
*/
|
|
16
|
+
import pino from 'pino';
|
|
17
|
+
const logger = pino({ name: 'clementine.integrity' });
|
|
18
|
+
/**
|
|
19
|
+
* Run all probes and apply safe repairs. Returns a report; never throws.
|
|
20
|
+
* The store argument is typed loose so this module can be called from
|
|
21
|
+
* maintenance.ts without an import cycle.
|
|
22
|
+
*/
|
|
23
|
+
export function runIntegrityProbes(store) {
|
|
24
|
+
const report = {
|
|
25
|
+
ftsOk: true,
|
|
26
|
+
ftsRebuilt: false,
|
|
27
|
+
orphanRefsNulled: 0,
|
|
28
|
+
missingEmbeddings: 0,
|
|
29
|
+
};
|
|
30
|
+
// 1. FTS5 integrity. Contentless tables can corrupt under specific failure
|
|
31
|
+
// modes (process kill mid-trigger, manual SQL on chunks_fts, etc.).
|
|
32
|
+
// integrity-check returns 'ok' on success; rebuild is the standard fix.
|
|
33
|
+
try {
|
|
34
|
+
const conn = store.conn;
|
|
35
|
+
if (conn) {
|
|
36
|
+
try {
|
|
37
|
+
const row = conn.prepare(`INSERT INTO chunks_fts(chunks_fts) VALUES('integrity-check') RETURNING ''`).get();
|
|
38
|
+
// 'integrity-check' is a no-op insert that throws on failure. If we
|
|
39
|
+
// got a row back, FTS is fine. (Some SQLite builds don't support the
|
|
40
|
+
// RETURNING form on virtual tables — fall back to plain run().)
|
|
41
|
+
void row;
|
|
42
|
+
}
|
|
43
|
+
catch (innerErr) {
|
|
44
|
+
// Try the plain form before declaring failure.
|
|
45
|
+
try {
|
|
46
|
+
conn.prepare(`INSERT INTO chunks_fts(chunks_fts) VALUES('integrity-check')`).run();
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
report.ftsOk = false;
|
|
50
|
+
logger.warn({ err: innerErr }, 'FTS5 integrity check failed — rebuilding');
|
|
51
|
+
try {
|
|
52
|
+
conn.prepare(`INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild')`).run();
|
|
53
|
+
report.ftsRebuilt = true;
|
|
54
|
+
}
|
|
55
|
+
catch (rebuildErr) {
|
|
56
|
+
logger.warn({ err: rebuildErr }, 'FTS5 rebuild failed');
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
logger.warn({ err }, 'FTS integrity probe error');
|
|
64
|
+
}
|
|
65
|
+
// 2. derived_from dangling references. Phase-2 janitor deletes a chunk
|
|
66
|
+
// that was a source for a summary; we keep the summary but the JSON
|
|
67
|
+
// array of source ids may now contain ids that no longer exist. Walk
|
|
68
|
+
// summary chunks, prune missing ids; fully empty array → null.
|
|
69
|
+
try {
|
|
70
|
+
const conn = store.conn;
|
|
71
|
+
if (conn) {
|
|
72
|
+
const summaries = conn.prepare(`SELECT id, derived_from FROM chunks
|
|
73
|
+
WHERE derived_from IS NOT NULL AND derived_from != ''`).all();
|
|
74
|
+
const liveCheck = conn.prepare('SELECT 1 FROM chunks WHERE id = ?');
|
|
75
|
+
const updateStmt = conn.prepare('UPDATE chunks SET derived_from = ? WHERE id = ?');
|
|
76
|
+
for (const s of summaries) {
|
|
77
|
+
let ids;
|
|
78
|
+
try {
|
|
79
|
+
ids = JSON.parse(s.derived_from);
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
if (!Array.isArray(ids))
|
|
85
|
+
continue;
|
|
86
|
+
const live = ids.filter((id) => {
|
|
87
|
+
if (typeof id !== 'number')
|
|
88
|
+
return false;
|
|
89
|
+
return !!liveCheck.get(id);
|
|
90
|
+
});
|
|
91
|
+
if (live.length !== ids.length) {
|
|
92
|
+
updateStmt.run(live.length === 0 ? null : JSON.stringify(live), s.id);
|
|
93
|
+
report.orphanRefsNulled++;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
catch (err) {
|
|
99
|
+
logger.warn({ err }, 'derived_from orphan probe failed');
|
|
100
|
+
}
|
|
101
|
+
// 3. Missing dense embeddings — a counter for the dashboard / next backfill
|
|
102
|
+
// cycle. Doesn't repair (backfill is async + heavy); just surfaces.
|
|
103
|
+
try {
|
|
104
|
+
const conn = store.conn;
|
|
105
|
+
if (conn) {
|
|
106
|
+
const row = conn.prepare(`SELECT COUNT(*) AS c FROM chunks c
|
|
107
|
+
LEFT JOIN chunk_soft_deletes sd ON sd.chunk_id = c.id
|
|
108
|
+
WHERE sd.chunk_id IS NULL
|
|
109
|
+
AND c.embedding_dense IS NULL
|
|
110
|
+
AND length(c.content) > 0`).get();
|
|
111
|
+
report.missingEmbeddings = row.c;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
catch (err) {
|
|
115
|
+
logger.warn({ err }, 'Missing-embedding probe failed');
|
|
116
|
+
}
|
|
117
|
+
return report;
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=integrity.js.map
|
|
@@ -4,9 +4,30 @@
|
|
|
4
4
|
* Runs startup and periodic maintenance so the memory store stays healthy
|
|
5
5
|
* without manual intervention. New users get this out of the box.
|
|
6
6
|
*
|
|
7
|
-
* Startup: decay salience, prune stale data, backfill embeddings
|
|
8
|
-
* Periodic (every 6h): full consolidation cycle + embedding rebuild
|
|
7
|
+
* Startup: decay salience, prune stale data, backfill embeddings, run janitor
|
|
8
|
+
* Periodic (every 6h): full consolidation cycle + embedding rebuild + janitor
|
|
9
|
+
* + idle-gated VACUUM at most once per week
|
|
9
10
|
*/
|
|
11
|
+
/**
|
|
12
|
+
* Janitor pass — keeps the store bounded. Safe to call repeatedly.
|
|
13
|
+
* Idempotent within a single run; surfaces totals for logging.
|
|
14
|
+
*/
|
|
15
|
+
export declare function runJanitor(store: any): {
|
|
16
|
+
softDeleted: number;
|
|
17
|
+
physicallyDeleted: number;
|
|
18
|
+
outcomesPruned: number;
|
|
19
|
+
extractionsCapped: number;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Run VACUUM if (a) it's been more than vacuumIntervalDays since the last
|
|
23
|
+
* one and (b) the store has been idle for at least vacuumIdleSeconds.
|
|
24
|
+
* Returns null when skipped, otherwise the size delta.
|
|
25
|
+
*/
|
|
26
|
+
export declare function maybeVacuum(store: any): {
|
|
27
|
+
sizeBeforeBytes: number;
|
|
28
|
+
sizeAfterBytes: number;
|
|
29
|
+
durationMs: number;
|
|
30
|
+
} | null;
|
|
10
31
|
/**
|
|
11
32
|
* Run one-time maintenance at daemon startup.
|
|
12
33
|
* Non-blocking — errors are logged but never thrown.
|
|
@@ -4,12 +4,82 @@
|
|
|
4
4
|
* Runs startup and periodic maintenance so the memory store stays healthy
|
|
5
5
|
* without manual intervention. New users get this out of the box.
|
|
6
6
|
*
|
|
7
|
-
* Startup: decay salience, prune stale data, backfill embeddings
|
|
8
|
-
* Periodic (every 6h): full consolidation cycle + embedding rebuild
|
|
7
|
+
* Startup: decay salience, prune stale data, backfill embeddings, run janitor
|
|
8
|
+
* Periodic (every 6h): full consolidation cycle + embedding rebuild + janitor
|
|
9
|
+
* + idle-gated VACUUM at most once per week
|
|
9
10
|
*/
|
|
10
11
|
import pino from 'pino';
|
|
12
|
+
import { MEMORY_JANITOR } from '../config.js';
|
|
13
|
+
import { runIntegrityProbes } from './integrity.js';
|
|
11
14
|
const logger = pino({ name: 'clementine.maintenance' });
|
|
12
15
|
const PERIODIC_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours
|
|
16
|
+
const VACUUM_META_KEY = 'last_vacuum_at';
|
|
17
|
+
/**
|
|
18
|
+
* Janitor pass — keeps the store bounded. Safe to call repeatedly.
|
|
19
|
+
* Idempotent within a single run; surfaces totals for logging.
|
|
20
|
+
*/
|
|
21
|
+
export function runJanitor(store) {
|
|
22
|
+
let softDeleted = 0;
|
|
23
|
+
let physicallyDeleted = 0;
|
|
24
|
+
try {
|
|
25
|
+
const result = store.expireConsolidated?.({
|
|
26
|
+
expireDays: MEMORY_JANITOR.consolidatedExpireDays,
|
|
27
|
+
salienceFloor: MEMORY_JANITOR.consolidatedSalienceFloor,
|
|
28
|
+
graceDays: MEMORY_JANITOR.softDeleteGraceDays,
|
|
29
|
+
});
|
|
30
|
+
if (result) {
|
|
31
|
+
softDeleted = result.softDeleted;
|
|
32
|
+
physicallyDeleted = result.physicallyDeleted;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
catch (err) {
|
|
36
|
+
logger.warn({ err }, 'expireConsolidated failed');
|
|
37
|
+
}
|
|
38
|
+
let outcomesPruned = 0;
|
|
39
|
+
try {
|
|
40
|
+
outcomesPruned = store.pruneOutcomes?.(MEMORY_JANITOR.auxRetentionDays) ?? 0;
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
logger.warn({ err }, 'pruneOutcomes failed');
|
|
44
|
+
}
|
|
45
|
+
let extractionsCapped = 0;
|
|
46
|
+
try {
|
|
47
|
+
extractionsCapped = store.capExtractions?.(MEMORY_JANITOR.extractionsMaxRows) ?? 0;
|
|
48
|
+
}
|
|
49
|
+
catch (err) {
|
|
50
|
+
logger.warn({ err }, 'capExtractions failed');
|
|
51
|
+
}
|
|
52
|
+
return { softDeleted, physicallyDeleted, outcomesPruned, extractionsCapped };
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Run VACUUM if (a) it's been more than vacuumIntervalDays since the last
|
|
56
|
+
* one and (b) the store has been idle for at least vacuumIdleSeconds.
|
|
57
|
+
* Returns null when skipped, otherwise the size delta.
|
|
58
|
+
*/
|
|
59
|
+
export function maybeVacuum(store) {
|
|
60
|
+
try {
|
|
61
|
+
const lastIso = store.getMaintenanceMeta?.(VACUUM_META_KEY);
|
|
62
|
+
if (lastIso) {
|
|
63
|
+
const last = new Date(lastIso).getTime();
|
|
64
|
+
const ageMs = Date.now() - last;
|
|
65
|
+
if (ageMs < MEMORY_JANITOR.vacuumIntervalDays * 86_400_000)
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
const lastActivity = store.lastActivityAt?.();
|
|
69
|
+
if (lastActivity !== null && lastActivity !== undefined) {
|
|
70
|
+
const idleMs = Date.now() - lastActivity;
|
|
71
|
+
if (idleMs < MEMORY_JANITOR.vacuumIdleSeconds * 1000)
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
const result = store.vacuum?.();
|
|
75
|
+
store.setMaintenanceMeta?.(VACUUM_META_KEY, new Date().toISOString());
|
|
76
|
+
return result ?? null;
|
|
77
|
+
}
|
|
78
|
+
catch (err) {
|
|
79
|
+
logger.warn({ err }, 'VACUUM failed');
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
13
83
|
/**
|
|
14
84
|
* Run one-time maintenance at daemon startup.
|
|
15
85
|
* Non-blocking — errors are logged but never thrown.
|
|
@@ -56,6 +126,32 @@ export async function runStartupMaintenance(store) {
|
|
|
56
126
|
catch {
|
|
57
127
|
// Table may not exist yet — non-fatal
|
|
58
128
|
}
|
|
129
|
+
// Janitor — bounded growth pass.
|
|
130
|
+
try {
|
|
131
|
+
const result = runJanitor(store);
|
|
132
|
+
if (result.softDeleted || result.physicallyDeleted || result.outcomesPruned || result.extractionsCapped) {
|
|
133
|
+
logger.info(result, 'Janitor pass complete');
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
catch (err) {
|
|
137
|
+
logger.warn({ err }, 'Startup janitor failed');
|
|
138
|
+
}
|
|
139
|
+
// Embedding warm-up — pre-embed the most-cited chunks in the background so
|
|
140
|
+
// the first retrievals after startup don't pay cold-start latency. Fire
|
|
141
|
+
// and forget; never blocks startup.
|
|
142
|
+
if (typeof store.warmDenseEmbeddings === 'function') {
|
|
143
|
+
void (async () => {
|
|
144
|
+
try {
|
|
145
|
+
const result = await store.warmDenseEmbeddings(200);
|
|
146
|
+
if (result.warmed > 0) {
|
|
147
|
+
logger.info(result, 'Embedding warm-up complete');
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
catch (err) {
|
|
151
|
+
logger.warn({ err }, 'Embedding warm-up failed');
|
|
152
|
+
}
|
|
153
|
+
})();
|
|
154
|
+
}
|
|
59
155
|
logger.info({ durationMs: Date.now() - start }, 'Startup maintenance complete');
|
|
60
156
|
}
|
|
61
157
|
/**
|
|
@@ -104,7 +200,7 @@ export function startPeriodicMaintenance(store, llmCall) {
|
|
|
104
200
|
logger.warn({ err }, 'Post-consolidation embedding build failed');
|
|
105
201
|
}
|
|
106
202
|
}
|
|
107
|
-
// 5. Extraction log pruning
|
|
203
|
+
// 5. Extraction log pruning (legacy 90-day rule retained alongside cap)
|
|
108
204
|
try {
|
|
109
205
|
const conn = store.conn;
|
|
110
206
|
if (conn) {
|
|
@@ -114,6 +210,47 @@ export function startPeriodicMaintenance(store, llmCall) {
|
|
|
114
210
|
}
|
|
115
211
|
}
|
|
116
212
|
catch { /* non-fatal */ }
|
|
213
|
+
// 6. Janitor — bounded growth.
|
|
214
|
+
try {
|
|
215
|
+
const result = runJanitor(store);
|
|
216
|
+
if (result.softDeleted || result.physicallyDeleted || result.outcomesPruned || result.extractionsCapped) {
|
|
217
|
+
logger.info(result, 'Janitor pass complete');
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
catch (err) {
|
|
221
|
+
logger.warn({ err }, 'Periodic janitor failed');
|
|
222
|
+
}
|
|
223
|
+
// 6b. Integrity probes — FTS health, orphan derived_from, embedding gaps.
|
|
224
|
+
try {
|
|
225
|
+
const report = runIntegrityProbes(store);
|
|
226
|
+
// Persist for the dashboard so the "last integrity check" surface
|
|
227
|
+
// doesn't depend on log scraping.
|
|
228
|
+
try {
|
|
229
|
+
store.setMaintenanceMeta?.('last_integrity_report', JSON.stringify({ ...report, ranAt: new Date().toISOString() }));
|
|
230
|
+
}
|
|
231
|
+
catch { /* meta write is best-effort */ }
|
|
232
|
+
if (!report.ftsOk || report.ftsRebuilt || report.orphanRefsNulled > 0 || report.missingEmbeddings > 0) {
|
|
233
|
+
logger.info(report, 'Integrity probes complete');
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
catch (err) {
|
|
237
|
+
logger.warn({ err }, 'Integrity probes failed');
|
|
238
|
+
}
|
|
239
|
+
// 7. VACUUM — idle-gated, at most once per vacuumIntervalDays.
|
|
240
|
+
try {
|
|
241
|
+
const vac = maybeVacuum(store);
|
|
242
|
+
if (vac) {
|
|
243
|
+
logger.info({
|
|
244
|
+
sizeBeforeBytes: vac.sizeBeforeBytes,
|
|
245
|
+
sizeAfterBytes: vac.sizeAfterBytes,
|
|
246
|
+
reclaimedBytes: vac.sizeBeforeBytes - vac.sizeAfterBytes,
|
|
247
|
+
durationMs: vac.durationMs,
|
|
248
|
+
}, 'VACUUM complete');
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
catch (err) {
|
|
252
|
+
logger.warn({ err }, 'Periodic VACUUM failed');
|
|
253
|
+
}
|
|
117
254
|
logger.info({ durationMs: Date.now() - start }, 'Periodic maintenance complete');
|
|
118
255
|
};
|
|
119
256
|
return setInterval(runCycle, PERIODIC_INTERVAL_MS);
|
package/dist/memory/store.d.ts
CHANGED
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
* (single-user, one MCP subprocess handles all writes).
|
|
11
11
|
*/
|
|
12
12
|
import type { Feedback, MemoryExtraction, SearchResult, SessionSummary, SyncStats, TranscriptTurn, WikilinkConnection } from '../types.js';
|
|
13
|
+
import { HotCache } from './hot-cache.js';
|
|
14
|
+
import { type WriteQueueOpts } from './write-queue.js';
|
|
13
15
|
export declare class MemoryStore {
|
|
14
16
|
private dbPath;
|
|
15
17
|
private vaultDir;
|
|
@@ -17,6 +19,8 @@ export declare class MemoryStore {
|
|
|
17
19
|
private _stmtChunkCount;
|
|
18
20
|
private _stmtInsertTranscript;
|
|
19
21
|
private _stmtInsertUsage;
|
|
22
|
+
private chunkRowCache;
|
|
23
|
+
private writeQueue;
|
|
20
24
|
constructor(dbPath: string, vaultDir: string);
|
|
21
25
|
/**
|
|
22
26
|
* Create the database and schema if needed.
|
|
@@ -215,6 +219,25 @@ export declare class MemoryStore {
|
|
|
215
219
|
category?: string;
|
|
216
220
|
topic?: string;
|
|
217
221
|
}, strict?: boolean): SearchResult[];
|
|
222
|
+
/**
|
|
223
|
+
* 1-hop wikilink expansion: for each seed chunk's source_file, find files
|
|
224
|
+
* that link to it or that it links to, and pull their top chunks. Returns
|
|
225
|
+
* SearchResult-shaped rows with a fractional boost so they enter the
|
|
226
|
+
* candidate pool below the seed scores but above pure noise.
|
|
227
|
+
*
|
|
228
|
+
* Pattern: 2026-frontier agent memory uses graph expansion (Mem0g, Zep
|
|
229
|
+
* Graphiti) to surface chunks that share an entity but miss the lexical
|
|
230
|
+
* match. Wikilinks are the cheapest available edge — Clementine already
|
|
231
|
+
* extracts them on every vault sync. The richer FalkorDB graph adds
|
|
232
|
+
* temporal validity and entity types but isn't required for this lift.
|
|
233
|
+
*/
|
|
234
|
+
expandViaWikilinks(seeds: SearchResult[], opts?: {
|
|
235
|
+
boost?: number;
|
|
236
|
+
limitPerFile?: number;
|
|
237
|
+
maxNeighbors?: number;
|
|
238
|
+
agentSlug?: string;
|
|
239
|
+
strict?: boolean;
|
|
240
|
+
}): SearchResult[];
|
|
218
241
|
/**
|
|
219
242
|
* Combined FTS5 relevance + recency search for context injection.
|
|
220
243
|
*
|
|
@@ -223,6 +246,7 @@ export declare class MemoryStore {
|
|
|
223
246
|
* 2. Recency fetch -> N most recent chunks
|
|
224
247
|
* 3. Deduplicate by (source_file, section)
|
|
225
248
|
* 4. Apply salience boost to FTS results
|
|
249
|
+
* 5. Wikilink graph expansion -> 1-hop neighbors of top seeds (boost 0.7×)
|
|
226
250
|
*/
|
|
227
251
|
searchContext(query: string, limitOrOpts?: number | {
|
|
228
252
|
limit?: number;
|
|
@@ -252,6 +276,15 @@ export declare class MemoryStore {
|
|
|
252
276
|
scores: number[];
|
|
253
277
|
agentSlug?: string | null;
|
|
254
278
|
}): void;
|
|
279
|
+
/** Internal sync recall_trace insert. Called by the WriteQueue. */
|
|
280
|
+
_logRecallTraceSync(opts: {
|
|
281
|
+
sessionKey: string;
|
|
282
|
+
messageId?: string | null;
|
|
283
|
+
query: string;
|
|
284
|
+
chunkIds: number[];
|
|
285
|
+
scores: number[];
|
|
286
|
+
agentSlug?: string | null;
|
|
287
|
+
}): void;
|
|
255
288
|
/**
|
|
256
289
|
* Fetch recent recall traces for a session, newest first.
|
|
257
290
|
* Used by the dashboard chat panel to show "what memory powered this answer".
|
|
@@ -303,6 +336,25 @@ export declare class MemoryStore {
|
|
|
303
336
|
salience: number;
|
|
304
337
|
updatedAt: string;
|
|
305
338
|
}>;
|
|
339
|
+
/** Cache stats for the dashboard / debugging. */
|
|
340
|
+
getChunkCacheStats(): ReturnType<HotCache<number, unknown>['stats']>;
|
|
341
|
+
/**
|
|
342
|
+
* Enable the write-behind queue. After this call, saveTurn / recordAccess /
|
|
343
|
+
* recordOutcome / logRecallTrace enqueue instead of running SQL on the
|
|
344
|
+
* caller's thread. Idempotent. Tests leave this off and rely on the sync path.
|
|
345
|
+
*/
|
|
346
|
+
enableWriteQueue(opts?: WriteQueueOpts): void;
|
|
347
|
+
/** Drain and stop the write queue. Call on graceful shutdown. */
|
|
348
|
+
flushWrites(): Promise<void>;
|
|
349
|
+
/** Stats for the dashboard / debugging. Returns null when queue disabled. */
|
|
350
|
+
getWriteQueueStats(): {
|
|
351
|
+
size: number;
|
|
352
|
+
dropped: number;
|
|
353
|
+
} | null;
|
|
354
|
+
/** Drop a single cache entry — called from mutations that touch a chunk. */
|
|
355
|
+
invalidateChunkCache(chunkId: number): void;
|
|
356
|
+
/** Drop the whole cache — fullSync and similar bulk operations call this. */
|
|
357
|
+
clearChunkCache(): void;
|
|
306
358
|
private _parseJsonArray;
|
|
307
359
|
/** The fixed slot vocabulary. Adding new slots is a code change so the
|
|
308
360
|
* agent doesn't sprawl into ad-hoc namespaces. */
|
|
@@ -366,6 +418,19 @@ export declare class MemoryStore {
|
|
|
366
418
|
* async, which propagates up the entire searchContext chain.
|
|
367
419
|
*/
|
|
368
420
|
private searchByDenseEmbedding;
|
|
421
|
+
/**
|
|
422
|
+
* Pre-embed the top N most-cited chunks at startup. Eliminates cold-start
|
|
423
|
+
* latency for the chunks the agent is most likely to retrieve next. Skips
|
|
424
|
+
* chunks that already have a current-model dense embedding.
|
|
425
|
+
*
|
|
426
|
+
* Ranking: by outcome citation count in the last 30d (chunks the agent
|
|
427
|
+
* actually used), tiebroken by recency. Soft-deleted excluded.
|
|
428
|
+
*/
|
|
429
|
+
warmDenseEmbeddings(topN?: number): Promise<{
|
|
430
|
+
warmed: number;
|
|
431
|
+
skipped: number;
|
|
432
|
+
failed: number;
|
|
433
|
+
}>;
|
|
369
434
|
/**
|
|
370
435
|
* Backfill dense embeddings on chunks that don't yet have one (or that
|
|
371
436
|
* were embedded by an older model). Async because the dense model itself
|
|
@@ -396,9 +461,12 @@ export declare class MemoryStore {
|
|
|
396
461
|
*/
|
|
397
462
|
getConnections(noteName: string): WikilinkConnection[];
|
|
398
463
|
/**
|
|
399
|
-
* Save a conversation turn to the transcripts table.
|
|
464
|
+
* Save a conversation turn to the transcripts table. Routes through the
|
|
465
|
+
* write queue when enabled so the request thread doesn't block on SQL.
|
|
400
466
|
*/
|
|
401
467
|
saveTurn(sessionKey: string, role: string, content: string, model?: string): void;
|
|
468
|
+
/** Internal sync transcript insert. Called directly by the WriteQueue. */
|
|
469
|
+
_saveTurnSync(sessionKey: string, role: string, content: string, model: string): void;
|
|
402
470
|
/**
|
|
403
471
|
* Get all turns for a given session, ordered chronologically.
|
|
404
472
|
*/
|
|
@@ -426,9 +494,12 @@ export declare class MemoryStore {
|
|
|
426
494
|
*/
|
|
427
495
|
getRecentSummaries(limit?: number): SessionSummary[];
|
|
428
496
|
/**
|
|
429
|
-
* Record that chunks were accessed (retrieved/displayed).
|
|
497
|
+
* Record that chunks were accessed (retrieved/displayed). Routes through
|
|
498
|
+
* the write queue when enabled.
|
|
430
499
|
*/
|
|
431
500
|
recordAccess(chunkIds: number[], accessType?: string): void;
|
|
501
|
+
/** Internal sync access log insert. Called directly by the WriteQueue. */
|
|
502
|
+
_recordAccessSync(chunkIds: number[], accessType: string): void;
|
|
432
503
|
/**
|
|
433
504
|
* Recompute salience score for a chunk based on access patterns.
|
|
434
505
|
*
|
|
@@ -451,6 +522,11 @@ export declare class MemoryStore {
|
|
|
451
522
|
chunkId: number;
|
|
452
523
|
referenced: boolean;
|
|
453
524
|
}>, sessionKey?: string | null): void;
|
|
525
|
+
/** Internal sync outcome insert + EMA update. Called by the WriteQueue. */
|
|
526
|
+
_recordOutcomeSync(outcomes: Array<{
|
|
527
|
+
chunkId: number;
|
|
528
|
+
referenced: boolean;
|
|
529
|
+
}>, sessionKey?: string | null): void;
|
|
454
530
|
/**
|
|
455
531
|
* Idempotent append for a batch of SDK session transcript entries.
|
|
456
532
|
* Entries with a uuid are upserted on (session_id, subpath, uuid);
|
|
@@ -663,6 +739,127 @@ export declare class MemoryStore {
|
|
|
663
739
|
usageLogPruned: number;
|
|
664
740
|
recallTracesPruned: number;
|
|
665
741
|
};
|
|
742
|
+
/**
|
|
743
|
+
* User-model slots whose `updated_at` is older than maxAgeDays. These are
|
|
744
|
+
* candidates for the "verify or refresh" nudge — high-relevance memories
|
|
745
|
+
* that may have become silently wrong (Mem0 2026 calls this out as an
|
|
746
|
+
* open problem; we surface it via observability rather than auto-decay).
|
|
747
|
+
*
|
|
748
|
+
* Empty content is skipped (an empty slot has no claim to verify).
|
|
749
|
+
*/
|
|
750
|
+
findStaleUserModelSlots(opts?: {
|
|
751
|
+
maxAgeDays?: number;
|
|
752
|
+
agentSlug?: string | null;
|
|
753
|
+
}): Array<{
|
|
754
|
+
slot: string;
|
|
755
|
+
ageDays: number;
|
|
756
|
+
agentSlug: string | null;
|
|
757
|
+
}>;
|
|
758
|
+
/**
|
|
759
|
+
* High-salience chunks whose outcome EMA has drifted negative — i.e., we
|
|
760
|
+
* keep ranking them high but the agent stopped citing them. Strong signal
|
|
761
|
+
* that the chunk is stale or wrong even though salience hasn't decayed.
|
|
762
|
+
*
|
|
763
|
+
* Conservative threshold: salience > 0.8 AND last_outcome_score < 0.
|
|
764
|
+
* Soft-deleted excluded.
|
|
765
|
+
*/
|
|
766
|
+
findStaleHighSalienceChunks(opts?: {
|
|
767
|
+
salienceFloor?: number;
|
|
768
|
+
outcomeCeiling?: number;
|
|
769
|
+
limit?: number;
|
|
770
|
+
}): Array<{
|
|
771
|
+
chunkId: number;
|
|
772
|
+
sourceFile: string;
|
|
773
|
+
section: string;
|
|
774
|
+
salience: number;
|
|
775
|
+
lastOutcomeScore: number;
|
|
776
|
+
}>;
|
|
777
|
+
/**
|
|
778
|
+
* Format staleness findings into ready-to-inject prompt text. Heartbeat
|
|
779
|
+
* builders can drop this into the system prompt verbatim. Returns null
|
|
780
|
+
* if there's nothing to nudge about — caller should not inject empty text.
|
|
781
|
+
*/
|
|
782
|
+
getStalenessNudges(opts?: {
|
|
783
|
+
agentSlug?: string | null;
|
|
784
|
+
maxSlotAgeDays?: number;
|
|
785
|
+
}): string | null;
|
|
786
|
+
/**
|
|
787
|
+
* Find procedure chunks whose frontmatter `triggers` overlap with words
|
|
788
|
+
* in the query. Used to surface learned workflows ("how Nate ships a
|
|
789
|
+
* release", "how to handle inbound replies") above generic facts when
|
|
790
|
+
* the user's intent matches.
|
|
791
|
+
*
|
|
792
|
+
* Match rule: case-insensitive substring of any trigger phrase appears
|
|
793
|
+
* in the query. Empty result if no procedure chunks exist or no triggers
|
|
794
|
+
* match — caller should treat this as additive context, not the whole
|
|
795
|
+
* answer.
|
|
796
|
+
*/
|
|
797
|
+
findRelevantProcedures(query: string, opts?: {
|
|
798
|
+
limit?: number;
|
|
799
|
+
agentSlug?: string | null;
|
|
800
|
+
}): Array<{
|
|
801
|
+
id: number;
|
|
802
|
+
sourceFile: string;
|
|
803
|
+
section: string;
|
|
804
|
+
content: string;
|
|
805
|
+
triggers: string[];
|
|
806
|
+
matched: string[];
|
|
807
|
+
}>;
|
|
808
|
+
/** Persistent key/value for janitor state (last vacuum, etc.). */
|
|
809
|
+
getMaintenanceMeta(key: string): string | null;
|
|
810
|
+
setMaintenanceMeta(key: string, value: string): void;
|
|
811
|
+
/**
|
|
812
|
+
* Two-phase delete for consolidated, low-salience, unused chunks.
|
|
813
|
+
*
|
|
814
|
+
* Phase 1: soft-delete chunks where consolidated=1, not pinned, salience
|
|
815
|
+
* below floor, and never accessed (or last access older than
|
|
816
|
+
* expireDays).
|
|
817
|
+
* Phase 2: physically delete chunks that have been in chunk_soft_deletes
|
|
818
|
+
* for graceDays. Cascades to access_log, outcomes, chunk_history
|
|
819
|
+
* for the same chunk_id.
|
|
820
|
+
*
|
|
821
|
+
* Summary chunks whose `derived_from` references the deleted IDs are
|
|
822
|
+
* intentionally NOT propagate-deleted — the summary still encodes signal.
|
|
823
|
+
*/
|
|
824
|
+
expireConsolidated(opts?: {
|
|
825
|
+
expireDays?: number;
|
|
826
|
+
salienceFloor?: number;
|
|
827
|
+
graceDays?: number;
|
|
828
|
+
}): {
|
|
829
|
+
softDeleted: number;
|
|
830
|
+
physicallyDeleted: number;
|
|
831
|
+
};
|
|
832
|
+
/** Trim outcomes table to a rolling window. Append-only, can grow fast. */
|
|
833
|
+
pruneOutcomes(retentionDays?: number): number;
|
|
834
|
+
/**
|
|
835
|
+
* Cap memory_extractions to maxRows. Deletes oldest non-active rows first;
|
|
836
|
+
* 'active' extractions are preserved regardless of count to protect the
|
|
837
|
+
* audit trail for in-flight work.
|
|
838
|
+
*/
|
|
839
|
+
capExtractions(maxRows?: number): number;
|
|
840
|
+
/** Approximate SQLite database file size on disk, in bytes. */
|
|
841
|
+
dbSizeBytes(): number;
|
|
842
|
+
/**
|
|
843
|
+
* VACUUM the database. Reclaims space from deleted rows. Holds an
|
|
844
|
+
* exclusive lock for the duration — caller is expected to gate on
|
|
845
|
+
* idleness (see lastActivityAt).
|
|
846
|
+
*/
|
|
847
|
+
vacuum(): {
|
|
848
|
+
sizeBeforeBytes: number;
|
|
849
|
+
sizeAfterBytes: number;
|
|
850
|
+
durationMs: number;
|
|
851
|
+
};
|
|
852
|
+
/**
|
|
853
|
+
* Most recent timestamp across the high-write activity tables, as a Unix
|
|
854
|
+
* milliseconds value. Returns null if all tables are empty. Used by the
|
|
855
|
+
* janitor's idle gate.
|
|
856
|
+
*
|
|
857
|
+
* Implementation note: SQLite's datetime() returns "YYYY-MM-DD HH:MM:SS"
|
|
858
|
+
* in UTC with no timezone marker — JS Date.parse interprets that as local
|
|
859
|
+
* time and skews by the offset. We compute the unix epoch in SQL to avoid
|
|
860
|
+
* the bug entirely.
|
|
861
|
+
*/
|
|
862
|
+
lastActivityAt(): number | null;
|
|
666
863
|
/**
|
|
667
864
|
* Get chunks within a date range, ordered chronologically.
|
|
668
865
|
* Useful for "what happened last week" type queries.
|
|
@@ -904,6 +1101,66 @@ export declare class MemoryStore {
|
|
|
904
1101
|
* Reduces salience so they appear lower in search results (but aren't deleted).
|
|
905
1102
|
*/
|
|
906
1103
|
markConsolidated(chunkIds: number[]): void;
|
|
1104
|
+
/**
|
|
1105
|
+
* Aggregate memory-health snapshot for the dashboard.
|
|
1106
|
+
*
|
|
1107
|
+
* Single-pass queries over each table; cheap enough to call on every
|
|
1108
|
+
* dashboard tab visit without caching. Adds graph stats only if a
|
|
1109
|
+
* graphStore is supplied and reachable.
|
|
1110
|
+
*/
|
|
1111
|
+
getMemoryHealth(opts?: {
|
|
1112
|
+
graphStore?: {
|
|
1113
|
+
isAvailable(): boolean;
|
|
1114
|
+
nodeCount?(): Promise<number>;
|
|
1115
|
+
edgeCount?(): Promise<number>;
|
|
1116
|
+
};
|
|
1117
|
+
topCitedLimit?: number;
|
|
1118
|
+
}): {
|
|
1119
|
+
chunks: {
|
|
1120
|
+
total: number;
|
|
1121
|
+
consolidated: number;
|
|
1122
|
+
pinned: number;
|
|
1123
|
+
softDeleted: number;
|
|
1124
|
+
zombieCount: number;
|
|
1125
|
+
};
|
|
1126
|
+
chunksByCategory: Array<{
|
|
1127
|
+
category: string | null;
|
|
1128
|
+
count: number;
|
|
1129
|
+
}>;
|
|
1130
|
+
tableRowCounts: Record<string, number>;
|
|
1131
|
+
topCitedLast30d: Array<{
|
|
1132
|
+
chunkId: number;
|
|
1133
|
+
sourceFile: string;
|
|
1134
|
+
section: string;
|
|
1135
|
+
refCount: number;
|
|
1136
|
+
}>;
|
|
1137
|
+
staleUserModelSlots: Array<{
|
|
1138
|
+
slot: string;
|
|
1139
|
+
ageDays: number;
|
|
1140
|
+
agentSlug: string | null;
|
|
1141
|
+
}>;
|
|
1142
|
+
staleHighSalienceChunks: Array<{
|
|
1143
|
+
chunkId: number;
|
|
1144
|
+
sourceFile: string;
|
|
1145
|
+
section: string;
|
|
1146
|
+
salience: number;
|
|
1147
|
+
lastOutcomeScore: number;
|
|
1148
|
+
}>;
|
|
1149
|
+
chunkCacheStats: ReturnType<HotCache<number, unknown>['stats']>;
|
|
1150
|
+
writeQueue: {
|
|
1151
|
+
size: number;
|
|
1152
|
+
dropped: number;
|
|
1153
|
+
} | null;
|
|
1154
|
+
lastIntegrityReport: {
|
|
1155
|
+
ftsOk: boolean;
|
|
1156
|
+
ftsRebuilt: boolean;
|
|
1157
|
+
orphanRefsNulled: number;
|
|
1158
|
+
missingEmbeddings: number;
|
|
1159
|
+
ranAt: string;
|
|
1160
|
+
} | null;
|
|
1161
|
+
dbSizeBytes: number;
|
|
1162
|
+
lastVacuumAt: string | null;
|
|
1163
|
+
};
|
|
907
1164
|
/**
|
|
908
1165
|
* Get consolidation stats for monitoring.
|
|
909
1166
|
*/
|