clawmem 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +13 -3
- package/CLAUDE.md +13 -3
- package/README.md +31 -2
- package/SKILL.md +11 -3
- package/package.json +1 -1
- package/src/clawmem.ts +30 -2
- package/src/consolidation.ts +146 -16
- package/src/conversation-synthesis.ts +637 -0
- package/src/maintenance.ts +540 -0
- package/src/mcp.ts +34 -0
- package/src/store.ts +35 -0
- package/src/worker-lease.ts +141 -0
|
@@ -0,0 +1,540 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ClawMem Heavy Maintenance Lane (v0.8.0 Ext 5)
|
|
3
|
+
*
|
|
4
|
+
* A second, longer-interval consolidation worker that runs during configured
|
|
5
|
+
* quiet windows with stale-first batching, DB-backed exclusivity via
|
|
6
|
+
* worker_leases, and journal rows in `maintenance_runs` for every attempt.
|
|
7
|
+
*
|
|
8
|
+
* Keeps Phase 2/3 consolidation + deductive synthesis running on large vaults
|
|
9
|
+
* without competing for CPU/GPU against the interactive light lane that
|
|
10
|
+
* ticks every 5 minutes. Off by default — enabled via `CLAWMEM_HEAVY_LANE=true`
|
|
11
|
+
* next to the existing light-lane `CLAWMEM_ENABLE_CONSOLIDATION` flag.
|
|
12
|
+
*
|
|
13
|
+
* Design notes:
|
|
14
|
+
* - Uses existing `context_usage` telemetry for query-rate gating. No new
|
|
15
|
+
* `query_activity` table — we count rows where `timestamp > -10 minutes`.
|
|
16
|
+
* - Stale-first selection prefers docs whose `recall_stats.last_recalled_at`
|
|
17
|
+
* is oldest/null, falling back to `documents.last_accessed_at` / `modified_at`.
|
|
18
|
+
* - Optional surprisal selector reuses `computeSurprisalScores` to bubble up
|
|
19
|
+
* high-anomaly observations for curator-style runs.
|
|
20
|
+
* - Every scheduled attempt writes a `maintenance_runs` row so operators can
|
|
21
|
+
* reconstruct the decision without reading worker logs.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import type { Store } from "./store.ts";
|
|
25
|
+
import type { LlamaCpp } from "./llm.ts";
|
|
26
|
+
import {
|
|
27
|
+
consolidateObservations,
|
|
28
|
+
generateDeductiveObservations,
|
|
29
|
+
computeSurprisalScores,
|
|
30
|
+
type DeductiveSynthesisStats,
|
|
31
|
+
} from "./consolidation.ts";
|
|
32
|
+
import { withWorkerLease } from "./worker-lease.ts";
|
|
33
|
+
|
|
34
|
+
// =============================================================================
|
|
35
|
+
// Config
|
|
36
|
+
// =============================================================================
|
|
37
|
+
|
|
38
|
+
export interface HeavyMaintenanceConfig {
|
|
39
|
+
/** Interval between heavy-lane ticks in milliseconds (default 30 min). */
|
|
40
|
+
intervalMs?: number;
|
|
41
|
+
/** Start hour (0-23) of the quiet window; null/undefined = no window. */
|
|
42
|
+
windowStartHour?: number | null;
|
|
43
|
+
/** End hour (0-23, exclusive) of the quiet window; null/undefined = no window. */
|
|
44
|
+
windowEndHour?: number | null;
|
|
45
|
+
/** Max context_usage rows in the last 10 min before the lane skips (default 30). */
|
|
46
|
+
maxContextUsagesPer10m?: number;
|
|
47
|
+
/** Batch size for Phase 2 consolidation (default 100). */
|
|
48
|
+
staleObservationLimit?: number;
|
|
49
|
+
/** Batch size for Phase 3 deductive synthesis (default 40). */
|
|
50
|
+
staleDeductiveLimit?: number;
|
|
51
|
+
/** When true, use computeSurprisalScores to select batches for Phase 2. */
|
|
52
|
+
useSurprisalSelector?: boolean;
|
|
53
|
+
/** Worker lease TTL in ms (default 10 min — covers worst-case run time). */
|
|
54
|
+
leaseTtlMs?: number;
|
|
55
|
+
/** Worker lease name. Override only in tests (default "heavy-maintenance"). */
|
|
56
|
+
workerName?: string;
|
|
57
|
+
/** Clock injection for unit tests — defaults to `() => new Date()`. */
|
|
58
|
+
clock?: () => Date;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Vault scoping note: the heavy lane operates on whatever Store it is handed
|
|
62
|
+
// — createStore(path) maps 1:1 to a single SQLite vault, so context_usage
|
|
63
|
+
// and recall_stats reads are implicitly vault-scoped via `store.db`. Multi-
|
|
64
|
+
// vault mode is out of scope for v0.8.0 and would require extending this
|
|
65
|
+
// config with an explicit vault list plus a per-vault lease name.
|
|
66
|
+
|
|
67
|
+
const DEFAULT_CONFIG: Required<Omit<HeavyMaintenanceConfig, "workerName" | "clock">> = {
|
|
68
|
+
intervalMs: 30 * 60 * 1000,
|
|
69
|
+
windowStartHour: null,
|
|
70
|
+
windowEndHour: null,
|
|
71
|
+
maxContextUsagesPer10m: 30,
|
|
72
|
+
staleObservationLimit: 100,
|
|
73
|
+
staleDeductiveLimit: 40,
|
|
74
|
+
useSurprisalSelector: false,
|
|
75
|
+
leaseTtlMs: 10 * 60 * 1000,
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const DEFAULT_WORKER_NAME = "heavy-maintenance";
|
|
79
|
+
|
|
80
|
+
// =============================================================================
|
|
81
|
+
// Journal helpers
|
|
82
|
+
// =============================================================================
|
|
83
|
+
|
|
84
|
+
export type MaintenanceStatus = "started" | "skipped" | "completed" | "failed";
|
|
85
|
+
|
|
86
|
+
export interface MaintenanceRunSummary {
|
|
87
|
+
id: number;
|
|
88
|
+
lane: string;
|
|
89
|
+
phase: string;
|
|
90
|
+
status: MaintenanceStatus;
|
|
91
|
+
reason: string | null;
|
|
92
|
+
selected_count: number;
|
|
93
|
+
processed_count: number;
|
|
94
|
+
created_count: number;
|
|
95
|
+
updated_count: number;
|
|
96
|
+
rejected_count: number;
|
|
97
|
+
null_call_count: number;
|
|
98
|
+
started_at: string;
|
|
99
|
+
finished_at: string | null;
|
|
100
|
+
metrics_json: string | null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export function insertMaintenanceRun(
|
|
104
|
+
store: Store,
|
|
105
|
+
row: {
|
|
106
|
+
lane: string;
|
|
107
|
+
phase: string;
|
|
108
|
+
status: MaintenanceStatus;
|
|
109
|
+
reason?: string | null;
|
|
110
|
+
selectedCount?: number;
|
|
111
|
+
processedCount?: number;
|
|
112
|
+
createdCount?: number;
|
|
113
|
+
updatedCount?: number;
|
|
114
|
+
rejectedCount?: number;
|
|
115
|
+
nullCallCount?: number;
|
|
116
|
+
startedAt?: string;
|
|
117
|
+
finishedAt?: string | null;
|
|
118
|
+
metrics?: Record<string, unknown> | null;
|
|
119
|
+
},
|
|
120
|
+
): number {
|
|
121
|
+
const startedAt = row.startedAt ?? new Date().toISOString();
|
|
122
|
+
const result = store.db.prepare(
|
|
123
|
+
`INSERT INTO maintenance_runs
|
|
124
|
+
(lane, phase, status, reason, selected_count, processed_count,
|
|
125
|
+
created_count, updated_count, rejected_count, null_call_count,
|
|
126
|
+
started_at, finished_at, metrics_json)
|
|
127
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
128
|
+
).run(
|
|
129
|
+
row.lane,
|
|
130
|
+
row.phase,
|
|
131
|
+
row.status,
|
|
132
|
+
row.reason ?? null,
|
|
133
|
+
row.selectedCount ?? 0,
|
|
134
|
+
row.processedCount ?? 0,
|
|
135
|
+
row.createdCount ?? 0,
|
|
136
|
+
row.updatedCount ?? 0,
|
|
137
|
+
row.rejectedCount ?? 0,
|
|
138
|
+
row.nullCallCount ?? 0,
|
|
139
|
+
startedAt,
|
|
140
|
+
row.finishedAt ?? null,
|
|
141
|
+
row.metrics ? JSON.stringify(row.metrics) : null,
|
|
142
|
+
);
|
|
143
|
+
return Number(result.lastInsertRowid);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function finalizeMaintenanceRun(
|
|
147
|
+
store: Store,
|
|
148
|
+
id: number,
|
|
149
|
+
patch: {
|
|
150
|
+
status: MaintenanceStatus;
|
|
151
|
+
reason?: string | null;
|
|
152
|
+
selectedCount?: number;
|
|
153
|
+
processedCount?: number;
|
|
154
|
+
createdCount?: number;
|
|
155
|
+
updatedCount?: number;
|
|
156
|
+
rejectedCount?: number;
|
|
157
|
+
nullCallCount?: number;
|
|
158
|
+
finishedAt?: string;
|
|
159
|
+
metrics?: Record<string, unknown> | null;
|
|
160
|
+
},
|
|
161
|
+
): void {
|
|
162
|
+
store.db.prepare(
|
|
163
|
+
`UPDATE maintenance_runs
|
|
164
|
+
SET status = ?, reason = ?,
|
|
165
|
+
selected_count = ?, processed_count = ?,
|
|
166
|
+
created_count = ?, updated_count = ?,
|
|
167
|
+
rejected_count = ?, null_call_count = ?,
|
|
168
|
+
finished_at = ?, metrics_json = ?
|
|
169
|
+
WHERE id = ?`,
|
|
170
|
+
).run(
|
|
171
|
+
patch.status,
|
|
172
|
+
patch.reason ?? null,
|
|
173
|
+
patch.selectedCount ?? 0,
|
|
174
|
+
patch.processedCount ?? 0,
|
|
175
|
+
patch.createdCount ?? 0,
|
|
176
|
+
patch.updatedCount ?? 0,
|
|
177
|
+
patch.rejectedCount ?? 0,
|
|
178
|
+
patch.nullCallCount ?? 0,
|
|
179
|
+
patch.finishedAt ?? new Date().toISOString(),
|
|
180
|
+
patch.metrics ? JSON.stringify(patch.metrics) : null,
|
|
181
|
+
id,
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// =============================================================================
|
|
186
|
+
// Gating logic
|
|
187
|
+
// =============================================================================
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* True when `now` falls inside [windowStartHour, windowEndHour). Both nulls
|
|
191
|
+
* mean "always in window". Handles midnight wraparound (e.g., 22→6) by
|
|
192
|
+
* accepting either hour >= start OR hour < end.
|
|
193
|
+
*/
|
|
194
|
+
export function isInQuietWindow(
|
|
195
|
+
now: Date,
|
|
196
|
+
windowStartHour: number | null | undefined,
|
|
197
|
+
windowEndHour: number | null | undefined,
|
|
198
|
+
): boolean {
|
|
199
|
+
if (windowStartHour == null || windowEndHour == null) return true;
|
|
200
|
+
if (windowStartHour < 0 || windowStartHour > 23 || windowEndHour < 0 || windowEndHour > 23) {
|
|
201
|
+
throw new Error(
|
|
202
|
+
`isInQuietWindow: hours must be 0-23, got start=${windowStartHour} end=${windowEndHour}`,
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
if (windowStartHour === windowEndHour) return false; // empty window
|
|
206
|
+
const hour = now.getHours();
|
|
207
|
+
if (windowStartHour < windowEndHour) {
|
|
208
|
+
return hour >= windowStartHour && hour < windowEndHour;
|
|
209
|
+
}
|
|
210
|
+
// Wraps midnight
|
|
211
|
+
return hour >= windowStartHour || hour < windowEndHour;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Count context_usage rows in the last `minutes` minutes. Used as a proxy
|
|
216
|
+
* for "how busy is the interactive light lane right now" — replaces the
|
|
217
|
+
* `query_activity` table that Turn 2 of Ext 5 proposed.
|
|
218
|
+
*
|
|
219
|
+
* The cutoff is computed in JS as an ISO 8601 string and bound as a
|
|
220
|
+
* parameter instead of using `datetime('now', '-N minutes')`. SQLite's
|
|
221
|
+
* `datetime()` returns a space-separated format (`YYYY-MM-DD HH:MM:SS`)
|
|
222
|
+
* while `context_usage.timestamp` is written in ISO 8601 with a T
|
|
223
|
+
* separator; lexicographic comparison across those two formats is wrong
|
|
224
|
+
* (space < T sorts ALL ISO rows as "newer" than any datetime() result).
|
|
225
|
+
*/
|
|
226
|
+
export function countRecentContextUsages(
|
|
227
|
+
store: Store,
|
|
228
|
+
minutes: number = 10,
|
|
229
|
+
): number {
|
|
230
|
+
const cutoff = new Date(Date.now() - minutes * 60 * 1000).toISOString();
|
|
231
|
+
const row = store.db.prepare(
|
|
232
|
+
`SELECT COUNT(*) AS cnt FROM context_usage WHERE timestamp > ?`,
|
|
233
|
+
).get(cutoff) as { cnt: number } | undefined;
|
|
234
|
+
return row?.cnt ?? 0;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Decide whether the heavy lane should run on this tick. Returns the
|
|
239
|
+
* reason for skipping so the journal row can record it.
|
|
240
|
+
*/
|
|
241
|
+
export function shouldRunHeavyMaintenance(
|
|
242
|
+
store: Store,
|
|
243
|
+
now: Date,
|
|
244
|
+
cfg: HeavyMaintenanceConfig = {},
|
|
245
|
+
): { run: boolean; reason?: string } {
|
|
246
|
+
const merged = { ...DEFAULT_CONFIG, ...cfg };
|
|
247
|
+
if (!isInQuietWindow(now, merged.windowStartHour, merged.windowEndHour)) {
|
|
248
|
+
return { run: false, reason: "outside_window" };
|
|
249
|
+
}
|
|
250
|
+
const usages = countRecentContextUsages(store, 10);
|
|
251
|
+
if (usages > merged.maxContextUsagesPer10m) {
|
|
252
|
+
return { run: false, reason: "query_rate_high" };
|
|
253
|
+
}
|
|
254
|
+
return { run: true };
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// =============================================================================
|
|
258
|
+
// Stale-first selection helpers
|
|
259
|
+
// =============================================================================
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Select up to `limit` observation doc IDs ordered by stale-first:
|
|
263
|
+
* least-recently-recalled (recall_stats.last_recalled_at ASC, NULL first)
|
|
264
|
+
* with documents.last_accessed_at as a fallback when recall_stats is empty.
|
|
265
|
+
*
|
|
266
|
+
* Used by tests and operators who want to inspect the heavy-lane batch
|
|
267
|
+
* without actually running Phase 2. The real Phase 2 SQL inside
|
|
268
|
+
* `consolidateObservations` applies its own stale-first ordering when
|
|
269
|
+
* `staleOnly: true` is passed.
|
|
270
|
+
*/
|
|
271
|
+
export function selectStaleObservationBatch(
|
|
272
|
+
store: Store,
|
|
273
|
+
limit: number,
|
|
274
|
+
): number[] {
|
|
275
|
+
const rows = store.db.prepare(
|
|
276
|
+
`SELECT d.id FROM documents d
|
|
277
|
+
LEFT JOIN recall_stats rs ON rs.doc_id = d.id
|
|
278
|
+
WHERE d.active = 1
|
|
279
|
+
AND d.content_type = 'observation'
|
|
280
|
+
ORDER BY
|
|
281
|
+
COALESCE(rs.last_recalled_at, d.last_accessed_at, d.modified_at) ASC,
|
|
282
|
+
d.modified_at ASC
|
|
283
|
+
LIMIT ?`,
|
|
284
|
+
).all(limit) as { id: number }[];
|
|
285
|
+
return rows.map(r => r.id);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Select up to `limit` decision/preference/milestone/problem doc IDs
|
|
290
|
+
* ordered by stale-first for Phase 3 deductive synthesis.
|
|
291
|
+
*/
|
|
292
|
+
export function selectStaleDeductiveBatch(
|
|
293
|
+
store: Store,
|
|
294
|
+
limit: number,
|
|
295
|
+
): number[] {
|
|
296
|
+
const DEDUCTIVE_TYPES = ["decision", "preference", "milestone", "problem"];
|
|
297
|
+
const placeholders = DEDUCTIVE_TYPES.map(() => "?").join(",");
|
|
298
|
+
const rows = store.db.prepare(
|
|
299
|
+
`SELECT d.id FROM documents d
|
|
300
|
+
LEFT JOIN recall_stats rs ON rs.doc_id = d.id
|
|
301
|
+
WHERE d.active = 1
|
|
302
|
+
AND d.content_type IN (${placeholders})
|
|
303
|
+
ORDER BY
|
|
304
|
+
COALESCE(rs.last_recalled_at, d.last_accessed_at, d.modified_at) ASC,
|
|
305
|
+
d.modified_at ASC
|
|
306
|
+
LIMIT ?`,
|
|
307
|
+
).all(...DEDUCTIVE_TYPES, limit) as { id: number }[];
|
|
308
|
+
return rows.map(r => r.id);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Select up to `limit` observation doc IDs ranked by surprisal score
|
|
313
|
+
* (k-NN average neighbor distance — higher = more anomalous). Wraps
|
|
314
|
+
* `computeSurprisalScores` so the heavy lane can swap in anomaly-first
|
|
315
|
+
* selection via `useSurprisalSelector: true`.
|
|
316
|
+
*/
|
|
317
|
+
export function selectSurprisingObservationBatch(
|
|
318
|
+
store: Store,
|
|
319
|
+
limit: number,
|
|
320
|
+
): number[] {
|
|
321
|
+
const results = computeSurprisalScores(store, { limit });
|
|
322
|
+
return results.map(r => r.docId);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
// =============================================================================
|
|
326
|
+
// Worker topology
|
|
327
|
+
// =============================================================================
|
|
328
|
+
|
|
329
|
+
let heavyTimer: Timer | null = null;
|
|
330
|
+
let heavyRunning = false;
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Run a single heavy-lane tick: gate check → worker lease → Phase 2 → Phase 3
|
|
334
|
+
* → journal row. Exported for tests and for manual invocation via a future
|
|
335
|
+
* `clawmem heavy-lane --once` CLI flag.
|
|
336
|
+
*/
|
|
337
|
+
export async function runHeavyMaintenanceTick(
|
|
338
|
+
store: Store,
|
|
339
|
+
llm: LlamaCpp,
|
|
340
|
+
cfg: HeavyMaintenanceConfig = {},
|
|
341
|
+
): Promise<MaintenanceRunSummary[]> {
|
|
342
|
+
const merged = { ...DEFAULT_CONFIG, ...cfg };
|
|
343
|
+
const workerName = cfg.workerName ?? DEFAULT_WORKER_NAME;
|
|
344
|
+
const clock = cfg.clock ?? (() => new Date());
|
|
345
|
+
const results: MaintenanceRunSummary[] = [];
|
|
346
|
+
|
|
347
|
+
const now = clock();
|
|
348
|
+
const gate = shouldRunHeavyMaintenance(store, now, cfg);
|
|
349
|
+
if (!gate.run) {
|
|
350
|
+
const skippedId = insertMaintenanceRun(store, {
|
|
351
|
+
lane: "heavy",
|
|
352
|
+
phase: "gate",
|
|
353
|
+
status: "skipped",
|
|
354
|
+
reason: gate.reason ?? "unknown",
|
|
355
|
+
finishedAt: new Date().toISOString(),
|
|
356
|
+
});
|
|
357
|
+
results.push(loadMaintenanceRun(store, skippedId));
|
|
358
|
+
return results;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
const lease = await withWorkerLease(
|
|
362
|
+
store,
|
|
363
|
+
workerName,
|
|
364
|
+
merged.leaseTtlMs,
|
|
365
|
+
async () => {
|
|
366
|
+
// Phase 2 — consolidation
|
|
367
|
+
const phase2Id = insertMaintenanceRun(store, {
|
|
368
|
+
lane: "heavy",
|
|
369
|
+
phase: "consolidate",
|
|
370
|
+
status: "started",
|
|
371
|
+
});
|
|
372
|
+
try {
|
|
373
|
+
// Surprisal selector path: compute anomaly-first candidate ids up
|
|
374
|
+
// front, then plumb them into consolidateObservations via
|
|
375
|
+
// candidateIds. When the surprisal backend returns empty (no
|
|
376
|
+
// embeddings, small vault, k-NN unavailable), fall through to
|
|
377
|
+
// stale-first ordering so the heavy lane still does useful work.
|
|
378
|
+
let candidateIds: number[] | undefined;
|
|
379
|
+
let selectorUsed: "stale-first" | "surprisal" | "surprisal-fallback-stale";
|
|
380
|
+
if (merged.useSurprisalSelector) {
|
|
381
|
+
candidateIds = selectSurprisingObservationBatch(
|
|
382
|
+
store,
|
|
383
|
+
merged.staleObservationLimit,
|
|
384
|
+
);
|
|
385
|
+
if (candidateIds.length === 0) {
|
|
386
|
+
// Nothing surprising — degrade to stale-first so the lane
|
|
387
|
+
// does not become a no-op on vaults without embeddings.
|
|
388
|
+
candidateIds = undefined;
|
|
389
|
+
selectorUsed = "surprisal-fallback-stale";
|
|
390
|
+
} else {
|
|
391
|
+
selectorUsed = "surprisal";
|
|
392
|
+
}
|
|
393
|
+
} else {
|
|
394
|
+
selectorUsed = "stale-first";
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
await consolidateObservations(store, llm, {
|
|
398
|
+
maxDocs: merged.staleObservationLimit,
|
|
399
|
+
guarded: true,
|
|
400
|
+
staleOnly: selectorUsed !== "surprisal",
|
|
401
|
+
candidateIds,
|
|
402
|
+
});
|
|
403
|
+
finalizeMaintenanceRun(store, phase2Id, {
|
|
404
|
+
status: "completed",
|
|
405
|
+
selectedCount: candidateIds
|
|
406
|
+
? candidateIds.length
|
|
407
|
+
: merged.staleObservationLimit,
|
|
408
|
+
metrics: {
|
|
409
|
+
selector: selectorUsed,
|
|
410
|
+
...(candidateIds ? { candidateCount: candidateIds.length } : {}),
|
|
411
|
+
},
|
|
412
|
+
});
|
|
413
|
+
} catch (err) {
|
|
414
|
+
finalizeMaintenanceRun(store, phase2Id, {
|
|
415
|
+
status: "failed",
|
|
416
|
+
reason: "phase2_exception",
|
|
417
|
+
metrics: { error: (err as Error).message },
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
results.push(loadMaintenanceRun(store, phase2Id));
|
|
421
|
+
|
|
422
|
+
// Phase 3 — deductive synthesis
|
|
423
|
+
const phase3Id = insertMaintenanceRun(store, {
|
|
424
|
+
lane: "heavy",
|
|
425
|
+
phase: "deductive",
|
|
426
|
+
status: "started",
|
|
427
|
+
});
|
|
428
|
+
try {
|
|
429
|
+
const stats: DeductiveSynthesisStats = await generateDeductiveObservations(
|
|
430
|
+
store,
|
|
431
|
+
llm,
|
|
432
|
+
{
|
|
433
|
+
maxRecent: merged.staleDeductiveLimit,
|
|
434
|
+
guarded: true,
|
|
435
|
+
staleOnly: true,
|
|
436
|
+
},
|
|
437
|
+
);
|
|
438
|
+
finalizeMaintenanceRun(store, phase3Id, {
|
|
439
|
+
status: "completed",
|
|
440
|
+
selectedCount: stats.considered,
|
|
441
|
+
processedCount: stats.drafted,
|
|
442
|
+
createdCount: stats.created,
|
|
443
|
+
rejectedCount: stats.rejected,
|
|
444
|
+
nullCallCount: stats.nullCalls,
|
|
445
|
+
metrics: {
|
|
446
|
+
accepted: stats.accepted,
|
|
447
|
+
contaminationRejects: stats.contaminationRejects,
|
|
448
|
+
invalidIndexRejects: stats.invalidIndexRejects,
|
|
449
|
+
unsupportedRejects: stats.unsupportedRejects,
|
|
450
|
+
emptyRejects: stats.emptyRejects,
|
|
451
|
+
dedupSkipped: stats.dedupSkipped,
|
|
452
|
+
validatorFallbackAccepts: stats.validatorFallbackAccepts,
|
|
453
|
+
},
|
|
454
|
+
});
|
|
455
|
+
} catch (err) {
|
|
456
|
+
finalizeMaintenanceRun(store, phase3Id, {
|
|
457
|
+
status: "failed",
|
|
458
|
+
reason: "phase3_exception",
|
|
459
|
+
metrics: { error: (err as Error).message },
|
|
460
|
+
});
|
|
461
|
+
}
|
|
462
|
+
results.push(loadMaintenanceRun(store, phase3Id));
|
|
463
|
+
},
|
|
464
|
+
);
|
|
465
|
+
|
|
466
|
+
if (!lease.acquired) {
|
|
467
|
+
const skippedId = insertMaintenanceRun(store, {
|
|
468
|
+
lane: "heavy",
|
|
469
|
+
phase: "gate",
|
|
470
|
+
status: "skipped",
|
|
471
|
+
reason: "lease_unavailable",
|
|
472
|
+
finishedAt: new Date().toISOString(),
|
|
473
|
+
});
|
|
474
|
+
results.push(loadMaintenanceRun(store, skippedId));
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
return results;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
function loadMaintenanceRun(store: Store, id: number): MaintenanceRunSummary {
|
|
481
|
+
const row = store.db.prepare(
|
|
482
|
+
`SELECT id, lane, phase, status, reason, selected_count, processed_count,
|
|
483
|
+
created_count, updated_count, rejected_count, null_call_count,
|
|
484
|
+
started_at, finished_at, metrics_json
|
|
485
|
+
FROM maintenance_runs WHERE id = ?`,
|
|
486
|
+
).get(id) as MaintenanceRunSummary | undefined;
|
|
487
|
+
if (!row) {
|
|
488
|
+
throw new Error(`loadMaintenanceRun: row ${id} not found`);
|
|
489
|
+
}
|
|
490
|
+
return row;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/**
|
|
494
|
+
* Start the heavy-maintenance worker loop. Fire-and-forget — returns a stop
|
|
495
|
+
* function the caller can invoke on process shutdown. Off by default — the
|
|
496
|
+
* caller decides whether to start it via `CLAWMEM_HEAVY_LANE=true` or equivalent.
|
|
497
|
+
*
|
|
498
|
+
* A reentrancy guard prevents overlapping ticks if a prior tick is still
|
|
499
|
+
* running when the next interval fires. Separate from the DB-backed lease,
|
|
500
|
+
* which prevents overlap across processes.
|
|
501
|
+
*/
|
|
502
|
+
export function startHeavyMaintenanceWorker(
|
|
503
|
+
store: Store,
|
|
504
|
+
llm: LlamaCpp,
|
|
505
|
+
cfg: HeavyMaintenanceConfig = {},
|
|
506
|
+
): () => void {
|
|
507
|
+
const merged = { ...DEFAULT_CONFIG, ...cfg };
|
|
508
|
+
// Clamp interval to minimum 30 seconds so buggy configs can't pin the CPU.
|
|
509
|
+
const interval = Math.max(30_000, merged.intervalMs);
|
|
510
|
+
|
|
511
|
+
console.log(
|
|
512
|
+
`[heavy-lane] Starting worker (interval=${interval}ms, ` +
|
|
513
|
+
`window=${merged.windowStartHour ?? "always"}-${merged.windowEndHour ?? "always"}, ` +
|
|
514
|
+
`maxUsagesPer10m=${merged.maxContextUsagesPer10m})`,
|
|
515
|
+
);
|
|
516
|
+
|
|
517
|
+
heavyTimer = setInterval(async () => {
|
|
518
|
+
if (heavyRunning) {
|
|
519
|
+
console.log("[heavy-lane] Skipping tick (still running)");
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
heavyRunning = true;
|
|
523
|
+
try {
|
|
524
|
+
await runHeavyMaintenanceTick(store, llm, cfg);
|
|
525
|
+
} catch (err) {
|
|
526
|
+
console.error("[heavy-lane] Tick failed:", err);
|
|
527
|
+
} finally {
|
|
528
|
+
heavyRunning = false;
|
|
529
|
+
}
|
|
530
|
+
}, interval);
|
|
531
|
+
heavyTimer.unref();
|
|
532
|
+
|
|
533
|
+
return () => {
|
|
534
|
+
if (heavyTimer) {
|
|
535
|
+
clearInterval(heavyTimer);
|
|
536
|
+
heavyTimer = null;
|
|
537
|
+
console.log("[heavy-lane] Worker stopped");
|
|
538
|
+
}
|
|
539
|
+
};
|
|
540
|
+
}
|
package/src/mcp.ts
CHANGED
|
@@ -39,6 +39,7 @@ import { classifyIntent, decomposeQuery, extractTemporalConstraint, type IntentT
|
|
|
39
39
|
import { adaptiveTraversal, mergeTraversalResults, mpfpTraversal } from "./graph-traversal.ts";
|
|
40
40
|
import { getDefaultLlamaCpp } from "./llm.ts";
|
|
41
41
|
import { startConsolidationWorker, stopConsolidationWorker } from "./consolidation.ts";
|
|
42
|
+
import { startHeavyMaintenanceWorker, type HeavyMaintenanceConfig } from "./maintenance.ts";
|
|
42
43
|
import { listVaults, loadVaultConfig } from "./config.ts";
|
|
43
44
|
import { getEntityGraphNeighbors, searchEntities } from "./entity.ts";
|
|
44
45
|
|
|
@@ -2604,10 +2605,42 @@ This is the recommended entry point for ALL memory queries.`,
|
|
|
2604
2605
|
startConsolidationWorker(store, llm, intervalMs);
|
|
2605
2606
|
}
|
|
2606
2607
|
|
|
2608
|
+
// v0.8.0 Ext 5: Start heavy-maintenance worker if enabled. Runs on a
|
|
2609
|
+
// longer interval than the light lane, only inside a configurable quiet
|
|
2610
|
+
// window, and gated by context_usage query-rate so interactive sessions
|
|
2611
|
+
// are never starved. Off by default.
|
|
2612
|
+
let stopHeavyLane: (() => void) | null = null;
|
|
2613
|
+
if (Bun.env.CLAWMEM_HEAVY_LANE === "true") {
|
|
2614
|
+
const llm = getDefaultLlamaCpp();
|
|
2615
|
+
const cfg: HeavyMaintenanceConfig = {
|
|
2616
|
+
intervalMs: Bun.env.CLAWMEM_HEAVY_LANE_INTERVAL
|
|
2617
|
+
? parseInt(Bun.env.CLAWMEM_HEAVY_LANE_INTERVAL, 10)
|
|
2618
|
+
: undefined,
|
|
2619
|
+
windowStartHour: Bun.env.CLAWMEM_HEAVY_LANE_WINDOW_START
|
|
2620
|
+
? parseInt(Bun.env.CLAWMEM_HEAVY_LANE_WINDOW_START, 10)
|
|
2621
|
+
: null,
|
|
2622
|
+
windowEndHour: Bun.env.CLAWMEM_HEAVY_LANE_WINDOW_END
|
|
2623
|
+
? parseInt(Bun.env.CLAWMEM_HEAVY_LANE_WINDOW_END, 10)
|
|
2624
|
+
: null,
|
|
2625
|
+
maxContextUsagesPer10m: Bun.env.CLAWMEM_HEAVY_LANE_MAX_USAGES
|
|
2626
|
+
? parseInt(Bun.env.CLAWMEM_HEAVY_LANE_MAX_USAGES, 10)
|
|
2627
|
+
: undefined,
|
|
2628
|
+
staleObservationLimit: Bun.env.CLAWMEM_HEAVY_LANE_OBS_LIMIT
|
|
2629
|
+
? parseInt(Bun.env.CLAWMEM_HEAVY_LANE_OBS_LIMIT, 10)
|
|
2630
|
+
: undefined,
|
|
2631
|
+
staleDeductiveLimit: Bun.env.CLAWMEM_HEAVY_LANE_DED_LIMIT
|
|
2632
|
+
? parseInt(Bun.env.CLAWMEM_HEAVY_LANE_DED_LIMIT, 10)
|
|
2633
|
+
: undefined,
|
|
2634
|
+
useSurprisalSelector: Bun.env.CLAWMEM_HEAVY_LANE_SURPRISAL === "true",
|
|
2635
|
+
};
|
|
2636
|
+
stopHeavyLane = startHeavyMaintenanceWorker(store, llm, cfg);
|
|
2637
|
+
}
|
|
2638
|
+
|
|
2607
2639
|
// Signal handlers for graceful shutdown
|
|
2608
2640
|
process.on("SIGINT", () => {
|
|
2609
2641
|
console.error("\n[mcp] Received SIGINT, shutting down...");
|
|
2610
2642
|
stopConsolidationWorker();
|
|
2643
|
+
if (stopHeavyLane) stopHeavyLane();
|
|
2611
2644
|
closeAllStores();
|
|
2612
2645
|
process.exit(0);
|
|
2613
2646
|
});
|
|
@@ -2615,6 +2648,7 @@ This is the recommended entry point for ALL memory queries.`,
|
|
|
2615
2648
|
process.on("SIGTERM", () => {
|
|
2616
2649
|
console.error("\n[mcp] Received SIGTERM, shutting down...");
|
|
2617
2650
|
stopConsolidationWorker();
|
|
2651
|
+
if (stopHeavyLane) stopHeavyLane();
|
|
2618
2652
|
closeAllStores();
|
|
2619
2653
|
process.exit(0);
|
|
2620
2654
|
});
|
package/src/store.ts
CHANGED
|
@@ -854,6 +854,41 @@ function initializeDatabase(db: Database): void {
|
|
|
854
854
|
if (!mrColNames.has("contradict_confidence")) {
|
|
855
855
|
try { db.exec(`ALTER TABLE memory_relations ADD COLUMN contradict_confidence REAL`); } catch { /* column exists */ }
|
|
856
856
|
}
|
|
857
|
+
|
|
858
|
+
// v0.8.0 Ext 5: Heavy maintenance lane journal. Every scheduled attempt
|
|
859
|
+
// writes one row — including skips — so operators can reconstruct why a
|
|
860
|
+
// lane did or did not run on any tick.
|
|
861
|
+
db.exec(`
|
|
862
|
+
CREATE TABLE IF NOT EXISTS maintenance_runs (
|
|
863
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
864
|
+
lane TEXT NOT NULL,
|
|
865
|
+
phase TEXT NOT NULL,
|
|
866
|
+
status TEXT NOT NULL,
|
|
867
|
+
reason TEXT,
|
|
868
|
+
selected_count INTEGER NOT NULL DEFAULT 0,
|
|
869
|
+
processed_count INTEGER NOT NULL DEFAULT 0,
|
|
870
|
+
created_count INTEGER NOT NULL DEFAULT 0,
|
|
871
|
+
updated_count INTEGER NOT NULL DEFAULT 0,
|
|
872
|
+
rejected_count INTEGER NOT NULL DEFAULT 0,
|
|
873
|
+
null_call_count INTEGER NOT NULL DEFAULT 0,
|
|
874
|
+
started_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
875
|
+
finished_at TEXT,
|
|
876
|
+
metrics_json TEXT
|
|
877
|
+
)
|
|
878
|
+
`);
|
|
879
|
+
db.exec(`CREATE INDEX IF NOT EXISTS idx_maintenance_runs_lane_started ON maintenance_runs(lane, started_at DESC)`);
|
|
880
|
+
|
|
881
|
+
// v0.8.0 Ext 5: DB-backed worker lease table for multi-process exclusivity
|
|
882
|
+
// on the heavy lane. Lease holders fence via random token; expired leases
|
|
883
|
+
// are reclaimed via atomic upsert inside a transaction.
|
|
884
|
+
db.exec(`
|
|
885
|
+
CREATE TABLE IF NOT EXISTS worker_leases (
|
|
886
|
+
worker_name TEXT PRIMARY KEY,
|
|
887
|
+
lease_token TEXT NOT NULL,
|
|
888
|
+
acquired_at TEXT NOT NULL,
|
|
889
|
+
expires_at TEXT NOT NULL
|
|
890
|
+
)
|
|
891
|
+
`);
|
|
857
892
|
}
|
|
858
893
|
|
|
859
894
|
|