akm-cli 0.9.0-beta.54 → 0.9.0-beta.55

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/dist/cli.js +5 -3
  2. package/dist/commands/agent/contribute-cli.js +2 -3
  3. package/dist/commands/env/env-cli.js +187 -202
  4. package/dist/commands/env/secret-cli.js +109 -121
  5. package/dist/commands/feedback-cli.js +152 -155
  6. package/dist/commands/health/advisories.js +151 -0
  7. package/dist/commands/health/improve-metrics.js +754 -0
  8. package/dist/commands/health/llm-usage.js +65 -0
  9. package/dist/commands/health/md-report.js +103 -0
  10. package/dist/commands/health/metrics.js +278 -0
  11. package/dist/commands/health/task-runs.js +135 -0
  12. package/dist/commands/health/types.js +18 -0
  13. package/dist/commands/health/windows.js +196 -0
  14. package/dist/commands/health.js +14 -1624
  15. package/dist/commands/improve/anti-collapse.js +170 -0
  16. package/dist/commands/improve/collapse-detector.js +3 -2
  17. package/dist/commands/improve/consolidate.js +636 -633
  18. package/dist/commands/improve/dedup.js +1 -1
  19. package/dist/commands/improve/distill/content-repair.js +202 -0
  20. package/dist/commands/improve/distill/promote-memory.js +228 -0
  21. package/dist/commands/improve/distill/quality-gate.js +233 -0
  22. package/dist/commands/improve/distill-guards.js +127 -0
  23. package/dist/commands/improve/distill.js +49 -575
  24. package/dist/commands/improve/extract-cli.js +74 -76
  25. package/dist/commands/improve/extract.js +6 -4
  26. package/dist/commands/improve/hot-probation.js +45 -0
  27. package/dist/commands/improve/improve-auto-accept.js +3 -2
  28. package/dist/commands/improve/improve-cli.js +14 -13
  29. package/dist/commands/improve/improve-result-file.js +2 -1
  30. package/dist/commands/improve/improve.js +6 -5
  31. package/dist/commands/improve/loop-stages.js +19 -21
  32. package/dist/commands/improve/preparation.js +4 -2
  33. package/dist/commands/improve/procedural.js +10 -31
  34. package/dist/commands/improve/recombine.js +19 -43
  35. package/dist/commands/improve/reflect.js +1 -1
  36. package/dist/commands/improve/schema-similarity-gate.js +168 -0
  37. package/dist/commands/improve/shared.js +48 -0
  38. package/dist/commands/observability-cli.js +4 -4
  39. package/dist/commands/proposal/drain-policies.js +2 -2
  40. package/dist/commands/proposal/drain.js +1 -1
  41. package/dist/commands/proposal/legacy-import.js +115 -0
  42. package/dist/commands/proposal/proposal-cli.js +3 -3
  43. package/dist/commands/proposal/proposal.js +2 -1
  44. package/dist/commands/proposal/propose.js +1 -1
  45. package/dist/commands/proposal/repository.js +829 -0
  46. package/dist/commands/proposal/validators/proposals.js +5 -920
  47. package/dist/commands/read/remember-cli.js +132 -137
  48. package/dist/commands/read/search-cli.js +1 -1
  49. package/dist/commands/registry-cli.js +76 -87
  50. package/dist/commands/sources/add-cli.js +90 -94
  51. package/dist/commands/sources/history.js +1 -1
  52. package/dist/commands/sources/schema-repair.js +1 -1
  53. package/dist/commands/sources/sources-cli.js +3 -3
  54. package/dist/commands/sources/stash-cli.js +1 -1
  55. package/dist/commands/tasks/tasks-cli.js +1 -2
  56. package/dist/commands/wiki-cli.js +2 -3
  57. package/dist/core/common.js +3 -3
  58. package/dist/core/config/config-schema.js +6 -0
  59. package/dist/core/deep-merge.js +38 -0
  60. package/dist/core/events.js +2 -1
  61. package/dist/core/logs-db.js +8 -13
  62. package/dist/core/paths.js +14 -14
  63. package/dist/core/state-db.js +13 -1140
  64. package/dist/indexer/db/db.js +66 -709
  65. package/dist/indexer/db/entry-mapper.js +41 -0
  66. package/dist/indexer/db/schema.js +516 -0
  67. package/dist/indexer/feedback/utility-policy.js +85 -0
  68. package/dist/indexer/graph/graph-extraction.js +2 -1
  69. package/dist/indexer/index-writer-lock.js +9 -0
  70. package/dist/indexer/indexer.js +78 -23
  71. package/dist/indexer/search/fts-query.js +51 -0
  72. package/dist/integrations/agent/spawn.js +15 -66
  73. package/dist/output/text/helpers.js +13 -0
  74. package/dist/scripts/migrate-storage.js +6891 -7436
  75. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +44 -43
  76. package/dist/setup/legacy-config.js +106 -0
  77. package/dist/setup/prompt.js +57 -0
  78. package/dist/setup/providers.js +14 -0
  79. package/dist/setup/semantic-assets.js +124 -0
  80. package/dist/setup/setup.js +24 -1607
  81. package/dist/setup/steps/connection.js +734 -0
  82. package/dist/setup/steps/output.js +31 -0
  83. package/dist/setup/steps/platforms.js +124 -0
  84. package/dist/setup/steps/semantic.js +27 -0
  85. package/dist/setup/steps/sources.js +222 -0
  86. package/dist/setup/steps/stashdir.js +42 -0
  87. package/dist/setup/steps/tasks.js +152 -0
  88. package/dist/storage/repositories/canaries-repository.js +107 -0
  89. package/dist/storage/repositories/consolidation-repository.js +38 -0
  90. package/dist/storage/repositories/embeddings-repository.js +72 -0
  91. package/dist/storage/repositories/events-repository.js +187 -0
  92. package/dist/storage/repositories/extract-sessions-repository.js +96 -0
  93. package/dist/storage/repositories/improve-runs-repository.js +130 -0
  94. package/dist/storage/repositories/index-db.js +4 -7
  95. package/dist/storage/repositories/proposals-repository.js +220 -0
  96. package/dist/storage/repositories/recombine-repository.js +213 -0
  97. package/dist/storage/repositories/task-history-repository.js +93 -0
  98. package/dist/storage/sqlite-pragmas.js +3 -3
  99. package/dist/tasks/runner.js +2 -1
  100. package/package.json +1 -1
  101. package/dist/commands/improve/homeostatic.js +0 -497
@@ -4,11 +4,17 @@
4
4
  /**
5
5
  * state.db — Durable SQLite database for non-regenerable akm state.
6
6
  *
7
- * This module owns THREE tables that replace flat-file storage:
8
- *
9
- * events — replaces events.jsonl (append-only event bus)
10
- * proposals — replaces per-uuid JSON directories under .akm/proposals/
11
- * task_history replaces per-task JSONL files under <cacheDir>/tasks/history/
7
+ * This module OWNS the state database's shared infrastructure: path resolution,
8
+ * the managed-db open/loan wrappers, the `BEGIN IMMEDIATE` transaction helper,
9
+ * and schema introspection. The table-specific query helpers live by domain in
10
+ * `src/storage/repositories/*-repository.ts` (events, proposals, task-history,
11
+ * improve-runs, extract-sessions, consolidation, recombine, embeddings,
12
+ * canaries); importers reference those modules directly. The migration engine
13
+ * lives in `./state/migrations`.
14
+ *
15
+ * The state DB replaces flat-file storage for data that is NON-REGENERABLE —
16
+ * events (events.jsonl), proposals (per-uuid JSON directories), task history
17
+ * (per-task JSONL), and the improve-pipeline ledgers.
12
18
  *
13
19
  * ## Why a separate database from index.db
14
20
  *
@@ -51,9 +57,7 @@
51
57
  */
52
58
  import path from "node:path";
53
59
  import { openManagedDatabase, withManagedDb, withManagedDbAsync } from "../storage/managed-db.js";
54
- import { classifyImproveAction } from "./improve-types.js";
55
60
  import { getDataDir } from "./paths.js";
56
- import { error } from "./warn.js";
57
61
  // ── Path helper ──────────────────────────────────────────────────────────────
58
62
  /**
59
63
  * Default path: `<dataDir>/state.db`.
@@ -116,350 +120,9 @@ export function withStateDbAsync(fn, opts) {
116
120
  //
117
121
  // The MIGRATIONS registry + runMigrations live in ./state/migrations (the single
118
122
  // append-only ordered source of truth). Imported for internal use by
119
- // openStateDatabase + re-exported so existing importers keep resolving.
123
+ // openStateDatabase.
120
124
  import { runMigrations } from "./state/migrations.js";
121
- export { runMigrations } from "./state/migrations.js";
122
- /**
123
- * Convert a raw `EventRow` from the database to the public `EventEnvelope`
124
- * interface used throughout the events module.
125
- */
126
- export function eventRowToEnvelope(row) {
127
- let metadata;
128
- try {
129
- const parsed = JSON.parse(row.metadata_json);
130
- // Only attach metadata when the JSON blob is non-empty so downstream
131
- // consumers that check `envelope.metadata !== undefined` keep working.
132
- if (Object.keys(parsed).length > 0) {
133
- metadata = parsed;
134
- }
135
- }
136
- catch {
137
- // Corrupt JSON in the DB — treat as no metadata.
138
- }
139
- return {
140
- schemaVersion: 1,
141
- id: row.id,
142
- ts: row.ts,
143
- eventType: row.event_type,
144
- ...(row.ref !== null ? { ref: row.ref } : {}),
145
- ...(metadata !== undefined ? { metadata } : {}),
146
- };
147
- }
148
- /**
149
- * Convert a raw `ProposalRow` to the public `Proposal` shape.
150
- */
151
- export function proposalRowToProposal(row) {
152
- let frontmatter;
153
- if (row.frontmatter_json) {
154
- try {
155
- frontmatter = JSON.parse(row.frontmatter_json);
156
- }
157
- catch {
158
- /* ignore corrupt frontmatter JSON */
159
- }
160
- }
161
- let meta = {};
162
- try {
163
- meta = JSON.parse(row.metadata_json);
164
- }
165
- catch {
166
- /* ignore */
167
- }
168
- return {
169
- id: row.id,
170
- ref: row.ref,
171
- status: row.status,
172
- source: row.source,
173
- ...(typeof meta.sourceRun === "string" ? { sourceRun: meta.sourceRun } : {}),
174
- createdAt: row.created_at,
175
- updatedAt: row.updated_at,
176
- payload: {
177
- content: row.content,
178
- ...(frontmatter !== undefined ? { frontmatter } : {}),
179
- },
180
- ...(meta.review !== undefined ? { review: meta.review } : {}),
181
- ...(typeof meta.confidence === "number" ? { confidence: meta.confidence } : {}),
182
- ...(meta.gateDecision !== undefined ? { gateDecision: meta.gateDecision } : {}),
183
- ...(typeof meta.backupContent === "string" ? { backupContent: meta.backupContent } : {}),
184
- ...(typeof meta.eligibilitySource === "string"
185
- ? { eligibilitySource: meta.eligibilitySource }
186
- : {}),
187
- };
188
- }
189
- /**
190
- * Convert a public `Proposal` to column values ready for an INSERT/UPDATE.
191
- * The `stash_dir` comes from the call site (proposals.ts has it in scope).
192
- */
193
- export function proposalToRowValues(proposal, stashDir) {
194
- // Fields that have no dedicated column live in metadata_json.
195
- const metaObj = {};
196
- if (proposal.sourceRun !== undefined)
197
- metaObj.sourceRun = proposal.sourceRun;
198
- if (proposal.review !== undefined)
199
- metaObj.review = proposal.review;
200
- if (proposal.confidence !== undefined)
201
- metaObj.confidence = proposal.confidence;
202
- if (proposal.gateDecision !== undefined)
203
- metaObj.gateDecision = proposal.gateDecision;
204
- if (proposal.backupContent !== undefined)
205
- metaObj.backupContent = proposal.backupContent;
206
- if (proposal.eligibilitySource !== undefined)
207
- metaObj.eligibilitySource = proposal.eligibilitySource;
208
- return {
209
- id: proposal.id,
210
- stash_dir: stashDir,
211
- ref: proposal.ref,
212
- status: proposal.status,
213
- source: proposal.source,
214
- created_at: proposal.createdAt,
215
- updated_at: proposal.updatedAt,
216
- content: proposal.payload.content,
217
- frontmatter_json: proposal.payload.frontmatter ? JSON.stringify(proposal.payload.frontmatter) : null,
218
- metadata_json: JSON.stringify(metaObj),
219
- };
220
- }
221
- // ── events table helpers ─────────────────────────────────────────────────────
222
- /**
223
- * Insert a single event. Returns the auto-assigned monotonic rowid, which
224
- * callers can store as a "sinceId" cursor for future `readEventsSince` calls.
225
- *
226
- * Best-effort: mirrors the behaviour of the old `appendEvent` — errors are
227
- * caught and logged to stderr rather than propagated so observability never
228
- * breaks mutation.
229
- */
230
- export function insertEvent(db, input) {
231
- try {
232
- const result = db
233
- .prepare(`INSERT INTO events (event_type, ts, ref, metadata_json)
234
- VALUES (?, ?, ?, ?)
235
- RETURNING id`)
236
- .get(input.eventType, input.ts, input.ref ?? null, JSON.stringify(input.metadata ?? {}));
237
- return result?.id;
238
- }
239
- catch (err) {
240
- const message = err instanceof Error ? err.message : String(err);
241
- error(`akm: state.db event insert failed (${message})`);
242
- return undefined;
243
- }
244
- }
245
- /**
246
- * Read events from the database matching the filter. Returns events in
247
- * ascending id order so consumers can process them in emission order.
248
- *
249
- * The returned `nextId` is the maximum id seen (or `sinceId` when no rows
250
- * match), suitable as the next `sinceId` cursor value.
251
- */
252
- export function readStateEvents(db, options = {}) {
253
- const conditions = [];
254
- const params = [];
255
- if (options.sinceId !== undefined && options.sinceId > 0) {
256
- conditions.push("id > ?");
257
- params.push(options.sinceId);
258
- }
259
- if (options.since) {
260
- conditions.push("ts >= ?");
261
- params.push(options.since);
262
- }
263
- if (options.type) {
264
- conditions.push("event_type = ?");
265
- params.push(options.type);
266
- }
267
- if (options.ref) {
268
- conditions.push("ref = ?");
269
- params.push(options.ref);
270
- }
271
- const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
272
- const rows = db
273
- .prepare(`SELECT id, event_type, ts, ref, metadata_json FROM events ${where} ORDER BY id ASC`)
274
- .all(...params);
275
- const events = rows.map(eventRowToEnvelope);
276
- const nextId = events.length > 0 ? events[events.length - 1].id : (options.sinceId ?? 0);
277
- return { events, nextId };
278
- }
279
- /**
280
- * Delete events older than `retentionDays` (default: 90). Safe to call from
281
- * a maintenance cron; uses a single DELETE with an index-covered ts predicate.
282
- *
283
- * Returns the number of rows actually deleted so callers can emit an
284
- * `events_purged` observability event. A non-positive or non-finite
285
- * `retentionDays` is treated as "disabled" and returns 0 without scanning.
286
- */
287
- export function purgeOldEvents(db, retentionDays = 90) {
288
- if (!Number.isFinite(retentionDays) || retentionDays <= 0)
289
- return 0;
290
- const cutoff = new Date(Date.now() - retentionDays * 86_400_000).toISOString();
291
- const result = db.prepare("DELETE FROM events WHERE ts < ?").run(cutoff);
292
- // bun:sqlite's run() returns { changes, lastInsertRowid }. `changes` may be
293
- // a number or bigint depending on the underlying lib; coerce to number for
294
- // the metadata payload.
295
- const changes = result.changes ?? 0;
296
- return typeof changes === "bigint" ? Number(changes) : changes;
297
- }
298
- // ── proposals table helpers ──────────────────────────────────────────────────
299
- /**
300
- * Upsert a proposal row. Called by the proposal write path when state.db is
301
- * the active backend.
302
- */
303
- export function upsertProposal(db, proposal, stashDir) {
304
- const v = proposalToRowValues(proposal, stashDir);
305
- db.prepare(`
306
- INSERT INTO proposals
307
- (id, stash_dir, ref, status, source, created_at, updated_at, content, frontmatter_json, metadata_json)
308
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
309
- ON CONFLICT(id) DO UPDATE SET
310
- stash_dir = excluded.stash_dir,
311
- ref = excluded.ref,
312
- status = excluded.status,
313
- source = excluded.source,
314
- updated_at = excluded.updated_at,
315
- content = excluded.content,
316
- frontmatter_json = excluded.frontmatter_json,
317
- metadata_json = excluded.metadata_json
318
- `).run(v.id, v.stash_dir, v.ref, v.status, v.source, v.created_at, v.updated_at, v.content, v.frontmatter_json, v.metadata_json);
319
- }
320
- /**
321
- * List proposals, optionally filtered by stashDir, status, and/or ref.
322
- *
323
- * Results are ordered by `created_at ASC` (matching the historical
324
- * `listProposals()` sort), with `rowid` as a deterministic tiebreak so two
325
- * proposals created in the same millisecond list in insertion order.
326
- */
327
- export function listStateProposals(db, options = {}) {
328
- const conditions = [];
329
- const params = [];
330
- if (options.stashDir) {
331
- conditions.push("stash_dir = ?");
332
- params.push(options.stashDir);
333
- }
334
- if (options.status) {
335
- conditions.push("status = ?");
336
- params.push(options.status);
337
- }
338
- if (options.ref) {
339
- conditions.push("ref = ?");
340
- params.push(options.ref);
341
- }
342
- const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
343
- const rows = db
344
- .prepare(`SELECT id, stash_dir, ref, status, source, created_at, updated_at,
345
- content, frontmatter_json, metadata_json
346
- FROM proposals ${where} ORDER BY created_at ASC, rowid ASC`)
347
- .all(...params);
348
- return rows.map(proposalRowToProposal);
349
- }
350
- /**
351
- * Read every proposal's `gateDecision` record across all stashes (#612).
352
- *
353
- * Calibration reads the auto-accept gate's per-proposal decisions regardless of
354
- * the proposal's current lifecycle status — a proposal that was auto-accepted
355
- * is now `accepted`, an auto-rejected one stays `pending`, so filtering by
356
- * status would drop half the join. Rows without a `gateDecision` (created
357
- * before #577, or never gated) are skipped. The result is ordered by
358
- * `decidedAt ASC` for deterministic downstream aggregation, falling back to
359
- * `created_at` ordering from the SQL layer for rows with equal/missing
360
- * timestamps.
361
- */
362
- export function listProposalGateDecisions(db) {
363
- const rows = db.prepare("SELECT metadata_json FROM proposals ORDER BY created_at ASC, rowid ASC").all();
364
- const decisions = [];
365
- for (const row of rows) {
366
- let meta;
367
- try {
368
- meta = JSON.parse(row.metadata_json);
369
- }
370
- catch {
371
- continue;
372
- }
373
- const decision = meta.gateDecision;
374
- if (decision && typeof decision === "object" && typeof decision.outcome === "string") {
375
- decisions.push(decision);
376
- }
377
- }
378
- decisions.sort((a, b) => new Date(a.decidedAt).getTime() - new Date(b.decidedAt).getTime());
379
- return decisions;
380
- }
381
- // ── WS-4: Per-phase gate threshold store (Migration 012) ─────────────────────
382
- /**
383
- * Read the persisted auto-tuned threshold for a gate phase.
384
- *
385
- * Returns `undefined` when no row exists yet (first run, or the phase has
386
- * never been tuned). The caller falls back to the global `options.autoAccept`
387
- * in that case.
388
- */
389
- export function getPhaseThreshold(db, phase) {
390
- const row = db.prepare("SELECT threshold FROM improve_gate_thresholds WHERE phase = ?").get(phase);
391
- return row?.threshold;
392
- }
393
- /**
394
- * Persist the auto-tuned threshold for a gate phase.
395
- * Uses INSERT OR REPLACE so the call is idempotent (upsert semantics).
396
- */
397
- export function persistPhaseThreshold(db, phase, threshold) {
398
- db.prepare(`INSERT OR REPLACE INTO improve_gate_thresholds (phase, threshold, updated_at)
399
- VALUES (?, ?, ?)`).run(phase, Math.round(threshold), Date.now());
400
- }
401
- /**
402
- * Look up a single proposal by id, optionally scoped to one stash root.
403
- * Returns undefined when not found.
404
- */
405
- export function getStateProposal(db, id, stashDir) {
406
- const sql = `SELECT id, stash_dir, ref, status, source, created_at, updated_at,
407
- content, frontmatter_json, metadata_json
408
- FROM proposals WHERE id = ?${stashDir ? " AND stash_dir = ?" : ""}`;
409
- const row = (stashDir ? db.prepare(sql).get(id, stashDir) : db.prepare(sql).get(id));
410
- return row ? proposalRowToProposal(row) : undefined;
411
- }
412
- /**
413
- * Find PENDING proposal ids in one stash whose id starts with `idPrefix`.
414
- * Backs the UUID-prefix form of `akm proposal show/accept/... <prefix>` —
415
- * prefix resolution is deliberately scoped to the live (pending) queue,
416
- * mirroring the historical behaviour of scanning only the live directory.
417
- *
418
- * `%` / `_` / `\` in the prefix are escaped so the LIKE pattern is literal.
419
- */
420
- export function listStateProposalIdsByPrefix(db, stashDir, idPrefix) {
421
- const escaped = idPrefix.replace(/[\\%_]/g, (ch) => `\\${ch}`);
422
- const rows = db
423
- .prepare(`SELECT id FROM proposals
424
- WHERE stash_dir = ? AND status = 'pending' AND id LIKE ? ESCAPE '\\'
425
- ORDER BY id ASC`)
426
- .all(stashDir, `${escaped}%`);
427
- return rows.map((r) => r.id);
428
- }
429
- /**
430
- * Whether the legacy filesystem proposal import has already run for `stashDir`.
431
- * See migration 005 (`proposal_fs_imports`).
432
- */
433
- export function hasImportedFsProposals(db, stashDir) {
434
- // Drivers disagree on the no-row sentinel (bun:sqlite → null,
435
- // better-sqlite3 → undefined) — Boolean() covers both.
436
- return Boolean(db.prepare("SELECT 1 FROM proposal_fs_imports WHERE stash_dir = ?").get(stashDir));
437
- }
438
- /**
439
- * Record that the legacy filesystem proposal import completed for `stashDir`
440
- * so subsequent invocations skip the directory walk. INSERT OR REPLACE keeps
441
- * the call idempotent.
442
- */
443
- export function recordFsProposalsImport(db, stashDir, importedCount) {
444
- db.prepare("INSERT OR REPLACE INTO proposal_fs_imports (stash_dir, imported_at, imported_count) VALUES (?, ?, ?)").run(stashDir, new Date().toISOString(), importedCount);
445
- }
446
- /**
447
- * Insert a proposal row ONLY when the id is not already present (used by the
448
- * legacy filesystem import so re-runs never clobber rows that have since been
449
- * mutated through the canonical store). Returns true when a row was inserted.
450
- */
451
- export function insertProposalIfAbsent(db, proposal, stashDir) {
452
- const v = proposalToRowValues(proposal, stashDir);
453
- const result = db
454
- .prepare(`
455
- INSERT OR IGNORE INTO proposals
456
- (id, stash_dir, ref, status, source, created_at, updated_at, content, frontmatter_json, metadata_json)
457
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
458
- `)
459
- .run(v.id, v.stash_dir, v.ref, v.status, v.source, v.created_at, v.updated_at, v.content, v.frontmatter_json, v.metadata_json);
460
- const changes = result.changes ?? 0;
461
- return Number(changes) > 0;
462
- }
125
+ // ── BEGIN IMMEDIATE transaction helper ───────────────────────────────────────
463
126
  /**
464
127
  * Run `fn` inside a `BEGIN IMMEDIATE` transaction.
465
128
  *
@@ -569,97 +232,6 @@ export function withImmediateTransaction(db, fn) {
569
232
  // Exhausted retries on transient begin failures.
570
233
  throw lastBeginErr;
571
234
  }
572
- // ── task_history table helpers ───────────────────────────────────────────────
573
- /**
574
- * Upsert a task history row.
575
- */
576
- export function upsertTaskHistory(db, row) {
577
- // INSERT OR IGNORE: if a run with the same (task_id, started_at) was already
578
- // imported (e.g. by the migration script), skip it silently.
579
- db.prepare(`
580
- INSERT OR IGNORE INTO task_history
581
- (task_id, status, started_at, completed_at, failed_at, log_path,
582
- target_kind, target_ref, metadata_json)
583
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
584
- `).run(row.task_id, row.status, row.started_at, row.completed_at ?? null, row.failed_at ?? null, row.log_path ?? null, row.target_kind ?? null, row.target_ref ?? null, row.metadata_json);
585
- }
586
- /**
587
- * Look up a task history row by task_id. Returns undefined when not found.
588
- */
589
- /**
590
- * Return the most recent run for a given task_id, or undefined if no runs exist.
591
- */
592
- export function getTaskHistory(db, taskId) {
593
- return db
594
- .prepare(`SELECT id, task_id, status, started_at, completed_at, failed_at, log_path,
595
- target_kind, target_ref, metadata_json
596
- FROM task_history WHERE task_id = ? ORDER BY started_at DESC LIMIT 1`)
597
- .get(taskId);
598
- }
599
- /**
600
- * Return all runs for a given task_id, newest first.
601
- */
602
- export function getTaskHistoryRuns(db, taskId, limit = 50) {
603
- return db
604
- .prepare(`SELECT id, task_id, status, started_at, completed_at, failed_at, log_path,
605
- target_kind, target_ref, metadata_json
606
- FROM task_history WHERE task_id = ? ORDER BY started_at DESC LIMIT ?`)
607
- .all(taskId, limit);
608
- }
609
- /**
610
- * Query task history rows by started_at range and/or status.
611
- */
612
- export function queryTaskHistory(db, options = {}) {
613
- const conditions = [];
614
- const params = [];
615
- if (options.since) {
616
- conditions.push("started_at >= ?");
617
- params.push(options.since);
618
- }
619
- if (options.until) {
620
- conditions.push("started_at <= ?");
621
- params.push(options.until);
622
- }
623
- if (options.status) {
624
- conditions.push("status = ?");
625
- params.push(options.status);
626
- }
627
- if (options.targetKind) {
628
- conditions.push("target_kind = ?");
629
- params.push(options.targetKind);
630
- }
631
- if (options.targetRef) {
632
- conditions.push("target_ref = ?");
633
- params.push(options.targetRef);
634
- }
635
- const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
636
- return db
637
- .prepare(`SELECT task_id, status, started_at, completed_at, failed_at, log_path,
638
- target_kind, target_ref, metadata_json
639
- FROM task_history ${where} ORDER BY started_at DESC`)
640
- .all(...params);
641
- }
642
- /**
643
- * Read COMPLETED `akm-improve` task_history runs whose `started_at` falls in
644
- * `[since, until)` (or `started_at >= since` when `until` is omitted), ordered
645
- * oldest-first by `started_at`. Only rows with a non-null `completed_at` are
646
- * returned (in-flight runs are excluded). The `task_id = 'akm-improve'`
647
- * predicate is fixed because the only caller (commands/health.ts
648
- * `loadTaskIntervals`) builds wall-time intervals for the improve cron task.
649
- *
650
- * Owns the SQL formerly inlined in commands/health.ts. Note the bound is
651
- * EXCLUSIVE on the upper end (`started_at < ?`) — callers pass an already
652
- * widened window; this helper does not widen.
653
- *
654
- * Connection-lifetime rule (WS5): `.all()` materializes a plain array before
655
- * returning.
656
- */
657
- export function queryCompletedTaskIntervals(db, since, until) {
658
- const sql = until
659
- ? "SELECT started_at, completed_at FROM task_history WHERE task_id = 'akm-improve' AND started_at >= ? AND started_at < ? AND completed_at IS NOT NULL ORDER BY started_at"
660
- : "SELECT started_at, completed_at FROM task_history WHERE task_id = 'akm-improve' AND started_at >= ? AND completed_at IS NOT NULL ORDER BY started_at";
661
- return (until ? db.prepare(sql).all(since, until) : db.prepare(sql).all(since));
662
- }
663
235
  // ── schema introspection ─────────────────────────────────────────────────────
664
236
  /**
665
237
  * Return the subset of `names` that exist as TABLEs in this database, ordered
@@ -680,702 +252,3 @@ export function listExistingTableNames(db, names) {
680
252
  .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (${placeholders}) ORDER BY name`)
681
253
  .all(...names);
682
254
  }
683
- // ── events.jsonl import ──────────────────────────────────────────────────────
684
- /**
685
- * Import all events from an `events.jsonl` file into the `events` table.
686
- *
687
- * The old byte-offset `id` is NOT preserved — the database assigns new
688
- * monotonic integer ids. Callers that persisted a byte-offset cursor must
689
- * discard it after migration and use the returned `maxId` as the new cursor.
690
- *
691
- * **Idempotency**: each line is pre-checked against the `events` table using
692
- * `(event_type, ts, ref, metadata_json)` as the duplicate key. Lines whose
693
- * exact tuple is already present are skipped and reported as `skipped` in the
694
- * return value. This makes the migration safe to re-run (the v0.7→v0.8
695
- * migration guide recommends re-running the script as a recovery path; without
696
- * this guard, every re-run would double-import the entire event log).
697
- *
698
- * Duplicate detection is per-import-tuple, not a table-wide UNIQUE constraint:
699
- * the events table has no UNIQUE constraint at runtime so that
700
- * `appendEvent` can write multiple events with the same ts (sub-millisecond
701
- * bursts produce identical `(event_type, ts, ref)` triples in practice). The
702
- * SELECT-first check is scoped to the import path only.
703
- *
704
- * The import is wrapped in a single transaction for atomicity.
705
- *
706
- * @param db - Open state.db connection.
707
- * @param jsonlPath - Absolute path to the events.jsonl file to import.
708
- * @returns Number of rows inserted, the max id assigned, and the
709
- * count of rows skipped because an identical event already
710
- * existed in the table.
711
- */
712
- export async function importEventsJsonl(db, jsonlPath) {
713
- const { readFileSync, existsSync } = await import("node:fs");
714
- if (!existsSync(jsonlPath)) {
715
- return { imported: 0, maxId: 0, skipped: 0 };
716
- }
717
- const text = readFileSync(jsonlPath, "utf8");
718
- const lines = text.split("\n").filter((l) => l.trim().length > 0);
719
- let imported = 0;
720
- let maxId = 0;
721
- let skipped = 0;
722
- const insertStmt = db.prepare(`INSERT INTO events (event_type, ts, ref, metadata_json)
723
- VALUES (?, ?, ?, ?)
724
- RETURNING id`);
725
- // Dedup pre-check: matches by the full tuple including metadata_json so an
726
- // import is idempotent over identical rows but does not collide with two
727
- // genuinely different events that happen to share (event_type, ts, ref).
728
- //
729
- // Uses IS for ref so two NULL refs compare equal (a plain `=` would treat
730
- // NULL = NULL as NULL and the row would be re-inserted on every run).
731
- const existsStmt = db.prepare(`SELECT 1 FROM events
732
- WHERE event_type = ?
733
- AND ts = ?
734
- AND ref IS ?
735
- AND metadata_json = ?
736
- LIMIT 1`);
737
- db.transaction(() => {
738
- for (const line of lines) {
739
- let parsed;
740
- try {
741
- parsed = JSON.parse(line);
742
- }
743
- catch {
744
- continue; // skip malformed lines — same behaviour as readEvents()
745
- }
746
- const eventType = typeof parsed.eventType === "string" ? parsed.eventType : "unknown";
747
- const ts = typeof parsed.ts === "string" ? parsed.ts : new Date().toISOString();
748
- const ref = typeof parsed.ref === "string" ? parsed.ref : null;
749
- const metadata = parsed.metadata !== undefined && typeof parsed.metadata === "object" ? JSON.stringify(parsed.metadata) : "{}";
750
- const duplicate = existsStmt.get(eventType, ts, ref, metadata);
751
- if (duplicate) {
752
- skipped++;
753
- continue;
754
- }
755
- const result = insertStmt.get(eventType, ts, ref, metadata);
756
- if (result) {
757
- imported++;
758
- if (result.id > maxId)
759
- maxId = result.id;
760
- }
761
- }
762
- })();
763
- return { imported, maxId, skipped };
764
- }
765
- /**
766
- * Compute the cheap aggregate metrics blob from a full improve result.
767
- *
768
- * Pure function — no I/O. Used by {@link recordImproveRun} to populate
769
- * `metrics_json`. Exposed for tests and for any future call site that wants
770
- * the same aggregation logic without hitting state.db.
771
- */
772
- export function computeImproveRunMetrics(result) {
773
- const plannedCount = Array.isArray(result.plannedRefs) ? result.plannedRefs.length : 0;
774
- const actions = Array.isArray(result.actions) ? result.actions : [];
775
- const actionsCount = actions.length;
776
- let acceptedCount = 0;
777
- let rejectedCount = 0;
778
- let skippedCount = 0;
779
- let autoAcceptedCount = 0;
780
- let errorCount = 0;
781
- for (const action of actions) {
782
- // Bucketing delegated to the shared classifyImproveAction so this aggregate
783
- // and the improve_completed event in improve.ts can never disagree, and so a
784
- // new union variant is a compile error rather than a silent drop. Gated skips
785
- // (cooldown / signal-delta / distill pool-delta) bucket to "skipped", NOT
786
- // "rejected" — only a guard-rejected produced change is a true rejection.
787
- // "noop" (memory-prune) is intentionally counted in none of the buckets.
788
- switch (classifyImproveAction(action.mode)) {
789
- case "accepted":
790
- acceptedCount++;
791
- break;
792
- case "rejected":
793
- rejectedCount++;
794
- break;
795
- case "skipped":
796
- skippedCount++;
797
- break;
798
- case "error":
799
- errorCount++;
800
- break;
801
- case "noop":
802
- break;
803
- }
804
- // Legacy: pre-gate action results may carry autoAccepted: true (reflect path).
805
- const r = action.result;
806
- if (r && r.autoAccepted === true)
807
- autoAcceptedCount++;
808
- }
809
- // Add gate-promoted count from the unified PostPhaseAutoAcceptGate (all phases).
810
- autoAcceptedCount += result.gateAutoAcceptedCount ?? 0;
811
- return { plannedCount, actionsCount, acceptedCount, rejectedCount, skippedCount, autoAcceptedCount, errorCount };
812
- }
813
- /**
814
- * Insert a single improve-run row into `improve_runs`. Uses parameterised SQL.
815
- *
816
- * Idempotency: the table's PRIMARY KEY is `id`, so re-running with the same
817
- * runId would error. Callers mint a fresh runId per invocation via
818
- * {@link buildImproveRunId} so this is not a concern in practice — but the
819
- * default behaviour is INSERT (not REPLACE) so accidental dupes surface as
820
- * a SQLite constraint error rather than silently overwriting a prior record.
821
- *
822
- * The `metrics` parameter defaults to the output of
823
- * {@link computeImproveRunMetrics} when not supplied. Pass an explicit
824
- * `metrics` object to override the derivation (e.g. tests).
825
- */
826
- export function recordImproveRun(db, input) {
827
- const metricsObj = input.metrics ?? computeImproveRunMetrics(input.result);
828
- db.prepare(`
829
- INSERT INTO improve_runs
830
- (id, started_at, completed_at, stash_dir, dry_run, profile,
831
- scope_mode, scope_value, guidance, ok, result_json, metrics_json, metadata_json)
832
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
833
- `).run(input.id, input.startedAt, input.completedAt, input.stashDir, input.dryRun ? 1 : 0, input.profile, input.scopeMode, input.scopeValue, input.guidance, input.ok ? 1 : 0, JSON.stringify(input.result), JSON.stringify(metricsObj), JSON.stringify(input.metadata ?? {}));
834
- }
835
- /**
836
- * Read real (non-dry-run) improve_runs rows whose `started_at` falls in the
837
- * window `[since, until)`. When `until` is omitted the window is open-ended
838
- * (`started_at >= since`). Rows are returned newest-first (`ORDER BY
839
- * started_at DESC`).
840
- *
841
- * Owns the SQL formerly inlined in commands/health.ts (`loadImproveRunRows`).
842
- * The `dry_run = 0` filter is first-class so dry-run probes never pollute
843
- * productivity audits.
844
- *
845
- * Connection-lifetime rule (WS5): `.all()` fully materializes the result set
846
- * into a plain array before returning — no live cursor escapes the caller's
847
- * `openStateDatabase` scope.
848
- */
849
- export function queryImproveRuns(db, since, until) {
850
- const sql = until
851
- ? "SELECT id, started_at, completed_at, ok, scope_mode, scope_value, result_json FROM improve_runs WHERE started_at >= ? AND started_at < ? AND dry_run = 0 ORDER BY started_at DESC"
852
- : "SELECT id, started_at, completed_at, ok, scope_mode, scope_value, result_json FROM improve_runs WHERE started_at >= ? AND dry_run = 0 ORDER BY started_at DESC";
853
- return (until ? db.prepare(sql).all(since, until) : db.prepare(sql).all(since));
854
- }
855
- /**
856
- * Delete improve_runs rows older than `retentionDays` (default: 90). Mirrors
857
- * {@link purgeOldEvents} — same default, same return shape (number of rows
858
- * actually deleted), same disabled-when-non-finite semantics.
859
- *
860
- * Safe to call from the improve post-loop maintenance pass alongside
861
- * `purgeOldEvents(db, retentionDays)`.
862
- */
863
- export function purgeOldImproveRuns(db, retentionDays = 90) {
864
- if (!Number.isFinite(retentionDays) || retentionDays <= 0)
865
- return 0;
866
- const cutoff = new Date(Date.now() - retentionDays * 86_400_000).toISOString();
867
- const result = db.prepare("DELETE FROM improve_runs WHERE started_at < ?").run(cutoff);
868
- const changes = result.changes ?? 0;
869
- return typeof changes === "bigint" ? Number(changes) : changes;
870
- }
871
- /**
872
- * Record (or update) one session's extract outcome. INSERT-OR-REPLACE so the
873
- * row reflects the most recent run. The `content_hash` persisted here is what
874
- * the NEXT run compares against (#602): a byte-identical session is skipped, a
875
- * changed session is re-processed, and a NULL-backfill row becomes hash-stable
876
- * after its one reprocess. `session_ended_at` is still written for
877
- * telemetry/forensics but is no longer the skip authority.
878
- */
879
- export function upsertExtractedSession(db, input) {
880
- const endedAtIso = typeof input.sessionEndedAt === "number" && Number.isFinite(input.sessionEndedAt)
881
- ? new Date(input.sessionEndedAt).toISOString()
882
- : null;
883
- db.prepare(`
884
- INSERT OR REPLACE INTO extract_sessions_seen
885
- (harness, session_id, processed_at, session_ended_at, outcome,
886
- candidate_count, proposal_count, rationale, source_run, metadata_json,
887
- content_hash)
888
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
889
- `).run(input.harness, input.sessionId, input.processedAt, endedAtIso, input.outcome, input.candidateCount, input.proposalCount, input.rationale ?? null, input.sourceRun ?? null, JSON.stringify(input.metadata ?? {}), input.contentHash);
890
- }
891
- /**
892
- * Fetch a single session's last extract record, or `undefined` when the
893
- * session has never been processed.
894
- */
895
- export function getExtractedSession(db, harness, sessionId) {
896
- // bun:sqlite returns null (not undefined) when no row matches — normalize so
897
- // callers can rely on `if (!row)` and `toBeUndefined()` equivalently.
898
- const row = db
899
- .prepare("SELECT * FROM extract_sessions_seen WHERE harness = ? AND session_id = ?")
900
- .get(harness, sessionId);
901
- return row ?? undefined;
902
- }
903
- /**
904
- * Bulk-fetch session-extract status for a list of sessionIds in one harness.
905
- * Returns a Map keyed by sessionId so callers can do O(1) lookups while
906
- * iterating the discovery list.
907
- */
908
- export function getExtractedSessionsMap(db, harness, sessionIds) {
909
- const out = new Map();
910
- if (sessionIds.length === 0)
911
- return out;
912
- // SQLite has a ~999 param ceiling; chunk if a caller ever exceeds that.
913
- const CHUNK = 500;
914
- for (let i = 0; i < sessionIds.length; i += CHUNK) {
915
- const chunk = sessionIds.slice(i, i + CHUNK);
916
- const placeholders = chunk.map(() => "?").join(",");
917
- const rows = db
918
- .prepare(`SELECT * FROM extract_sessions_seen
919
- WHERE harness = ? AND session_id IN (${placeholders})`)
920
- .all(harness, ...chunk);
921
- for (const row of rows)
922
- out.set(row.session_id, row);
923
- }
924
- return out;
925
- }
926
- /**
927
- * The most recent extract-run time for a harness — `MAX(processed_at)` across
928
- * its ledger rows, as ms epoch — or `null` when the harness has never been
929
- * extracted. Used to default the discovery window to "since the last run" so an
930
- * intermittently-online host that was off for days still rediscovers sessions
931
- * that ended during the gap (the content-hash ledger keeps the widened window
932
- * free of redundant LLM cost).
933
- */
934
- export function getLastExtractRunAt(db, harness) {
935
- const row = db
936
- .prepare("SELECT MAX(processed_at) AS last FROM extract_sessions_seen WHERE harness = ?")
937
- .get(harness);
938
- if (!row?.last)
939
- return null;
940
- const ms = Date.parse(row.last);
941
- return Number.isFinite(ms) ? ms : null;
942
- }
943
- /**
944
- * Decide whether a session should be skipped because the extractor has already
945
- * processed BYTE-IDENTICAL content (#602). The skip authority is the content
946
- * hash, NOT `session_ended_at` — this is clock-independent, so it is immune to
947
- * the clock-skew / out-of-order-endedAt problems that caused the Jun 11-12
948
- * double-extract + over-throttle incident.
949
- *
950
- * Rules:
951
- * - no prior row → `false` (never seen → process; AC3).
952
- * - prior.content_hash == null → `false` (legacy / hash-less row → process
953
- * exactly once to backfill the hash, then it becomes hash-stable; AC4).
954
- * - hashes equal → `true` (unchanged content → skip; AC1).
955
- * - hashes differ → `false` (changed content → re-process; AC2).
956
- */
957
- export function shouldSkipAlreadyExtractedSession(prior, currentContentHash) {
958
- if (!prior)
959
- return false;
960
- if (prior.content_hash == null)
961
- return false;
962
- return prior.content_hash === currentContentHash;
963
- }
964
- /**
965
- * Bulk-fetch the judged-state cache for a set of entry keys in one query.
966
- * Returns a Map keyed by entry_key so the consolidate pool-selection loop can
967
- * do O(1) "has this memory been judged at this content hash?" lookups.
968
- * Empty input → empty map (no query issued).
969
- */
970
- export function getConsolidationJudgedMap(db, entryKeys) {
971
- const out = new Map();
972
- if (entryKeys.length === 0)
973
- return out;
974
- // SQLite has a ~999 param ceiling; chunk if a caller ever exceeds that.
975
- const CHUNK = 500;
976
- for (let i = 0; i < entryKeys.length; i += CHUNK) {
977
- const chunk = entryKeys.slice(i, i + CHUNK);
978
- const placeholders = chunk.map(() => "?").join(",");
979
- const rows = db
980
- .prepare(`SELECT * FROM consolidation_judged WHERE entry_key IN (${placeholders})`)
981
- .all(...chunk);
982
- for (const row of rows)
983
- out.set(row.entry_key, row);
984
- }
985
- return out;
986
- }
987
- /**
988
- * Record (or update) the judged state for one memory. INSERT-OR-REPLACE so the
989
- * row always reflects the most recent judge of that entry_key. Called once per
990
- * memory the consolidate LLM saw in a successfully-judged chunk.
991
- */
992
- export function upsertConsolidationJudged(db, input) {
993
- db.prepare(`
994
- INSERT OR REPLACE INTO consolidation_judged
995
- (entry_key, content_hash, judged_at, outcome)
996
- VALUES (?, ?, ?, ?)
997
- `).run(input.entryKey, input.contentHash, input.judgedAt, input.outcome);
998
- }
999
- /**
1000
- * Record an induction of a recombine hypothesis and return the new consecutive
1001
- * count. INSERT … ON CONFLICT increments the streak, but the `last_run` guard
1002
- * makes a repeated call within the SAME run idempotent (no double-increment if
1003
- * the same ref appears twice in one run). On insert the streak starts at 1.
1004
- */
1005
- export function recordRecombineInduction(db, input) {
1006
- const row = db
1007
- .prepare(`
1008
- INSERT INTO recombine_hypotheses
1009
- (hypothesis_ref, signature, member_key, consecutive_count, first_seen_at, last_seen_at, last_run)
1010
- VALUES (?, ?, ?, 1, ?, ?, ?)
1011
- ON CONFLICT(hypothesis_ref) DO UPDATE SET
1012
- consecutive_count = consecutive_count + (CASE WHEN last_run IS excluded.last_run THEN 0 ELSE 1 END),
1013
- last_seen_at = excluded.last_seen_at,
1014
- last_run = excluded.last_run,
1015
- signature = excluded.signature,
1016
- member_key = excluded.member_key
1017
- RETURNING consecutive_count
1018
- `)
1019
- .get(input.hypothesisRef, input.signature, input.memberKey, input.seenAt, input.seenAt, input.run);
1020
- return row?.consecutive_count ?? 0;
1021
- }
1022
- /**
1023
- * #633 — find an existing pending (non-promoted) hypothesis row whose cluster
1024
- * is the SAME generalization as a newly-induced one, matched by SIGNATURE plus
1025
- * a Jaccard membership-overlap test, rather than an exact member-set hash.
1026
- *
1027
- * In a growing stash any added/removed memory changes the exact member set, so
1028
- * the ref hash (and member_key) shift every run → a fresh row at count=1 → the
1029
- * streak never reaches `confirmThreshold` and nothing ever promotes. Matching
1030
- * on overlap lets a drifting-but-stable cluster keep accumulating under one row.
1031
- *
1032
- * Returns the matched row with the HIGHEST overlap (ties broken by most-recent
1033
- * `last_seen_at`), or `undefined` when none clears `minOverlap`. Already-promoted
1034
- * rows are ignored so a confirmed lesson is not reopened by a later induction.
1035
- *
1036
- * @param memberKey the candidate cluster's membership fingerprint
1037
- * (sorted member entryKeys joined by `|`).
1038
- * @param minOverlap Jaccard threshold in [0,1]; a candidate matches when
1039
- * |A∩B| / |A∪B| >= minOverlap.
1040
- */
1041
- export function findMatchingRecombineHypothesis(db, input) {
1042
- const candidateMembers = new Set(input.memberKey.split("|").filter((m) => m.length > 0));
1043
- if (candidateMembers.size === 0)
1044
- return undefined;
1045
- const rows = db
1046
- .prepare("SELECT * FROM recombine_hypotheses WHERE signature = ? AND promoted_at IS NULL ORDER BY last_seen_at DESC")
1047
- .all(input.signature);
1048
- let best;
1049
- let bestOverlap = -1;
1050
- for (const row of rows) {
1051
- const rowMembers = row.member_key.split("|").filter((m) => m.length > 0);
1052
- if (rowMembers.length === 0)
1053
- continue;
1054
- let intersection = 0;
1055
- for (const m of rowMembers) {
1056
- if (candidateMembers.has(m))
1057
- intersection += 1;
1058
- }
1059
- const union = candidateMembers.size + rowMembers.length - intersection;
1060
- const overlap = union === 0 ? 0 : intersection / union;
1061
- // rows are ordered last_seen_at DESC, so a strict `>` keeps the most-recent
1062
- // row on ties.
1063
- if (overlap >= input.minOverlap && overlap > bestOverlap) {
1064
- best = row;
1065
- bestOverlap = overlap;
1066
- }
1067
- }
1068
- return best;
1069
- }
1070
- /**
1071
- * Fetch a single recombine hypothesis row, or `undefined` when the ref has
1072
- * never been induced. Normalizes bun:sqlite null → undefined like
1073
- * {@link getExtractedSession}.
1074
- */
1075
- export function getRecombineHypothesis(db, hypothesisRef) {
1076
- const row = db
1077
- .prepare("SELECT * FROM recombine_hypotheses WHERE hypothesis_ref = ?")
1078
- .get(hypothesisRef);
1079
- return row ?? undefined;
1080
- }
1081
- /**
1082
- * Mark a hypothesis promoted: stamp `promoted_at` and reset the consecutive
1083
- * count to 0, so it must re-accumulate a full confirmation streak before it can
1084
- * promote again. The `promoted_at` non-null state is the double-promotion guard.
1085
- */
1086
- export function markRecombineHypothesisPromoted(db, hypothesisRef, promotedAt) {
1087
- db.prepare("UPDATE recombine_hypotheses SET promoted_at = ?, consecutive_count = 0 WHERE hypothesis_ref = ?").run(promotedAt, hypothesisRef);
1088
- }
1089
- /**
1090
- * #658 — does any current-run cluster match this hypothesis row under the SAME
1091
- * signature + Jaccard-overlap rule used for re-induction? A match means the
1092
- * cluster genuinely re-formed this run (it was merely cap-displaced out of the
1093
- * processed top-`maxClustersPerRun` slice), so its streak must NOT be reset.
1094
- */
1095
- function hypothesisMatchesAnyPresentCluster(row, presentClusters, minOverlap) {
1096
- const rowMembers = row.member_key.split("|").filter((m) => m.length > 0);
1097
- if (rowMembers.length === 0)
1098
- return false;
1099
- const rowSet = new Set(rowMembers);
1100
- for (const cluster of presentClusters) {
1101
- if (cluster.signature !== row.signature)
1102
- continue;
1103
- const clusterMembers = cluster.memberKey.split("|").filter((m) => m.length > 0);
1104
- if (clusterMembers.length === 0)
1105
- continue;
1106
- let intersection = 0;
1107
- for (const m of clusterMembers) {
1108
- if (rowSet.has(m))
1109
- intersection += 1;
1110
- }
1111
- const union = rowSet.size + clusterMembers.length - intersection;
1112
- const overlap = union === 0 ? 0 : intersection / union;
1113
- if (overlap >= minOverlap)
1114
- return true;
1115
- }
1116
- return false;
1117
- }
1118
- /**
1119
- * Decay-to-zero every NON-promoted hypothesis NOT re-induced in the current run.
1120
- *
1121
- * A generalization that stops being supported by the corpus has lost its
1122
- * confirmation streak, so we hard-reset `consecutive_count` to 0 (the
1123
- * alternative — `count - 1` floored at 0 — tolerates a single noisy run but
1124
- * blurs the "consecutive" semantics; hard-reset is the conservative choice).
1125
- *
1126
- * Only rows whose `hypothesis_ref` is NOT in `seenRefs` AND whose `last_run` is
1127
- * NOT the current run are decayed. Already-promoted rows are left alone.
1128
- *
1129
- * #658 — CAP-AWARE decay. The recombine pass only re-inducts (and thus marks
1130
- * `seen`) the top-`maxClustersPerRun` clusters, but a cluster genuinely
1131
- * re-forms every run even when it is displaced below that cap. Resetting such a
1132
- * row treats a SCHEDULING miss as a SUBSTANCE miss and traps the hypothesis
1133
- * below `confirmThreshold` forever. When `opts.presentClusters` is supplied, a
1134
- * row is SPARED from decay if it Jaccard-matches any present cluster (same
1135
- * signature, overlap >= `opts.minOverlap`) — i.e. its cluster re-formed this run
1136
- * but was cap-displaced. This does NOT advance the streak (only re-induction in
1137
- * the processed slice does that, via {@link recordRecombineInduction}), so the
1138
- * recurrence bar for promotion is unchanged; it only stops the cap from
1139
- * manufacturing artificial misses. Omitting `presentClusters` preserves the
1140
- * pre-#658 hard-reset-after-one-miss behaviour exactly.
1141
- *
1142
- * Returns the number of rows reset.
1143
- */
1144
- export function decayUnseenRecombineHypotheses(db, currentRun, seenRefs, opts) {
1145
- // #658 — when cap-aware sparing is requested, fold the cap-displaced rows into
1146
- // the "seen" exclusion set: the underlying reset SQL already protects every
1147
- // ref it is given, so sparing == treating a spared row exactly like a seen
1148
- // row for this sweep (its count is left untouched, never advanced).
1149
- let effectiveSeen = seenRefs;
1150
- if (opts && opts.presentClusters.length > 0) {
1151
- const candidates = db
1152
- .prepare("SELECT hypothesis_ref, signature, member_key FROM recombine_hypotheses WHERE promoted_at IS NULL AND (last_run IS NULL OR last_run != ?) AND consecutive_count != 0")
1153
- .all(currentRun);
1154
- const seenSet = new Set(seenRefs);
1155
- for (const row of candidates) {
1156
- if (seenSet.has(row.hypothesis_ref))
1157
- continue;
1158
- if (hypothesisMatchesAnyPresentCluster(row, opts.presentClusters, opts.minOverlap)) {
1159
- seenSet.add(row.hypothesis_ref);
1160
- }
1161
- }
1162
- effectiveSeen = [...seenSet];
1163
- }
1164
- return decayUnseenRecombineHypothesesInner(db, currentRun, effectiveSeen);
1165
- }
1166
- /**
1167
- * The raw reset sweep shared by the cap-aware wrapper above. Resets every
1168
- * non-promoted row from a prior run whose ref is NOT in `seenRefs`. Kept private
1169
- * so the param-ceiling chunking logic lives in one place.
1170
- */
1171
- function decayUnseenRecombineHypothesesInner(db, currentRun, seenRefs) {
1172
- // Reset every eligible row, then exclude the seen refs in chunks to respect
1173
- // the ~999 SQLite param ceiling. With no seen refs we reset all non-promoted
1174
- // rows from prior runs in a single statement.
1175
- if (seenRefs.length === 0) {
1176
- const res = db
1177
- .prepare("UPDATE recombine_hypotheses SET consecutive_count = 0 WHERE promoted_at IS NULL AND (last_run IS NULL OR last_run != ?) AND consecutive_count != 0")
1178
- .run(currentRun);
1179
- return Number(res.changes);
1180
- }
1181
- // A single NOT IN keeps the exclusion atomic (a chunked NOT IN would let a ref
1182
- // excluded by one chunk still be reset by another chunk's statement). The
1183
- // recombine pass caps RE-INDUCED clusters at `maxClustersPerRun` (a handful) —
1184
- // but with #658 cap-aware sparing the caller folds every cap-displaced
1185
- // (present-but-unprocessed) hypothesis into `effectiveSeen` too, so on a large
1186
- // stash `seenRefs` here can carry MANY spared refs, not just the handful that
1187
- // were processed. We cap defensively at ~900 (under SQLite's ~999 param
1188
- // ceiling): if `effectiveSeen` somehow exceeds it we fall back to resetting all
1189
- // eligible rows — which re-introduces the cap-displacement trap for THAT run
1190
- // (spared rows get decayed because the NOT IN protection is dropped). That is a
1191
- // rare, bounded degradation; a stash with >900 simultaneously-spared
1192
- // hypotheses is far beyond current scale.
1193
- if (seenRefs.length > 900) {
1194
- const res = db
1195
- .prepare("UPDATE recombine_hypotheses SET consecutive_count = 0 WHERE promoted_at IS NULL AND (last_run IS NULL OR last_run != ?) AND consecutive_count != 0")
1196
- .run(currentRun);
1197
- return Number(res.changes);
1198
- }
1199
- const placeholders = seenRefs.map(() => "?").join(",");
1200
- const res = db
1201
- .prepare(`UPDATE recombine_hypotheses SET consecutive_count = 0
1202
- WHERE promoted_at IS NULL
1203
- AND (last_run IS NULL OR last_run != ?)
1204
- AND consecutive_count != 0
1205
- AND hypothesis_ref NOT IN (${placeholders})`)
1206
- .run(currentRun, ...seenRefs);
1207
- return Number(res.changes);
1208
- }
1209
- /**
1210
- * Convert a `number[]` embedding vector to the `Float32Array` byte
1211
- * representation stored in the `body_embeddings.embedding` BLOB column.
1212
- */
1213
- export function embeddingToBlob(vec) {
1214
- const f32 = new Float32Array(vec);
1215
- return new Uint8Array(f32.buffer);
1216
- }
1217
- /**
1218
- * Convert the raw `Uint8Array` bytes from the `body_embeddings.embedding`
1219
- * BLOB column back to a `number[]` embedding vector.
1220
- */
1221
- export function blobToEmbedding(blob) {
1222
- // SQLite BLOB columns are returned as Uint8Array; re-interpret as Float32.
1223
- const f32 = new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
1224
- return Array.from(f32);
1225
- }
1226
- /**
1227
- * Bulk-fetch cached body embeddings for a set of content hashes.
1228
- * Returns a Map keyed by `content_hash` (embedding decoded to `number[]`).
1229
- * Empty input → empty map (no query issued).
1230
- *
1231
- * If the stored `model_id` does not match `expectedModelId` the entire table
1232
- * is cleared (drop-all on model mismatch) and an empty map is returned so
1233
- * callers re-embed everything on this run.
1234
- */
1235
- export function getBodyEmbeddings(db, contentHashes, expectedModelId) {
1236
- const out = new Map();
1237
- if (contentHashes.length === 0)
1238
- return out;
1239
- // Model-id mismatch: vectors are in the wrong metric space — drop all rows.
1240
- const firstRow = db.prepare("SELECT model_id FROM body_embeddings LIMIT 1").get();
1241
- if (firstRow && firstRow.model_id !== expectedModelId) {
1242
- db.exec("DELETE FROM body_embeddings");
1243
- return out;
1244
- }
1245
- // SQLite has a ~999 param ceiling; chunk if needed.
1246
- const CHUNK = 500;
1247
- for (let i = 0; i < contentHashes.length; i += CHUNK) {
1248
- const chunk = contentHashes.slice(i, i + CHUNK);
1249
- const placeholders = chunk.map(() => "?").join(",");
1250
- const rows = db
1251
- .prepare(`SELECT content_hash, embedding FROM body_embeddings WHERE content_hash IN (${placeholders})`)
1252
- .all(...chunk);
1253
- for (const row of rows) {
1254
- out.set(row.content_hash, blobToEmbedding(row.embedding));
1255
- }
1256
- }
1257
- return out;
1258
- }
1259
- /**
1260
- * Upsert body-embedding rows in a single transaction.
1261
- * Each entry maps a `cacheHash` → `number[]` vector. `model_id` is stored
1262
- * so a future model change can trigger a drop-all purge.
1263
- */
1264
- export function upsertBodyEmbeddings(db, entries) {
1265
- if (entries.length === 0)
1266
- return;
1267
- const now = Date.now();
1268
- const stmt = db.prepare(`
1269
- INSERT OR REPLACE INTO body_embeddings (content_hash, embedding, model_id, created_at)
1270
- VALUES (?, ?, ?, ?)
1271
- `);
1272
- db.transaction(() => {
1273
- for (const { contentHash, embedding, modelId } of entries) {
1274
- stmt.run(contentHash, embeddingToBlob(embedding), modelId, now);
1275
- }
1276
- })();
1277
- }
1278
- /** Insert a freshly minted canary set (all rows active, one shared set id). */
1279
- export function insertCanaries(db, canarySetId, canaries, now) {
1280
- if (canaries.length === 0)
1281
- return;
1282
- const ts = now ?? new Date().toISOString();
1283
- const stmt = db.prepare(`
1284
- INSERT INTO canary_queries (canary_set_id, anchor_ref, query, source, active, created_at)
1285
- VALUES (?, ?, ?, ?, 1, ?)
1286
- `);
1287
- db.transaction(() => {
1288
- for (const c of canaries) {
1289
- stmt.run(canarySetId, c.anchorRef, c.query, c.source ?? "auto", ts);
1290
- }
1291
- })();
1292
- }
1293
- /** Load the active canary set (empty array = never minted). */
1294
- export function getActiveCanaries(db) {
1295
- // Scope to the NEWEST active set: if an interrupted refresh (or a bug) ever
1296
- // leaves two sets active, mixing their rows would silently corrupt the
1297
- // recall/entropy trend baselines. The newest set wins; stale-active rows are
1298
- // simply never returned.
1299
- return db
1300
- .prepare(`SELECT * FROM canary_queries
1301
- WHERE active = 1 AND canary_set_id = (
1302
- SELECT canary_set_id FROM canary_queries WHERE active = 1
1303
- ORDER BY created_at DESC, id DESC LIMIT 1
1304
- )
1305
- ORDER BY id`)
1306
- .all();
1307
- }
1308
- /** Load one canary set's rows by its exact set id (any active state), insertion order. */
1309
- export function getCanariesBySetId(db, canarySetId) {
1310
- return db
1311
- .prepare(`SELECT * FROM canary_queries WHERE canary_set_id = ? ORDER BY id`)
1312
- .all(canarySetId);
1313
- }
1314
- /** List every distinct canary_set_id that still has active rows. */
1315
- export function listActiveCanarySetIds(db) {
1316
- const rows = db.prepare(`SELECT DISTINCT canary_set_id FROM canary_queries WHERE active = 1`).all();
1317
- return rows.map((r) => r.canary_set_id);
1318
- }
1319
- /**
1320
- * Deactivate every canary row in a set. Rows are RETAINED (active = 0) so
1321
- * historical improve_cycle_metrics rows keyed on the old canary_set_id stay
1322
- * interpretable; only `akm improve canary --refresh` calls this.
1323
- */
1324
- export function deactivateCanarySet(db, canarySetId) {
1325
- const result = db
1326
- .prepare(`UPDATE canary_queries SET active = 0 WHERE canary_set_id = ? AND active = 1`)
1327
- .run(canarySetId);
1328
- const changes = result.changes ?? 0;
1329
- return typeof changes === "bigint" ? Number(changes) : changes;
1330
- }
1331
- /** Persist one qualifying cycle's store-health snapshot. */
1332
- export function insertCycleMetrics(db, row) {
1333
- db.prepare(`
1334
- INSERT INTO improve_cycle_metrics
1335
- (run_id, ts, pass, canary_set_id, mean_recall, mean_ndcg, mean_mrr,
1336
- canary_ranks_json, store_total, store_by_type_json, distinct_content_ratio,
1337
- mean_bigram_diversity, over_generation_count, accepted_actions,
1338
- merge_floor_violations, alerts_json)
1339
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1340
- `).run(row.run_id, row.ts, row.pass, row.canary_set_id, row.mean_recall, row.mean_ndcg, row.mean_mrr, row.canary_ranks_json, row.store_total, row.store_by_type_json, row.distinct_content_ratio, row.mean_bigram_diversity, row.over_generation_count, row.accepted_actions, row.merge_floor_violations, row.alerts_json);
1341
- }
1342
- /**
1343
- * Load the most recent cycle rows for one canary set, OLDEST-first (the alert
1344
- * evaluator's window order). Scoped by canary_set_id so trends never compare
1345
- * across canary re-mints.
1346
- */
1347
- export function queryRecentCycleMetrics(db, canarySetId, limit) {
1348
- const rows = db
1349
- .prepare(`SELECT run_id, ts, pass, canary_set_id, mean_recall, mean_ndcg, mean_mrr,
1350
- canary_ranks_json, store_total, store_by_type_json, distinct_content_ratio,
1351
- mean_bigram_diversity, over_generation_count, accepted_actions,
1352
- merge_floor_violations, alerts_json
1353
- FROM improve_cycle_metrics WHERE canary_set_id = ?
1354
- ORDER BY ts DESC, id DESC LIMIT ?`)
1355
- .all(canarySetId, Math.max(0, limit));
1356
- return rows.reverse();
1357
- }
1358
- /** Load the single most recent cycle row across all canary sets (health surface). */
1359
- export function getLatestCycleMetrics(db) {
1360
- const row = db
1361
- .prepare(`SELECT run_id, ts, pass, canary_set_id, mean_recall, mean_ndcg, mean_mrr,
1362
- canary_ranks_json, store_total, store_by_type_json, distinct_content_ratio,
1363
- mean_bigram_diversity, over_generation_count, accepted_actions,
1364
- merge_floor_violations, alerts_json
1365
- FROM improve_cycle_metrics ORDER BY ts DESC, id DESC LIMIT 1`)
1366
- .get();
1367
- return row == null ? undefined : row;
1368
- }
1369
- /**
1370
- * Delete cycle rows older than `retentionDays` (default 365 — owner-approved;
1371
- * a slow collapse needs a longer trend window than the 90-day events log).
1372
- * Returns the purged row count. canary_queries rows are never purged.
1373
- */
1374
- export function purgeOldCycleMetrics(db, retentionDays = 365) {
1375
- if (!Number.isFinite(retentionDays) || retentionDays <= 0)
1376
- return 0;
1377
- const cutoff = new Date(Date.now() - retentionDays * 86_400_000).toISOString();
1378
- const result = db.prepare("DELETE FROM improve_cycle_metrics WHERE ts < ?").run(cutoff);
1379
- const changes = result.changes ?? 0;
1380
- return typeof changes === "bigint" ? Number(changes) : changes;
1381
- }