akm-cli 0.9.0-beta.53 → 0.9.0-beta.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/clack.js +56 -0
- package/dist/cli/confirm.js +1 -1
- package/dist/cli.js +5 -3
- package/dist/commands/agent/contribute-cli.js +2 -3
- package/dist/commands/env/env-cli.js +187 -202
- package/dist/commands/env/secret-cli.js +109 -121
- package/dist/commands/feedback-cli.js +152 -155
- package/dist/commands/health/advisories.js +151 -0
- package/dist/commands/health/html-report.js +33 -10
- package/dist/commands/health/improve-metrics.js +754 -0
- package/dist/commands/health/llm-usage.js +65 -0
- package/dist/commands/health/md-report.js +103 -0
- package/dist/commands/health/metrics.js +278 -0
- package/dist/commands/health/task-runs.js +135 -0
- package/dist/commands/health/types.js +18 -0
- package/dist/commands/health/windows.js +196 -0
- package/dist/commands/health.js +15 -1492
- package/dist/commands/improve/anti-collapse.js +170 -0
- package/dist/commands/improve/collapse-detector.js +3 -2
- package/dist/commands/improve/consolidate.js +636 -633
- package/dist/commands/improve/dedup.js +1 -1
- package/dist/commands/improve/distill/content-repair.js +202 -0
- package/dist/commands/improve/distill/promote-memory.js +228 -0
- package/dist/commands/improve/distill/quality-gate.js +233 -0
- package/dist/commands/improve/distill-guards.js +127 -0
- package/dist/commands/improve/distill.js +49 -575
- package/dist/commands/improve/extract-cli.js +74 -76
- package/dist/commands/improve/extract.js +6 -4
- package/dist/commands/improve/hot-probation.js +45 -0
- package/dist/commands/improve/improve-auto-accept.js +3 -2
- package/dist/commands/improve/improve-cli.js +14 -13
- package/dist/commands/improve/improve-result-file.js +2 -1
- package/dist/commands/improve/improve.js +6 -5
- package/dist/commands/improve/loop-stages.js +19 -21
- package/dist/commands/improve/outcome-loop.js +18 -16
- package/dist/commands/improve/preparation.js +23 -5
- package/dist/commands/improve/procedural.js +10 -31
- package/dist/commands/improve/recombine.js +19 -43
- package/dist/commands/improve/reflect.js +1 -1
- package/dist/commands/improve/schema-similarity-gate.js +168 -0
- package/dist/commands/improve/shared.js +48 -0
- package/dist/commands/observability-cli.js +4 -4
- package/dist/commands/proposal/drain-policies.js +2 -2
- package/dist/commands/proposal/drain.js +1 -1
- package/dist/commands/proposal/legacy-import.js +115 -0
- package/dist/commands/proposal/proposal-cli.js +3 -3
- package/dist/commands/proposal/proposal.js +2 -1
- package/dist/commands/proposal/propose.js +1 -1
- package/dist/commands/proposal/repository.js +829 -0
- package/dist/commands/proposal/validators/proposals.js +5 -920
- package/dist/commands/read/curate.js +4 -4
- package/dist/commands/read/remember-cli.js +132 -137
- package/dist/commands/read/search-cli.js +7 -5
- package/dist/commands/read/search.js +7 -3
- package/dist/commands/read/show.js +3 -5
- package/dist/commands/registry-cli.js +76 -87
- package/dist/commands/sources/add-cli.js +91 -95
- package/dist/commands/sources/history.js +1 -1
- package/dist/commands/sources/init.js +12 -0
- package/dist/commands/sources/schema-repair.js +1 -1
- package/dist/commands/sources/sources-cli.js +3 -3
- package/dist/commands/sources/stash-cli.js +2 -2
- package/dist/commands/tasks/default-tasks.js +12 -0
- package/dist/commands/tasks/tasks-cli.js +1 -2
- package/dist/commands/wiki-cli.js +2 -3
- package/dist/core/common.js +3 -3
- package/dist/core/config/config-schema.js +6 -0
- package/dist/core/config/config.js +12 -0
- package/dist/core/deep-merge.js +38 -0
- package/dist/core/events.js +2 -1
- package/dist/core/logs-db.js +8 -13
- package/dist/core/paths.js +14 -14
- package/dist/core/state-db.js +13 -1140
- package/dist/core/warn.js +21 -0
- package/dist/indexer/db/db.js +72 -709
- package/dist/indexer/db/entry-mapper.js +41 -0
- package/dist/indexer/db/schema.js +516 -0
- package/dist/indexer/ensure-index.js +3 -2
- package/dist/indexer/feedback/utility-policy.js +85 -0
- package/dist/indexer/graph/graph-extraction.js +2 -1
- package/dist/indexer/index-writer-lock.js +18 -0
- package/dist/indexer/indexer.js +94 -27
- package/dist/indexer/read-preflight.js +23 -0
- package/dist/indexer/search/fts-query.js +51 -0
- package/dist/indexer/walk/walker.js +21 -13
- package/dist/integrations/agent/detect.js +9 -0
- package/dist/integrations/agent/index.js +1 -1
- package/dist/integrations/agent/spawn.js +15 -66
- package/dist/llm/client.js +12 -0
- package/dist/llm/embedder.js +26 -2
- package/dist/llm/embedders/local.js +7 -1
- package/dist/output/text/helpers.js +13 -0
- package/dist/scripts/migrate-storage.js +6903 -7424
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +49 -44
- package/dist/setup/detect.js +9 -0
- package/dist/setup/legacy-config.js +106 -0
- package/dist/setup/prompt.js +57 -0
- package/dist/setup/providers.js +14 -0
- package/dist/setup/registry-stash-loader.js +12 -0
- package/dist/setup/semantic-assets.js +124 -0
- package/dist/setup/setup.js +25 -1608
- package/dist/setup/steps/connection.js +734 -0
- package/dist/setup/steps/output.js +31 -0
- package/dist/setup/steps/platforms.js +124 -0
- package/dist/setup/steps/semantic.js +27 -0
- package/dist/setup/steps/sources.js +222 -0
- package/dist/setup/steps/stashdir.js +42 -0
- package/dist/setup/steps/tasks.js +152 -0
- package/dist/storage/repositories/canaries-repository.js +107 -0
- package/dist/storage/repositories/consolidation-repository.js +38 -0
- package/dist/storage/repositories/embeddings-repository.js +72 -0
- package/dist/storage/repositories/events-repository.js +187 -0
- package/dist/storage/repositories/extract-sessions-repository.js +96 -0
- package/dist/storage/repositories/improve-runs-repository.js +130 -0
- package/dist/storage/repositories/index-db.js +4 -7
- package/dist/storage/repositories/proposals-repository.js +220 -0
- package/dist/storage/repositories/recombine-repository.js +213 -0
- package/dist/storage/repositories/task-history-repository.js +93 -0
- package/dist/storage/sqlite-pragmas.js +3 -3
- package/dist/tasks/backends/index.js +9 -0
- package/dist/tasks/runner.js +11 -1
- package/package.json +2 -2
- package/dist/commands/improve/homeostatic.js +0 -497
package/dist/core/state-db.js
CHANGED
|
@@ -4,11 +4,17 @@
|
|
|
4
4
|
/**
|
|
5
5
|
* state.db — Durable SQLite database for non-regenerable akm state.
|
|
6
6
|
*
|
|
7
|
-
* This module
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
7
|
+
* This module OWNS the state database's shared infrastructure: path resolution,
|
|
8
|
+
* the managed-db open/loan wrappers, the `BEGIN IMMEDIATE` transaction helper,
|
|
9
|
+
* and schema introspection. The table-specific query helpers live by domain in
|
|
10
|
+
* `src/storage/repositories/*-repository.ts` (events, proposals, task-history,
|
|
11
|
+
* improve-runs, extract-sessions, consolidation, recombine, embeddings,
|
|
12
|
+
* canaries); importers reference those modules directly. The migration engine
|
|
13
|
+
* lives in `./state/migrations`.
|
|
14
|
+
*
|
|
15
|
+
* The state DB replaces flat-file storage for data that is NON-REGENERABLE —
|
|
16
|
+
* events (events.jsonl), proposals (per-uuid JSON directories), task history
|
|
17
|
+
* (per-task JSONL), and the improve-pipeline ledgers.
|
|
12
18
|
*
|
|
13
19
|
* ## Why a separate database from index.db
|
|
14
20
|
*
|
|
@@ -51,9 +57,7 @@
|
|
|
51
57
|
*/
|
|
52
58
|
import path from "node:path";
|
|
53
59
|
import { openManagedDatabase, withManagedDb, withManagedDbAsync } from "../storage/managed-db.js";
|
|
54
|
-
import { classifyImproveAction } from "./improve-types.js";
|
|
55
60
|
import { getDataDir } from "./paths.js";
|
|
56
|
-
import { error } from "./warn.js";
|
|
57
61
|
// ── Path helper ──────────────────────────────────────────────────────────────
|
|
58
62
|
/**
|
|
59
63
|
* Default path: `<dataDir>/state.db`.
|
|
@@ -116,350 +120,9 @@ export function withStateDbAsync(fn, opts) {
|
|
|
116
120
|
//
|
|
117
121
|
// The MIGRATIONS registry + runMigrations live in ./state/migrations (the single
|
|
118
122
|
// append-only ordered source of truth). Imported for internal use by
|
|
119
|
-
// openStateDatabase
|
|
123
|
+
// openStateDatabase.
|
|
120
124
|
import { runMigrations } from "./state/migrations.js";
|
|
121
|
-
|
|
122
|
-
/**
|
|
123
|
-
* Convert a raw `EventRow` from the database to the public `EventEnvelope`
|
|
124
|
-
* interface used throughout the events module.
|
|
125
|
-
*/
|
|
126
|
-
export function eventRowToEnvelope(row) {
|
|
127
|
-
let metadata;
|
|
128
|
-
try {
|
|
129
|
-
const parsed = JSON.parse(row.metadata_json);
|
|
130
|
-
// Only attach metadata when the JSON blob is non-empty so downstream
|
|
131
|
-
// consumers that check `envelope.metadata !== undefined` keep working.
|
|
132
|
-
if (Object.keys(parsed).length > 0) {
|
|
133
|
-
metadata = parsed;
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
catch {
|
|
137
|
-
// Corrupt JSON in the DB — treat as no metadata.
|
|
138
|
-
}
|
|
139
|
-
return {
|
|
140
|
-
schemaVersion: 1,
|
|
141
|
-
id: row.id,
|
|
142
|
-
ts: row.ts,
|
|
143
|
-
eventType: row.event_type,
|
|
144
|
-
...(row.ref !== null ? { ref: row.ref } : {}),
|
|
145
|
-
...(metadata !== undefined ? { metadata } : {}),
|
|
146
|
-
};
|
|
147
|
-
}
|
|
148
|
-
/**
|
|
149
|
-
* Convert a raw `ProposalRow` to the public `Proposal` shape.
|
|
150
|
-
*/
|
|
151
|
-
export function proposalRowToProposal(row) {
|
|
152
|
-
let frontmatter;
|
|
153
|
-
if (row.frontmatter_json) {
|
|
154
|
-
try {
|
|
155
|
-
frontmatter = JSON.parse(row.frontmatter_json);
|
|
156
|
-
}
|
|
157
|
-
catch {
|
|
158
|
-
/* ignore corrupt frontmatter JSON */
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
let meta = {};
|
|
162
|
-
try {
|
|
163
|
-
meta = JSON.parse(row.metadata_json);
|
|
164
|
-
}
|
|
165
|
-
catch {
|
|
166
|
-
/* ignore */
|
|
167
|
-
}
|
|
168
|
-
return {
|
|
169
|
-
id: row.id,
|
|
170
|
-
ref: row.ref,
|
|
171
|
-
status: row.status,
|
|
172
|
-
source: row.source,
|
|
173
|
-
...(typeof meta.sourceRun === "string" ? { sourceRun: meta.sourceRun } : {}),
|
|
174
|
-
createdAt: row.created_at,
|
|
175
|
-
updatedAt: row.updated_at,
|
|
176
|
-
payload: {
|
|
177
|
-
content: row.content,
|
|
178
|
-
...(frontmatter !== undefined ? { frontmatter } : {}),
|
|
179
|
-
},
|
|
180
|
-
...(meta.review !== undefined ? { review: meta.review } : {}),
|
|
181
|
-
...(typeof meta.confidence === "number" ? { confidence: meta.confidence } : {}),
|
|
182
|
-
...(meta.gateDecision !== undefined ? { gateDecision: meta.gateDecision } : {}),
|
|
183
|
-
...(typeof meta.backupContent === "string" ? { backupContent: meta.backupContent } : {}),
|
|
184
|
-
...(typeof meta.eligibilitySource === "string"
|
|
185
|
-
? { eligibilitySource: meta.eligibilitySource }
|
|
186
|
-
: {}),
|
|
187
|
-
};
|
|
188
|
-
}
|
|
189
|
-
/**
|
|
190
|
-
* Convert a public `Proposal` to column values ready for an INSERT/UPDATE.
|
|
191
|
-
* The `stash_dir` comes from the call site (proposals.ts has it in scope).
|
|
192
|
-
*/
|
|
193
|
-
export function proposalToRowValues(proposal, stashDir) {
|
|
194
|
-
// Fields that have no dedicated column live in metadata_json.
|
|
195
|
-
const metaObj = {};
|
|
196
|
-
if (proposal.sourceRun !== undefined)
|
|
197
|
-
metaObj.sourceRun = proposal.sourceRun;
|
|
198
|
-
if (proposal.review !== undefined)
|
|
199
|
-
metaObj.review = proposal.review;
|
|
200
|
-
if (proposal.confidence !== undefined)
|
|
201
|
-
metaObj.confidence = proposal.confidence;
|
|
202
|
-
if (proposal.gateDecision !== undefined)
|
|
203
|
-
metaObj.gateDecision = proposal.gateDecision;
|
|
204
|
-
if (proposal.backupContent !== undefined)
|
|
205
|
-
metaObj.backupContent = proposal.backupContent;
|
|
206
|
-
if (proposal.eligibilitySource !== undefined)
|
|
207
|
-
metaObj.eligibilitySource = proposal.eligibilitySource;
|
|
208
|
-
return {
|
|
209
|
-
id: proposal.id,
|
|
210
|
-
stash_dir: stashDir,
|
|
211
|
-
ref: proposal.ref,
|
|
212
|
-
status: proposal.status,
|
|
213
|
-
source: proposal.source,
|
|
214
|
-
created_at: proposal.createdAt,
|
|
215
|
-
updated_at: proposal.updatedAt,
|
|
216
|
-
content: proposal.payload.content,
|
|
217
|
-
frontmatter_json: proposal.payload.frontmatter ? JSON.stringify(proposal.payload.frontmatter) : null,
|
|
218
|
-
metadata_json: JSON.stringify(metaObj),
|
|
219
|
-
};
|
|
220
|
-
}
|
|
221
|
-
// ── events table helpers ─────────────────────────────────────────────────────
|
|
222
|
-
/**
|
|
223
|
-
* Insert a single event. Returns the auto-assigned monotonic rowid, which
|
|
224
|
-
* callers can store as a "sinceId" cursor for future `readEventsSince` calls.
|
|
225
|
-
*
|
|
226
|
-
* Best-effort: mirrors the behaviour of the old `appendEvent` — errors are
|
|
227
|
-
* caught and logged to stderr rather than propagated so observability never
|
|
228
|
-
* breaks mutation.
|
|
229
|
-
*/
|
|
230
|
-
export function insertEvent(db, input) {
|
|
231
|
-
try {
|
|
232
|
-
const result = db
|
|
233
|
-
.prepare(`INSERT INTO events (event_type, ts, ref, metadata_json)
|
|
234
|
-
VALUES (?, ?, ?, ?)
|
|
235
|
-
RETURNING id`)
|
|
236
|
-
.get(input.eventType, input.ts, input.ref ?? null, JSON.stringify(input.metadata ?? {}));
|
|
237
|
-
return result?.id;
|
|
238
|
-
}
|
|
239
|
-
catch (err) {
|
|
240
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
241
|
-
error(`akm: state.db event insert failed (${message})`);
|
|
242
|
-
return undefined;
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
/**
|
|
246
|
-
* Read events from the database matching the filter. Returns events in
|
|
247
|
-
* ascending id order so consumers can process them in emission order.
|
|
248
|
-
*
|
|
249
|
-
* The returned `nextId` is the maximum id seen (or `sinceId` when no rows
|
|
250
|
-
* match), suitable as the next `sinceId` cursor value.
|
|
251
|
-
*/
|
|
252
|
-
export function readStateEvents(db, options = {}) {
|
|
253
|
-
const conditions = [];
|
|
254
|
-
const params = [];
|
|
255
|
-
if (options.sinceId !== undefined && options.sinceId > 0) {
|
|
256
|
-
conditions.push("id > ?");
|
|
257
|
-
params.push(options.sinceId);
|
|
258
|
-
}
|
|
259
|
-
if (options.since) {
|
|
260
|
-
conditions.push("ts >= ?");
|
|
261
|
-
params.push(options.since);
|
|
262
|
-
}
|
|
263
|
-
if (options.type) {
|
|
264
|
-
conditions.push("event_type = ?");
|
|
265
|
-
params.push(options.type);
|
|
266
|
-
}
|
|
267
|
-
if (options.ref) {
|
|
268
|
-
conditions.push("ref = ?");
|
|
269
|
-
params.push(options.ref);
|
|
270
|
-
}
|
|
271
|
-
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
272
|
-
const rows = db
|
|
273
|
-
.prepare(`SELECT id, event_type, ts, ref, metadata_json FROM events ${where} ORDER BY id ASC`)
|
|
274
|
-
.all(...params);
|
|
275
|
-
const events = rows.map(eventRowToEnvelope);
|
|
276
|
-
const nextId = events.length > 0 ? events[events.length - 1].id : (options.sinceId ?? 0);
|
|
277
|
-
return { events, nextId };
|
|
278
|
-
}
|
|
279
|
-
/**
|
|
280
|
-
* Delete events older than `retentionDays` (default: 90). Safe to call from
|
|
281
|
-
* a maintenance cron; uses a single DELETE with an index-covered ts predicate.
|
|
282
|
-
*
|
|
283
|
-
* Returns the number of rows actually deleted so callers can emit an
|
|
284
|
-
* `events_purged` observability event. A non-positive or non-finite
|
|
285
|
-
* `retentionDays` is treated as "disabled" and returns 0 without scanning.
|
|
286
|
-
*/
|
|
287
|
-
export function purgeOldEvents(db, retentionDays = 90) {
|
|
288
|
-
if (!Number.isFinite(retentionDays) || retentionDays <= 0)
|
|
289
|
-
return 0;
|
|
290
|
-
const cutoff = new Date(Date.now() - retentionDays * 86_400_000).toISOString();
|
|
291
|
-
const result = db.prepare("DELETE FROM events WHERE ts < ?").run(cutoff);
|
|
292
|
-
// bun:sqlite's run() returns { changes, lastInsertRowid }. `changes` may be
|
|
293
|
-
// a number or bigint depending on the underlying lib; coerce to number for
|
|
294
|
-
// the metadata payload.
|
|
295
|
-
const changes = result.changes ?? 0;
|
|
296
|
-
return typeof changes === "bigint" ? Number(changes) : changes;
|
|
297
|
-
}
|
|
298
|
-
// ── proposals table helpers ──────────────────────────────────────────────────
|
|
299
|
-
/**
|
|
300
|
-
* Upsert a proposal row. Called by the proposal write path when state.db is
|
|
301
|
-
* the active backend.
|
|
302
|
-
*/
|
|
303
|
-
export function upsertProposal(db, proposal, stashDir) {
|
|
304
|
-
const v = proposalToRowValues(proposal, stashDir);
|
|
305
|
-
db.prepare(`
|
|
306
|
-
INSERT INTO proposals
|
|
307
|
-
(id, stash_dir, ref, status, source, created_at, updated_at, content, frontmatter_json, metadata_json)
|
|
308
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
309
|
-
ON CONFLICT(id) DO UPDATE SET
|
|
310
|
-
stash_dir = excluded.stash_dir,
|
|
311
|
-
ref = excluded.ref,
|
|
312
|
-
status = excluded.status,
|
|
313
|
-
source = excluded.source,
|
|
314
|
-
updated_at = excluded.updated_at,
|
|
315
|
-
content = excluded.content,
|
|
316
|
-
frontmatter_json = excluded.frontmatter_json,
|
|
317
|
-
metadata_json = excluded.metadata_json
|
|
318
|
-
`).run(v.id, v.stash_dir, v.ref, v.status, v.source, v.created_at, v.updated_at, v.content, v.frontmatter_json, v.metadata_json);
|
|
319
|
-
}
|
|
320
|
-
/**
|
|
321
|
-
* List proposals, optionally filtered by stashDir, status, and/or ref.
|
|
322
|
-
*
|
|
323
|
-
* Results are ordered by `created_at ASC` (matching the historical
|
|
324
|
-
* `listProposals()` sort), with `rowid` as a deterministic tiebreak so two
|
|
325
|
-
* proposals created in the same millisecond list in insertion order.
|
|
326
|
-
*/
|
|
327
|
-
export function listStateProposals(db, options = {}) {
|
|
328
|
-
const conditions = [];
|
|
329
|
-
const params = [];
|
|
330
|
-
if (options.stashDir) {
|
|
331
|
-
conditions.push("stash_dir = ?");
|
|
332
|
-
params.push(options.stashDir);
|
|
333
|
-
}
|
|
334
|
-
if (options.status) {
|
|
335
|
-
conditions.push("status = ?");
|
|
336
|
-
params.push(options.status);
|
|
337
|
-
}
|
|
338
|
-
if (options.ref) {
|
|
339
|
-
conditions.push("ref = ?");
|
|
340
|
-
params.push(options.ref);
|
|
341
|
-
}
|
|
342
|
-
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
343
|
-
const rows = db
|
|
344
|
-
.prepare(`SELECT id, stash_dir, ref, status, source, created_at, updated_at,
|
|
345
|
-
content, frontmatter_json, metadata_json
|
|
346
|
-
FROM proposals ${where} ORDER BY created_at ASC, rowid ASC`)
|
|
347
|
-
.all(...params);
|
|
348
|
-
return rows.map(proposalRowToProposal);
|
|
349
|
-
}
|
|
350
|
-
/**
|
|
351
|
-
* Read every proposal's `gateDecision` record across all stashes (#612).
|
|
352
|
-
*
|
|
353
|
-
* Calibration reads the auto-accept gate's per-proposal decisions regardless of
|
|
354
|
-
* the proposal's current lifecycle status — a proposal that was auto-accepted
|
|
355
|
-
* is now `accepted`, an auto-rejected one stays `pending`, so filtering by
|
|
356
|
-
* status would drop half the join. Rows without a `gateDecision` (created
|
|
357
|
-
* before #577, or never gated) are skipped. The result is ordered by
|
|
358
|
-
* `decidedAt ASC` for deterministic downstream aggregation, falling back to
|
|
359
|
-
* `created_at` ordering from the SQL layer for rows with equal/missing
|
|
360
|
-
* timestamps.
|
|
361
|
-
*/
|
|
362
|
-
export function listProposalGateDecisions(db) {
|
|
363
|
-
const rows = db.prepare("SELECT metadata_json FROM proposals ORDER BY created_at ASC, rowid ASC").all();
|
|
364
|
-
const decisions = [];
|
|
365
|
-
for (const row of rows) {
|
|
366
|
-
let meta;
|
|
367
|
-
try {
|
|
368
|
-
meta = JSON.parse(row.metadata_json);
|
|
369
|
-
}
|
|
370
|
-
catch {
|
|
371
|
-
continue;
|
|
372
|
-
}
|
|
373
|
-
const decision = meta.gateDecision;
|
|
374
|
-
if (decision && typeof decision === "object" && typeof decision.outcome === "string") {
|
|
375
|
-
decisions.push(decision);
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
decisions.sort((a, b) => new Date(a.decidedAt).getTime() - new Date(b.decidedAt).getTime());
|
|
379
|
-
return decisions;
|
|
380
|
-
}
|
|
381
|
-
// ── WS-4: Per-phase gate threshold store (Migration 012) ─────────────────────
|
|
382
|
-
/**
|
|
383
|
-
* Read the persisted auto-tuned threshold for a gate phase.
|
|
384
|
-
*
|
|
385
|
-
* Returns `undefined` when no row exists yet (first run, or the phase has
|
|
386
|
-
* never been tuned). The caller falls back to the global `options.autoAccept`
|
|
387
|
-
* in that case.
|
|
388
|
-
*/
|
|
389
|
-
export function getPhaseThreshold(db, phase) {
|
|
390
|
-
const row = db.prepare("SELECT threshold FROM improve_gate_thresholds WHERE phase = ?").get(phase);
|
|
391
|
-
return row?.threshold;
|
|
392
|
-
}
|
|
393
|
-
/**
|
|
394
|
-
* Persist the auto-tuned threshold for a gate phase.
|
|
395
|
-
* Uses INSERT OR REPLACE so the call is idempotent (upsert semantics).
|
|
396
|
-
*/
|
|
397
|
-
export function persistPhaseThreshold(db, phase, threshold) {
|
|
398
|
-
db.prepare(`INSERT OR REPLACE INTO improve_gate_thresholds (phase, threshold, updated_at)
|
|
399
|
-
VALUES (?, ?, ?)`).run(phase, Math.round(threshold), Date.now());
|
|
400
|
-
}
|
|
401
|
-
/**
|
|
402
|
-
* Look up a single proposal by id, optionally scoped to one stash root.
|
|
403
|
-
* Returns undefined when not found.
|
|
404
|
-
*/
|
|
405
|
-
export function getStateProposal(db, id, stashDir) {
|
|
406
|
-
const sql = `SELECT id, stash_dir, ref, status, source, created_at, updated_at,
|
|
407
|
-
content, frontmatter_json, metadata_json
|
|
408
|
-
FROM proposals WHERE id = ?${stashDir ? " AND stash_dir = ?" : ""}`;
|
|
409
|
-
const row = (stashDir ? db.prepare(sql).get(id, stashDir) : db.prepare(sql).get(id));
|
|
410
|
-
return row ? proposalRowToProposal(row) : undefined;
|
|
411
|
-
}
|
|
412
|
-
/**
|
|
413
|
-
* Find PENDING proposal ids in one stash whose id starts with `idPrefix`.
|
|
414
|
-
* Backs the UUID-prefix form of `akm proposal show/accept/... <prefix>` —
|
|
415
|
-
* prefix resolution is deliberately scoped to the live (pending) queue,
|
|
416
|
-
* mirroring the historical behaviour of scanning only the live directory.
|
|
417
|
-
*
|
|
418
|
-
* `%` / `_` / `\` in the prefix are escaped so the LIKE pattern is literal.
|
|
419
|
-
*/
|
|
420
|
-
export function listStateProposalIdsByPrefix(db, stashDir, idPrefix) {
|
|
421
|
-
const escaped = idPrefix.replace(/[\\%_]/g, (ch) => `\\${ch}`);
|
|
422
|
-
const rows = db
|
|
423
|
-
.prepare(`SELECT id FROM proposals
|
|
424
|
-
WHERE stash_dir = ? AND status = 'pending' AND id LIKE ? ESCAPE '\\'
|
|
425
|
-
ORDER BY id ASC`)
|
|
426
|
-
.all(stashDir, `${escaped}%`);
|
|
427
|
-
return rows.map((r) => r.id);
|
|
428
|
-
}
|
|
429
|
-
/**
|
|
430
|
-
* Whether the legacy filesystem proposal import has already run for `stashDir`.
|
|
431
|
-
* See migration 005 (`proposal_fs_imports`).
|
|
432
|
-
*/
|
|
433
|
-
export function hasImportedFsProposals(db, stashDir) {
|
|
434
|
-
// Drivers disagree on the no-row sentinel (bun:sqlite → null,
|
|
435
|
-
// better-sqlite3 → undefined) — Boolean() covers both.
|
|
436
|
-
return Boolean(db.prepare("SELECT 1 FROM proposal_fs_imports WHERE stash_dir = ?").get(stashDir));
|
|
437
|
-
}
|
|
438
|
-
/**
|
|
439
|
-
* Record that the legacy filesystem proposal import completed for `stashDir`
|
|
440
|
-
* so subsequent invocations skip the directory walk. INSERT OR REPLACE keeps
|
|
441
|
-
* the call idempotent.
|
|
442
|
-
*/
|
|
443
|
-
export function recordFsProposalsImport(db, stashDir, importedCount) {
|
|
444
|
-
db.prepare("INSERT OR REPLACE INTO proposal_fs_imports (stash_dir, imported_at, imported_count) VALUES (?, ?, ?)").run(stashDir, new Date().toISOString(), importedCount);
|
|
445
|
-
}
|
|
446
|
-
/**
|
|
447
|
-
* Insert a proposal row ONLY when the id is not already present (used by the
|
|
448
|
-
* legacy filesystem import so re-runs never clobber rows that have since been
|
|
449
|
-
* mutated through the canonical store). Returns true when a row was inserted.
|
|
450
|
-
*/
|
|
451
|
-
export function insertProposalIfAbsent(db, proposal, stashDir) {
|
|
452
|
-
const v = proposalToRowValues(proposal, stashDir);
|
|
453
|
-
const result = db
|
|
454
|
-
.prepare(`
|
|
455
|
-
INSERT OR IGNORE INTO proposals
|
|
456
|
-
(id, stash_dir, ref, status, source, created_at, updated_at, content, frontmatter_json, metadata_json)
|
|
457
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
458
|
-
`)
|
|
459
|
-
.run(v.id, v.stash_dir, v.ref, v.status, v.source, v.created_at, v.updated_at, v.content, v.frontmatter_json, v.metadata_json);
|
|
460
|
-
const changes = result.changes ?? 0;
|
|
461
|
-
return Number(changes) > 0;
|
|
462
|
-
}
|
|
125
|
+
// ── BEGIN IMMEDIATE transaction helper ───────────────────────────────────────
|
|
463
126
|
/**
|
|
464
127
|
* Run `fn` inside a `BEGIN IMMEDIATE` transaction.
|
|
465
128
|
*
|
|
@@ -569,97 +232,6 @@ export function withImmediateTransaction(db, fn) {
|
|
|
569
232
|
// Exhausted retries on transient begin failures.
|
|
570
233
|
throw lastBeginErr;
|
|
571
234
|
}
|
|
572
|
-
// ── task_history table helpers ───────────────────────────────────────────────
|
|
573
|
-
/**
|
|
574
|
-
* Upsert a task history row.
|
|
575
|
-
*/
|
|
576
|
-
export function upsertTaskHistory(db, row) {
|
|
577
|
-
// INSERT OR IGNORE: if a run with the same (task_id, started_at) was already
|
|
578
|
-
// imported (e.g. by the migration script), skip it silently.
|
|
579
|
-
db.prepare(`
|
|
580
|
-
INSERT OR IGNORE INTO task_history
|
|
581
|
-
(task_id, status, started_at, completed_at, failed_at, log_path,
|
|
582
|
-
target_kind, target_ref, metadata_json)
|
|
583
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
584
|
-
`).run(row.task_id, row.status, row.started_at, row.completed_at ?? null, row.failed_at ?? null, row.log_path ?? null, row.target_kind ?? null, row.target_ref ?? null, row.metadata_json);
|
|
585
|
-
}
|
|
586
|
-
/**
|
|
587
|
-
* Look up a task history row by task_id. Returns undefined when not found.
|
|
588
|
-
*/
|
|
589
|
-
/**
|
|
590
|
-
* Return the most recent run for a given task_id, or undefined if no runs exist.
|
|
591
|
-
*/
|
|
592
|
-
export function getTaskHistory(db, taskId) {
|
|
593
|
-
return db
|
|
594
|
-
.prepare(`SELECT id, task_id, status, started_at, completed_at, failed_at, log_path,
|
|
595
|
-
target_kind, target_ref, metadata_json
|
|
596
|
-
FROM task_history WHERE task_id = ? ORDER BY started_at DESC LIMIT 1`)
|
|
597
|
-
.get(taskId);
|
|
598
|
-
}
|
|
599
|
-
/**
|
|
600
|
-
* Return all runs for a given task_id, newest first.
|
|
601
|
-
*/
|
|
602
|
-
export function getTaskHistoryRuns(db, taskId, limit = 50) {
|
|
603
|
-
return db
|
|
604
|
-
.prepare(`SELECT id, task_id, status, started_at, completed_at, failed_at, log_path,
|
|
605
|
-
target_kind, target_ref, metadata_json
|
|
606
|
-
FROM task_history WHERE task_id = ? ORDER BY started_at DESC LIMIT ?`)
|
|
607
|
-
.all(taskId, limit);
|
|
608
|
-
}
|
|
609
|
-
/**
|
|
610
|
-
* Query task history rows by started_at range and/or status.
|
|
611
|
-
*/
|
|
612
|
-
export function queryTaskHistory(db, options = {}) {
|
|
613
|
-
const conditions = [];
|
|
614
|
-
const params = [];
|
|
615
|
-
if (options.since) {
|
|
616
|
-
conditions.push("started_at >= ?");
|
|
617
|
-
params.push(options.since);
|
|
618
|
-
}
|
|
619
|
-
if (options.until) {
|
|
620
|
-
conditions.push("started_at <= ?");
|
|
621
|
-
params.push(options.until);
|
|
622
|
-
}
|
|
623
|
-
if (options.status) {
|
|
624
|
-
conditions.push("status = ?");
|
|
625
|
-
params.push(options.status);
|
|
626
|
-
}
|
|
627
|
-
if (options.targetKind) {
|
|
628
|
-
conditions.push("target_kind = ?");
|
|
629
|
-
params.push(options.targetKind);
|
|
630
|
-
}
|
|
631
|
-
if (options.targetRef) {
|
|
632
|
-
conditions.push("target_ref = ?");
|
|
633
|
-
params.push(options.targetRef);
|
|
634
|
-
}
|
|
635
|
-
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
636
|
-
return db
|
|
637
|
-
.prepare(`SELECT task_id, status, started_at, completed_at, failed_at, log_path,
|
|
638
|
-
target_kind, target_ref, metadata_json
|
|
639
|
-
FROM task_history ${where} ORDER BY started_at DESC`)
|
|
640
|
-
.all(...params);
|
|
641
|
-
}
|
|
642
|
-
/**
|
|
643
|
-
* Read COMPLETED `akm-improve` task_history runs whose `started_at` falls in
|
|
644
|
-
* `[since, until)` (or `started_at >= since` when `until` is omitted), ordered
|
|
645
|
-
* oldest-first by `started_at`. Only rows with a non-null `completed_at` are
|
|
646
|
-
* returned (in-flight runs are excluded). The `task_id = 'akm-improve'`
|
|
647
|
-
* predicate is fixed because the only caller (commands/health.ts
|
|
648
|
-
* `loadTaskIntervals`) builds wall-time intervals for the improve cron task.
|
|
649
|
-
*
|
|
650
|
-
* Owns the SQL formerly inlined in commands/health.ts. Note the bound is
|
|
651
|
-
* EXCLUSIVE on the upper end (`started_at < ?`) — callers pass an already
|
|
652
|
-
* widened window; this helper does not widen.
|
|
653
|
-
*
|
|
654
|
-
* Connection-lifetime rule (WS5): `.all()` materializes a plain array before
|
|
655
|
-
* returning.
|
|
656
|
-
*/
|
|
657
|
-
export function queryCompletedTaskIntervals(db, since, until) {
|
|
658
|
-
const sql = until
|
|
659
|
-
? "SELECT started_at, completed_at FROM task_history WHERE task_id = 'akm-improve' AND started_at >= ? AND started_at < ? AND completed_at IS NOT NULL ORDER BY started_at"
|
|
660
|
-
: "SELECT started_at, completed_at FROM task_history WHERE task_id = 'akm-improve' AND started_at >= ? AND completed_at IS NOT NULL ORDER BY started_at";
|
|
661
|
-
return (until ? db.prepare(sql).all(since, until) : db.prepare(sql).all(since));
|
|
662
|
-
}
|
|
663
235
|
// ── schema introspection ─────────────────────────────────────────────────────
|
|
664
236
|
/**
|
|
665
237
|
* Return the subset of `names` that exist as TABLEs in this database, ordered
|
|
@@ -680,702 +252,3 @@ export function listExistingTableNames(db, names) {
|
|
|
680
252
|
.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (${placeholders}) ORDER BY name`)
|
|
681
253
|
.all(...names);
|
|
682
254
|
}
|
|
683
|
-
// ── events.jsonl import ──────────────────────────────────────────────────────
|
|
684
|
-
/**
|
|
685
|
-
* Import all events from an `events.jsonl` file into the `events` table.
|
|
686
|
-
*
|
|
687
|
-
* The old byte-offset `id` is NOT preserved — the database assigns new
|
|
688
|
-
* monotonic integer ids. Callers that persisted a byte-offset cursor must
|
|
689
|
-
* discard it after migration and use the returned `maxId` as the new cursor.
|
|
690
|
-
*
|
|
691
|
-
* **Idempotency**: each line is pre-checked against the `events` table using
|
|
692
|
-
* `(event_type, ts, ref, metadata_json)` as the duplicate key. Lines whose
|
|
693
|
-
* exact tuple is already present are skipped and reported as `skipped` in the
|
|
694
|
-
* return value. This makes the migration safe to re-run (the v0.7→v0.8
|
|
695
|
-
* migration guide recommends re-running the script as a recovery path; without
|
|
696
|
-
* this guard, every re-run would double-import the entire event log).
|
|
697
|
-
*
|
|
698
|
-
* Duplicate detection is per-import-tuple, not a table-wide UNIQUE constraint:
|
|
699
|
-
* the events table has no UNIQUE constraint at runtime so that
|
|
700
|
-
* `appendEvent` can write multiple events with the same ts (sub-millisecond
|
|
701
|
-
* bursts produce identical `(event_type, ts, ref)` triples in practice). The
|
|
702
|
-
* SELECT-first check is scoped to the import path only.
|
|
703
|
-
*
|
|
704
|
-
* The import is wrapped in a single transaction for atomicity.
|
|
705
|
-
*
|
|
706
|
-
* @param db - Open state.db connection.
|
|
707
|
-
* @param jsonlPath - Absolute path to the events.jsonl file to import.
|
|
708
|
-
* @returns Number of rows inserted, the max id assigned, and the
|
|
709
|
-
* count of rows skipped because an identical event already
|
|
710
|
-
* existed in the table.
|
|
711
|
-
*/
|
|
712
|
-
export async function importEventsJsonl(db, jsonlPath) {
|
|
713
|
-
const { readFileSync, existsSync } = await import("node:fs");
|
|
714
|
-
if (!existsSync(jsonlPath)) {
|
|
715
|
-
return { imported: 0, maxId: 0, skipped: 0 };
|
|
716
|
-
}
|
|
717
|
-
const text = readFileSync(jsonlPath, "utf8");
|
|
718
|
-
const lines = text.split("\n").filter((l) => l.trim().length > 0);
|
|
719
|
-
let imported = 0;
|
|
720
|
-
let maxId = 0;
|
|
721
|
-
let skipped = 0;
|
|
722
|
-
const insertStmt = db.prepare(`INSERT INTO events (event_type, ts, ref, metadata_json)
|
|
723
|
-
VALUES (?, ?, ?, ?)
|
|
724
|
-
RETURNING id`);
|
|
725
|
-
// Dedup pre-check: matches by the full tuple including metadata_json so an
|
|
726
|
-
// import is idempotent over identical rows but does not collide with two
|
|
727
|
-
// genuinely different events that happen to share (event_type, ts, ref).
|
|
728
|
-
//
|
|
729
|
-
// Uses IS for ref so two NULL refs compare equal (a plain `=` would treat
|
|
730
|
-
// NULL = NULL as NULL and the row would be re-inserted on every run).
|
|
731
|
-
const existsStmt = db.prepare(`SELECT 1 FROM events
|
|
732
|
-
WHERE event_type = ?
|
|
733
|
-
AND ts = ?
|
|
734
|
-
AND ref IS ?
|
|
735
|
-
AND metadata_json = ?
|
|
736
|
-
LIMIT 1`);
|
|
737
|
-
db.transaction(() => {
|
|
738
|
-
for (const line of lines) {
|
|
739
|
-
let parsed;
|
|
740
|
-
try {
|
|
741
|
-
parsed = JSON.parse(line);
|
|
742
|
-
}
|
|
743
|
-
catch {
|
|
744
|
-
continue; // skip malformed lines — same behaviour as readEvents()
|
|
745
|
-
}
|
|
746
|
-
const eventType = typeof parsed.eventType === "string" ? parsed.eventType : "unknown";
|
|
747
|
-
const ts = typeof parsed.ts === "string" ? parsed.ts : new Date().toISOString();
|
|
748
|
-
const ref = typeof parsed.ref === "string" ? parsed.ref : null;
|
|
749
|
-
const metadata = parsed.metadata !== undefined && typeof parsed.metadata === "object" ? JSON.stringify(parsed.metadata) : "{}";
|
|
750
|
-
const duplicate = existsStmt.get(eventType, ts, ref, metadata);
|
|
751
|
-
if (duplicate) {
|
|
752
|
-
skipped++;
|
|
753
|
-
continue;
|
|
754
|
-
}
|
|
755
|
-
const result = insertStmt.get(eventType, ts, ref, metadata);
|
|
756
|
-
if (result) {
|
|
757
|
-
imported++;
|
|
758
|
-
if (result.id > maxId)
|
|
759
|
-
maxId = result.id;
|
|
760
|
-
}
|
|
761
|
-
}
|
|
762
|
-
})();
|
|
763
|
-
return { imported, maxId, skipped };
|
|
764
|
-
}
|
|
765
|
-
/**
|
|
766
|
-
* Compute the cheap aggregate metrics blob from a full improve result.
|
|
767
|
-
*
|
|
768
|
-
* Pure function — no I/O. Used by {@link recordImproveRun} to populate
|
|
769
|
-
* `metrics_json`. Exposed for tests and for any future call site that wants
|
|
770
|
-
* the same aggregation logic without hitting state.db.
|
|
771
|
-
*/
|
|
772
|
-
export function computeImproveRunMetrics(result) {
|
|
773
|
-
const plannedCount = Array.isArray(result.plannedRefs) ? result.plannedRefs.length : 0;
|
|
774
|
-
const actions = Array.isArray(result.actions) ? result.actions : [];
|
|
775
|
-
const actionsCount = actions.length;
|
|
776
|
-
let acceptedCount = 0;
|
|
777
|
-
let rejectedCount = 0;
|
|
778
|
-
let skippedCount = 0;
|
|
779
|
-
let autoAcceptedCount = 0;
|
|
780
|
-
let errorCount = 0;
|
|
781
|
-
for (const action of actions) {
|
|
782
|
-
// Bucketing delegated to the shared classifyImproveAction so this aggregate
|
|
783
|
-
// and the improve_completed event in improve.ts can never disagree, and so a
|
|
784
|
-
// new union variant is a compile error rather than a silent drop. Gated skips
|
|
785
|
-
// (cooldown / signal-delta / distill pool-delta) bucket to "skipped", NOT
|
|
786
|
-
// "rejected" — only a guard-rejected produced change is a true rejection.
|
|
787
|
-
// "noop" (memory-prune) is intentionally counted in none of the buckets.
|
|
788
|
-
switch (classifyImproveAction(action.mode)) {
|
|
789
|
-
case "accepted":
|
|
790
|
-
acceptedCount++;
|
|
791
|
-
break;
|
|
792
|
-
case "rejected":
|
|
793
|
-
rejectedCount++;
|
|
794
|
-
break;
|
|
795
|
-
case "skipped":
|
|
796
|
-
skippedCount++;
|
|
797
|
-
break;
|
|
798
|
-
case "error":
|
|
799
|
-
errorCount++;
|
|
800
|
-
break;
|
|
801
|
-
case "noop":
|
|
802
|
-
break;
|
|
803
|
-
}
|
|
804
|
-
// Legacy: pre-gate action results may carry autoAccepted: true (reflect path).
|
|
805
|
-
const r = action.result;
|
|
806
|
-
if (r && r.autoAccepted === true)
|
|
807
|
-
autoAcceptedCount++;
|
|
808
|
-
}
|
|
809
|
-
// Add gate-promoted count from the unified PostPhaseAutoAcceptGate (all phases).
|
|
810
|
-
autoAcceptedCount += result.gateAutoAcceptedCount ?? 0;
|
|
811
|
-
return { plannedCount, actionsCount, acceptedCount, rejectedCount, skippedCount, autoAcceptedCount, errorCount };
|
|
812
|
-
}
|
|
813
|
-
/**
|
|
814
|
-
* Insert a single improve-run row into `improve_runs`. Uses parameterised SQL.
|
|
815
|
-
*
|
|
816
|
-
* Idempotency: the table's PRIMARY KEY is `id`, so re-running with the same
|
|
817
|
-
* runId would error. Callers mint a fresh runId per invocation via
|
|
818
|
-
* {@link buildImproveRunId} so this is not a concern in practice — but the
|
|
819
|
-
* default behaviour is INSERT (not REPLACE) so accidental dupes surface as
|
|
820
|
-
* a SQLite constraint error rather than silently overwriting a prior record.
|
|
821
|
-
*
|
|
822
|
-
* The `metrics` parameter defaults to the output of
|
|
823
|
-
* {@link computeImproveRunMetrics} when not supplied. Pass an explicit
|
|
824
|
-
* `metrics` object to override the derivation (e.g. tests).
|
|
825
|
-
*/
|
|
826
|
-
export function recordImproveRun(db, input) {
|
|
827
|
-
const metricsObj = input.metrics ?? computeImproveRunMetrics(input.result);
|
|
828
|
-
db.prepare(`
|
|
829
|
-
INSERT INTO improve_runs
|
|
830
|
-
(id, started_at, completed_at, stash_dir, dry_run, profile,
|
|
831
|
-
scope_mode, scope_value, guidance, ok, result_json, metrics_json, metadata_json)
|
|
832
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
833
|
-
`).run(input.id, input.startedAt, input.completedAt, input.stashDir, input.dryRun ? 1 : 0, input.profile, input.scopeMode, input.scopeValue, input.guidance, input.ok ? 1 : 0, JSON.stringify(input.result), JSON.stringify(metricsObj), JSON.stringify(input.metadata ?? {}));
|
|
834
|
-
}
|
|
835
|
-
/**
|
|
836
|
-
* Read real (non-dry-run) improve_runs rows whose `started_at` falls in the
|
|
837
|
-
* window `[since, until)`. When `until` is omitted the window is open-ended
|
|
838
|
-
* (`started_at >= since`). Rows are returned newest-first (`ORDER BY
|
|
839
|
-
* started_at DESC`).
|
|
840
|
-
*
|
|
841
|
-
* Owns the SQL formerly inlined in commands/health.ts (`loadImproveRunRows`).
|
|
842
|
-
* The `dry_run = 0` filter is first-class so dry-run probes never pollute
|
|
843
|
-
* productivity audits.
|
|
844
|
-
*
|
|
845
|
-
* Connection-lifetime rule (WS5): `.all()` fully materializes the result set
|
|
846
|
-
* into a plain array before returning — no live cursor escapes the caller's
|
|
847
|
-
* `openStateDatabase` scope.
|
|
848
|
-
*/
|
|
849
|
-
export function queryImproveRuns(db, since, until) {
|
|
850
|
-
const sql = until
|
|
851
|
-
? "SELECT id, started_at, completed_at, ok, scope_mode, scope_value, result_json FROM improve_runs WHERE started_at >= ? AND started_at < ? AND dry_run = 0 ORDER BY started_at DESC"
|
|
852
|
-
: "SELECT id, started_at, completed_at, ok, scope_mode, scope_value, result_json FROM improve_runs WHERE started_at >= ? AND dry_run = 0 ORDER BY started_at DESC";
|
|
853
|
-
return (until ? db.prepare(sql).all(since, until) : db.prepare(sql).all(since));
|
|
854
|
-
}
|
|
855
|
-
/**
|
|
856
|
-
* Delete improve_runs rows older than `retentionDays` (default: 90). Mirrors
|
|
857
|
-
* {@link purgeOldEvents} — same default, same return shape (number of rows
|
|
858
|
-
* actually deleted), same disabled-when-non-finite semantics.
|
|
859
|
-
*
|
|
860
|
-
* Safe to call from the improve post-loop maintenance pass alongside
|
|
861
|
-
* `purgeOldEvents(db, retentionDays)`.
|
|
862
|
-
*/
|
|
863
|
-
export function purgeOldImproveRuns(db, retentionDays = 90) {
|
|
864
|
-
if (!Number.isFinite(retentionDays) || retentionDays <= 0)
|
|
865
|
-
return 0;
|
|
866
|
-
const cutoff = new Date(Date.now() - retentionDays * 86_400_000).toISOString();
|
|
867
|
-
const result = db.prepare("DELETE FROM improve_runs WHERE started_at < ?").run(cutoff);
|
|
868
|
-
const changes = result.changes ?? 0;
|
|
869
|
-
return typeof changes === "bigint" ? Number(changes) : changes;
|
|
870
|
-
}
|
|
871
|
-
/**
|
|
872
|
-
* Record (or update) one session's extract outcome. INSERT-OR-REPLACE so the
|
|
873
|
-
* row reflects the most recent run. The `content_hash` persisted here is what
|
|
874
|
-
* the NEXT run compares against (#602): a byte-identical session is skipped, a
|
|
875
|
-
* changed session is re-processed, and a NULL-backfill row becomes hash-stable
|
|
876
|
-
* after its one reprocess. `session_ended_at` is still written for
|
|
877
|
-
* telemetry/forensics but is no longer the skip authority.
|
|
878
|
-
*/
|
|
879
|
-
export function upsertExtractedSession(db, input) {
|
|
880
|
-
const endedAtIso = typeof input.sessionEndedAt === "number" && Number.isFinite(input.sessionEndedAt)
|
|
881
|
-
? new Date(input.sessionEndedAt).toISOString()
|
|
882
|
-
: null;
|
|
883
|
-
db.prepare(`
|
|
884
|
-
INSERT OR REPLACE INTO extract_sessions_seen
|
|
885
|
-
(harness, session_id, processed_at, session_ended_at, outcome,
|
|
886
|
-
candidate_count, proposal_count, rationale, source_run, metadata_json,
|
|
887
|
-
content_hash)
|
|
888
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
889
|
-
`).run(input.harness, input.sessionId, input.processedAt, endedAtIso, input.outcome, input.candidateCount, input.proposalCount, input.rationale ?? null, input.sourceRun ?? null, JSON.stringify(input.metadata ?? {}), input.contentHash);
|
|
890
|
-
}
|
|
891
|
-
/**
|
|
892
|
-
* Fetch a single session's last extract record, or `undefined` when the
|
|
893
|
-
* session has never been processed.
|
|
894
|
-
*/
|
|
895
|
-
export function getExtractedSession(db, harness, sessionId) {
|
|
896
|
-
// bun:sqlite returns null (not undefined) when no row matches — normalize so
|
|
897
|
-
// callers can rely on `if (!row)` and `toBeUndefined()` equivalently.
|
|
898
|
-
const row = db
|
|
899
|
-
.prepare("SELECT * FROM extract_sessions_seen WHERE harness = ? AND session_id = ?")
|
|
900
|
-
.get(harness, sessionId);
|
|
901
|
-
return row ?? undefined;
|
|
902
|
-
}
|
|
903
|
-
/**
|
|
904
|
-
* Bulk-fetch session-extract status for a list of sessionIds in one harness.
|
|
905
|
-
* Returns a Map keyed by sessionId so callers can do O(1) lookups while
|
|
906
|
-
* iterating the discovery list.
|
|
907
|
-
*/
|
|
908
|
-
export function getExtractedSessionsMap(db, harness, sessionIds) {
|
|
909
|
-
const out = new Map();
|
|
910
|
-
if (sessionIds.length === 0)
|
|
911
|
-
return out;
|
|
912
|
-
// SQLite has a ~999 param ceiling; chunk if a caller ever exceeds that.
|
|
913
|
-
const CHUNK = 500;
|
|
914
|
-
for (let i = 0; i < sessionIds.length; i += CHUNK) {
|
|
915
|
-
const chunk = sessionIds.slice(i, i + CHUNK);
|
|
916
|
-
const placeholders = chunk.map(() => "?").join(",");
|
|
917
|
-
const rows = db
|
|
918
|
-
.prepare(`SELECT * FROM extract_sessions_seen
|
|
919
|
-
WHERE harness = ? AND session_id IN (${placeholders})`)
|
|
920
|
-
.all(harness, ...chunk);
|
|
921
|
-
for (const row of rows)
|
|
922
|
-
out.set(row.session_id, row);
|
|
923
|
-
}
|
|
924
|
-
return out;
|
|
925
|
-
}
|
|
926
|
-
/**
|
|
927
|
-
* The most recent extract-run time for a harness — `MAX(processed_at)` across
|
|
928
|
-
* its ledger rows, as ms epoch — or `null` when the harness has never been
|
|
929
|
-
* extracted. Used to default the discovery window to "since the last run" so an
|
|
930
|
-
* intermittently-online host that was off for days still rediscovers sessions
|
|
931
|
-
* that ended during the gap (the content-hash ledger keeps the widened window
|
|
932
|
-
* free of redundant LLM cost).
|
|
933
|
-
*/
|
|
934
|
-
export function getLastExtractRunAt(db, harness) {
|
|
935
|
-
const row = db
|
|
936
|
-
.prepare("SELECT MAX(processed_at) AS last FROM extract_sessions_seen WHERE harness = ?")
|
|
937
|
-
.get(harness);
|
|
938
|
-
if (!row?.last)
|
|
939
|
-
return null;
|
|
940
|
-
const ms = Date.parse(row.last);
|
|
941
|
-
return Number.isFinite(ms) ? ms : null;
|
|
942
|
-
}
|
|
943
|
-
/**
|
|
944
|
-
* Decide whether a session should be skipped because the extractor has already
|
|
945
|
-
* processed BYTE-IDENTICAL content (#602). The skip authority is the content
|
|
946
|
-
* hash, NOT `session_ended_at` — this is clock-independent, so it is immune to
|
|
947
|
-
* the clock-skew / out-of-order-endedAt problems that caused the Jun 11-12
|
|
948
|
-
* double-extract + over-throttle incident.
|
|
949
|
-
*
|
|
950
|
-
* Rules:
|
|
951
|
-
* - no prior row → `false` (never seen → process; AC3).
|
|
952
|
-
* - prior.content_hash == null → `false` (legacy / hash-less row → process
|
|
953
|
-
* exactly once to backfill the hash, then it becomes hash-stable; AC4).
|
|
954
|
-
* - hashes equal → `true` (unchanged content → skip; AC1).
|
|
955
|
-
* - hashes differ → `false` (changed content → re-process; AC2).
|
|
956
|
-
*/
|
|
957
|
-
export function shouldSkipAlreadyExtractedSession(prior, currentContentHash) {
|
|
958
|
-
if (!prior)
|
|
959
|
-
return false;
|
|
960
|
-
if (prior.content_hash == null)
|
|
961
|
-
return false;
|
|
962
|
-
return prior.content_hash === currentContentHash;
|
|
963
|
-
}
|
|
964
|
-
/**
|
|
965
|
-
* Bulk-fetch the judged-state cache for a set of entry keys in one query.
|
|
966
|
-
* Returns a Map keyed by entry_key so the consolidate pool-selection loop can
|
|
967
|
-
* do O(1) "has this memory been judged at this content hash?" lookups.
|
|
968
|
-
* Empty input → empty map (no query issued).
|
|
969
|
-
*/
|
|
970
|
-
export function getConsolidationJudgedMap(db, entryKeys) {
|
|
971
|
-
const out = new Map();
|
|
972
|
-
if (entryKeys.length === 0)
|
|
973
|
-
return out;
|
|
974
|
-
// SQLite has a ~999 param ceiling; chunk if a caller ever exceeds that.
|
|
975
|
-
const CHUNK = 500;
|
|
976
|
-
for (let i = 0; i < entryKeys.length; i += CHUNK) {
|
|
977
|
-
const chunk = entryKeys.slice(i, i + CHUNK);
|
|
978
|
-
const placeholders = chunk.map(() => "?").join(",");
|
|
979
|
-
const rows = db
|
|
980
|
-
.prepare(`SELECT * FROM consolidation_judged WHERE entry_key IN (${placeholders})`)
|
|
981
|
-
.all(...chunk);
|
|
982
|
-
for (const row of rows)
|
|
983
|
-
out.set(row.entry_key, row);
|
|
984
|
-
}
|
|
985
|
-
return out;
|
|
986
|
-
}
|
|
987
|
-
/**
|
|
988
|
-
* Record (or update) the judged state for one memory. INSERT-OR-REPLACE so the
|
|
989
|
-
* row always reflects the most recent judge of that entry_key. Called once per
|
|
990
|
-
* memory the consolidate LLM saw in a successfully-judged chunk.
|
|
991
|
-
*/
|
|
992
|
-
export function upsertConsolidationJudged(db, input) {
|
|
993
|
-
db.prepare(`
|
|
994
|
-
INSERT OR REPLACE INTO consolidation_judged
|
|
995
|
-
(entry_key, content_hash, judged_at, outcome)
|
|
996
|
-
VALUES (?, ?, ?, ?)
|
|
997
|
-
`).run(input.entryKey, input.contentHash, input.judgedAt, input.outcome);
|
|
998
|
-
}
|
|
999
|
-
/**
|
|
1000
|
-
* Record an induction of a recombine hypothesis and return the new consecutive
|
|
1001
|
-
* count. INSERT … ON CONFLICT increments the streak, but the `last_run` guard
|
|
1002
|
-
* makes a repeated call within the SAME run idempotent (no double-increment if
|
|
1003
|
-
* the same ref appears twice in one run). On insert the streak starts at 1.
|
|
1004
|
-
*/
|
|
1005
|
-
export function recordRecombineInduction(db, input) {
|
|
1006
|
-
const row = db
|
|
1007
|
-
.prepare(`
|
|
1008
|
-
INSERT INTO recombine_hypotheses
|
|
1009
|
-
(hypothesis_ref, signature, member_key, consecutive_count, first_seen_at, last_seen_at, last_run)
|
|
1010
|
-
VALUES (?, ?, ?, 1, ?, ?, ?)
|
|
1011
|
-
ON CONFLICT(hypothesis_ref) DO UPDATE SET
|
|
1012
|
-
consecutive_count = consecutive_count + (CASE WHEN last_run IS excluded.last_run THEN 0 ELSE 1 END),
|
|
1013
|
-
last_seen_at = excluded.last_seen_at,
|
|
1014
|
-
last_run = excluded.last_run,
|
|
1015
|
-
signature = excluded.signature,
|
|
1016
|
-
member_key = excluded.member_key
|
|
1017
|
-
RETURNING consecutive_count
|
|
1018
|
-
`)
|
|
1019
|
-
.get(input.hypothesisRef, input.signature, input.memberKey, input.seenAt, input.seenAt, input.run);
|
|
1020
|
-
return row?.consecutive_count ?? 0;
|
|
1021
|
-
}
|
|
1022
|
-
/**
|
|
1023
|
-
* #633 — find an existing pending (non-promoted) hypothesis row whose cluster
|
|
1024
|
-
* is the SAME generalization as a newly-induced one, matched by SIGNATURE plus
|
|
1025
|
-
* a Jaccard membership-overlap test, rather than an exact member-set hash.
|
|
1026
|
-
*
|
|
1027
|
-
* In a growing stash any added/removed memory changes the exact member set, so
|
|
1028
|
-
* the ref hash (and member_key) shift every run → a fresh row at count=1 → the
|
|
1029
|
-
* streak never reaches `confirmThreshold` and nothing ever promotes. Matching
|
|
1030
|
-
* on overlap lets a drifting-but-stable cluster keep accumulating under one row.
|
|
1031
|
-
*
|
|
1032
|
-
* Returns the matched row with the HIGHEST overlap (ties broken by most-recent
|
|
1033
|
-
* `last_seen_at`), or `undefined` when none clears `minOverlap`. Already-promoted
|
|
1034
|
-
* rows are ignored so a confirmed lesson is not reopened by a later induction.
|
|
1035
|
-
*
|
|
1036
|
-
* @param memberKey the candidate cluster's membership fingerprint
|
|
1037
|
-
* (sorted member entryKeys joined by `|`).
|
|
1038
|
-
* @param minOverlap Jaccard threshold in [0,1]; a candidate matches when
|
|
1039
|
-
* |A∩B| / |A∪B| >= minOverlap.
|
|
1040
|
-
*/
|
|
1041
|
-
export function findMatchingRecombineHypothesis(db, input) {
|
|
1042
|
-
const candidateMembers = new Set(input.memberKey.split("|").filter((m) => m.length > 0));
|
|
1043
|
-
if (candidateMembers.size === 0)
|
|
1044
|
-
return undefined;
|
|
1045
|
-
const rows = db
|
|
1046
|
-
.prepare("SELECT * FROM recombine_hypotheses WHERE signature = ? AND promoted_at IS NULL ORDER BY last_seen_at DESC")
|
|
1047
|
-
.all(input.signature);
|
|
1048
|
-
let best;
|
|
1049
|
-
let bestOverlap = -1;
|
|
1050
|
-
for (const row of rows) {
|
|
1051
|
-
const rowMembers = row.member_key.split("|").filter((m) => m.length > 0);
|
|
1052
|
-
if (rowMembers.length === 0)
|
|
1053
|
-
continue;
|
|
1054
|
-
let intersection = 0;
|
|
1055
|
-
for (const m of rowMembers) {
|
|
1056
|
-
if (candidateMembers.has(m))
|
|
1057
|
-
intersection += 1;
|
|
1058
|
-
}
|
|
1059
|
-
const union = candidateMembers.size + rowMembers.length - intersection;
|
|
1060
|
-
const overlap = union === 0 ? 0 : intersection / union;
|
|
1061
|
-
// rows are ordered last_seen_at DESC, so a strict `>` keeps the most-recent
|
|
1062
|
-
// row on ties.
|
|
1063
|
-
if (overlap >= input.minOverlap && overlap > bestOverlap) {
|
|
1064
|
-
best = row;
|
|
1065
|
-
bestOverlap = overlap;
|
|
1066
|
-
}
|
|
1067
|
-
}
|
|
1068
|
-
return best;
|
|
1069
|
-
}
|
|
1070
|
-
/**
|
|
1071
|
-
* Fetch a single recombine hypothesis row, or `undefined` when the ref has
|
|
1072
|
-
* never been induced. Normalizes bun:sqlite null → undefined like
|
|
1073
|
-
* {@link getExtractedSession}.
|
|
1074
|
-
*/
|
|
1075
|
-
export function getRecombineHypothesis(db, hypothesisRef) {
|
|
1076
|
-
const row = db
|
|
1077
|
-
.prepare("SELECT * FROM recombine_hypotheses WHERE hypothesis_ref = ?")
|
|
1078
|
-
.get(hypothesisRef);
|
|
1079
|
-
return row ?? undefined;
|
|
1080
|
-
}
|
|
1081
|
-
/**
|
|
1082
|
-
* Mark a hypothesis promoted: stamp `promoted_at` and reset the consecutive
|
|
1083
|
-
* count to 0, so it must re-accumulate a full confirmation streak before it can
|
|
1084
|
-
* promote again. The `promoted_at` non-null state is the double-promotion guard.
|
|
1085
|
-
*/
|
|
1086
|
-
export function markRecombineHypothesisPromoted(db, hypothesisRef, promotedAt) {
|
|
1087
|
-
db.prepare("UPDATE recombine_hypotheses SET promoted_at = ?, consecutive_count = 0 WHERE hypothesis_ref = ?").run(promotedAt, hypothesisRef);
|
|
1088
|
-
}
|
|
1089
|
-
/**
|
|
1090
|
-
* #658 — does any current-run cluster match this hypothesis row under the SAME
|
|
1091
|
-
* signature + Jaccard-overlap rule used for re-induction? A match means the
|
|
1092
|
-
* cluster genuinely re-formed this run (it was merely cap-displaced out of the
|
|
1093
|
-
* processed top-`maxClustersPerRun` slice), so its streak must NOT be reset.
|
|
1094
|
-
*/
|
|
1095
|
-
function hypothesisMatchesAnyPresentCluster(row, presentClusters, minOverlap) {
|
|
1096
|
-
const rowMembers = row.member_key.split("|").filter((m) => m.length > 0);
|
|
1097
|
-
if (rowMembers.length === 0)
|
|
1098
|
-
return false;
|
|
1099
|
-
const rowSet = new Set(rowMembers);
|
|
1100
|
-
for (const cluster of presentClusters) {
|
|
1101
|
-
if (cluster.signature !== row.signature)
|
|
1102
|
-
continue;
|
|
1103
|
-
const clusterMembers = cluster.memberKey.split("|").filter((m) => m.length > 0);
|
|
1104
|
-
if (clusterMembers.length === 0)
|
|
1105
|
-
continue;
|
|
1106
|
-
let intersection = 0;
|
|
1107
|
-
for (const m of clusterMembers) {
|
|
1108
|
-
if (rowSet.has(m))
|
|
1109
|
-
intersection += 1;
|
|
1110
|
-
}
|
|
1111
|
-
const union = rowSet.size + clusterMembers.length - intersection;
|
|
1112
|
-
const overlap = union === 0 ? 0 : intersection / union;
|
|
1113
|
-
if (overlap >= minOverlap)
|
|
1114
|
-
return true;
|
|
1115
|
-
}
|
|
1116
|
-
return false;
|
|
1117
|
-
}
|
|
1118
|
-
/**
|
|
1119
|
-
* Decay-to-zero every NON-promoted hypothesis NOT re-induced in the current run.
|
|
1120
|
-
*
|
|
1121
|
-
* A generalization that stops being supported by the corpus has lost its
|
|
1122
|
-
* confirmation streak, so we hard-reset `consecutive_count` to 0 (the
|
|
1123
|
-
* alternative — `count - 1` floored at 0 — tolerates a single noisy run but
|
|
1124
|
-
* blurs the "consecutive" semantics; hard-reset is the conservative choice).
|
|
1125
|
-
*
|
|
1126
|
-
* Only rows whose `hypothesis_ref` is NOT in `seenRefs` AND whose `last_run` is
|
|
1127
|
-
* NOT the current run are decayed. Already-promoted rows are left alone.
|
|
1128
|
-
*
|
|
1129
|
-
* #658 — CAP-AWARE decay. The recombine pass only re-inducts (and thus marks
|
|
1130
|
-
* `seen`) the top-`maxClustersPerRun` clusters, but a cluster genuinely
|
|
1131
|
-
* re-forms every run even when it is displaced below that cap. Resetting such a
|
|
1132
|
-
* row treats a SCHEDULING miss as a SUBSTANCE miss and traps the hypothesis
|
|
1133
|
-
* below `confirmThreshold` forever. When `opts.presentClusters` is supplied, a
|
|
1134
|
-
* row is SPARED from decay if it Jaccard-matches any present cluster (same
|
|
1135
|
-
* signature, overlap >= `opts.minOverlap`) — i.e. its cluster re-formed this run
|
|
1136
|
-
* but was cap-displaced. This does NOT advance the streak (only re-induction in
|
|
1137
|
-
* the processed slice does that, via {@link recordRecombineInduction}), so the
|
|
1138
|
-
* recurrence bar for promotion is unchanged; it only stops the cap from
|
|
1139
|
-
* manufacturing artificial misses. Omitting `presentClusters` preserves the
|
|
1140
|
-
* pre-#658 hard-reset-after-one-miss behaviour exactly.
|
|
1141
|
-
*
|
|
1142
|
-
* Returns the number of rows reset.
|
|
1143
|
-
*/
|
|
1144
|
-
export function decayUnseenRecombineHypotheses(db, currentRun, seenRefs, opts) {
|
|
1145
|
-
// #658 — when cap-aware sparing is requested, fold the cap-displaced rows into
|
|
1146
|
-
// the "seen" exclusion set: the underlying reset SQL already protects every
|
|
1147
|
-
// ref it is given, so sparing == treating a spared row exactly like a seen
|
|
1148
|
-
// row for this sweep (its count is left untouched, never advanced).
|
|
1149
|
-
let effectiveSeen = seenRefs;
|
|
1150
|
-
if (opts && opts.presentClusters.length > 0) {
|
|
1151
|
-
const candidates = db
|
|
1152
|
-
.prepare("SELECT hypothesis_ref, signature, member_key FROM recombine_hypotheses WHERE promoted_at IS NULL AND (last_run IS NULL OR last_run != ?) AND consecutive_count != 0")
|
|
1153
|
-
.all(currentRun);
|
|
1154
|
-
const seenSet = new Set(seenRefs);
|
|
1155
|
-
for (const row of candidates) {
|
|
1156
|
-
if (seenSet.has(row.hypothesis_ref))
|
|
1157
|
-
continue;
|
|
1158
|
-
if (hypothesisMatchesAnyPresentCluster(row, opts.presentClusters, opts.minOverlap)) {
|
|
1159
|
-
seenSet.add(row.hypothesis_ref);
|
|
1160
|
-
}
|
|
1161
|
-
}
|
|
1162
|
-
effectiveSeen = [...seenSet];
|
|
1163
|
-
}
|
|
1164
|
-
return decayUnseenRecombineHypothesesInner(db, currentRun, effectiveSeen);
|
|
1165
|
-
}
|
|
1166
|
-
/**
|
|
1167
|
-
* The raw reset sweep shared by the cap-aware wrapper above. Resets every
|
|
1168
|
-
* non-promoted row from a prior run whose ref is NOT in `seenRefs`. Kept private
|
|
1169
|
-
* so the param-ceiling chunking logic lives in one place.
|
|
1170
|
-
*/
|
|
1171
|
-
function decayUnseenRecombineHypothesesInner(db, currentRun, seenRefs) {
|
|
1172
|
-
// Reset every eligible row, then exclude the seen refs in chunks to respect
|
|
1173
|
-
// the ~999 SQLite param ceiling. With no seen refs we reset all non-promoted
|
|
1174
|
-
// rows from prior runs in a single statement.
|
|
1175
|
-
if (seenRefs.length === 0) {
|
|
1176
|
-
const res = db
|
|
1177
|
-
.prepare("UPDATE recombine_hypotheses SET consecutive_count = 0 WHERE promoted_at IS NULL AND (last_run IS NULL OR last_run != ?) AND consecutive_count != 0")
|
|
1178
|
-
.run(currentRun);
|
|
1179
|
-
return Number(res.changes);
|
|
1180
|
-
}
|
|
1181
|
-
// A single NOT IN keeps the exclusion atomic (a chunked NOT IN would let a ref
|
|
1182
|
-
// excluded by one chunk still be reset by another chunk's statement). The
|
|
1183
|
-
// recombine pass caps RE-INDUCED clusters at `maxClustersPerRun` (a handful) —
|
|
1184
|
-
// but with #658 cap-aware sparing the caller folds every cap-displaced
|
|
1185
|
-
// (present-but-unprocessed) hypothesis into `effectiveSeen` too, so on a large
|
|
1186
|
-
// stash `seenRefs` here can carry MANY spared refs, not just the handful that
|
|
1187
|
-
// were processed. We cap defensively at ~900 (under SQLite's ~999 param
|
|
1188
|
-
// ceiling): if `effectiveSeen` somehow exceeds it we fall back to resetting all
|
|
1189
|
-
// eligible rows — which re-introduces the cap-displacement trap for THAT run
|
|
1190
|
-
// (spared rows get decayed because the NOT IN protection is dropped). That is a
|
|
1191
|
-
// rare, bounded degradation; a stash with >900 simultaneously-spared
|
|
1192
|
-
// hypotheses is far beyond current scale.
|
|
1193
|
-
if (seenRefs.length > 900) {
|
|
1194
|
-
const res = db
|
|
1195
|
-
.prepare("UPDATE recombine_hypotheses SET consecutive_count = 0 WHERE promoted_at IS NULL AND (last_run IS NULL OR last_run != ?) AND consecutive_count != 0")
|
|
1196
|
-
.run(currentRun);
|
|
1197
|
-
return Number(res.changes);
|
|
1198
|
-
}
|
|
1199
|
-
const placeholders = seenRefs.map(() => "?").join(",");
|
|
1200
|
-
const res = db
|
|
1201
|
-
.prepare(`UPDATE recombine_hypotheses SET consecutive_count = 0
|
|
1202
|
-
WHERE promoted_at IS NULL
|
|
1203
|
-
AND (last_run IS NULL OR last_run != ?)
|
|
1204
|
-
AND consecutive_count != 0
|
|
1205
|
-
AND hypothesis_ref NOT IN (${placeholders})`)
|
|
1206
|
-
.run(currentRun, ...seenRefs);
|
|
1207
|
-
return Number(res.changes);
|
|
1208
|
-
}
|
|
1209
|
-
/**
|
|
1210
|
-
* Convert a `number[]` embedding vector to the `Float32Array` byte
|
|
1211
|
-
* representation stored in the `body_embeddings.embedding` BLOB column.
|
|
1212
|
-
*/
|
|
1213
|
-
export function embeddingToBlob(vec) {
|
|
1214
|
-
const f32 = new Float32Array(vec);
|
|
1215
|
-
return new Uint8Array(f32.buffer);
|
|
1216
|
-
}
|
|
1217
|
-
/**
|
|
1218
|
-
* Convert the raw `Uint8Array` bytes from the `body_embeddings.embedding`
|
|
1219
|
-
* BLOB column back to a `number[]` embedding vector.
|
|
1220
|
-
*/
|
|
1221
|
-
export function blobToEmbedding(blob) {
|
|
1222
|
-
// SQLite BLOB columns are returned as Uint8Array; re-interpret as Float32.
|
|
1223
|
-
const f32 = new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
|
|
1224
|
-
return Array.from(f32);
|
|
1225
|
-
}
|
|
1226
|
-
/**
|
|
1227
|
-
* Bulk-fetch cached body embeddings for a set of content hashes.
|
|
1228
|
-
* Returns a Map keyed by `content_hash` (embedding decoded to `number[]`).
|
|
1229
|
-
* Empty input → empty map (no query issued).
|
|
1230
|
-
*
|
|
1231
|
-
* If the stored `model_id` does not match `expectedModelId` the entire table
|
|
1232
|
-
* is cleared (drop-all on model mismatch) and an empty map is returned so
|
|
1233
|
-
* callers re-embed everything on this run.
|
|
1234
|
-
*/
|
|
1235
|
-
export function getBodyEmbeddings(db, contentHashes, expectedModelId) {
|
|
1236
|
-
const out = new Map();
|
|
1237
|
-
if (contentHashes.length === 0)
|
|
1238
|
-
return out;
|
|
1239
|
-
// Model-id mismatch: vectors are in the wrong metric space — drop all rows.
|
|
1240
|
-
const firstRow = db.prepare("SELECT model_id FROM body_embeddings LIMIT 1").get();
|
|
1241
|
-
if (firstRow && firstRow.model_id !== expectedModelId) {
|
|
1242
|
-
db.exec("DELETE FROM body_embeddings");
|
|
1243
|
-
return out;
|
|
1244
|
-
}
|
|
1245
|
-
// SQLite has a ~999 param ceiling; chunk if needed.
|
|
1246
|
-
const CHUNK = 500;
|
|
1247
|
-
for (let i = 0; i < contentHashes.length; i += CHUNK) {
|
|
1248
|
-
const chunk = contentHashes.slice(i, i + CHUNK);
|
|
1249
|
-
const placeholders = chunk.map(() => "?").join(",");
|
|
1250
|
-
const rows = db
|
|
1251
|
-
.prepare(`SELECT content_hash, embedding FROM body_embeddings WHERE content_hash IN (${placeholders})`)
|
|
1252
|
-
.all(...chunk);
|
|
1253
|
-
for (const row of rows) {
|
|
1254
|
-
out.set(row.content_hash, blobToEmbedding(row.embedding));
|
|
1255
|
-
}
|
|
1256
|
-
}
|
|
1257
|
-
return out;
|
|
1258
|
-
}
|
|
1259
|
-
/**
|
|
1260
|
-
* Upsert body-embedding rows in a single transaction.
|
|
1261
|
-
* Each entry maps a `cacheHash` → `number[]` vector. `model_id` is stored
|
|
1262
|
-
* so a future model change can trigger a drop-all purge.
|
|
1263
|
-
*/
|
|
1264
|
-
export function upsertBodyEmbeddings(db, entries) {
|
|
1265
|
-
if (entries.length === 0)
|
|
1266
|
-
return;
|
|
1267
|
-
const now = Date.now();
|
|
1268
|
-
const stmt = db.prepare(`
|
|
1269
|
-
INSERT OR REPLACE INTO body_embeddings (content_hash, embedding, model_id, created_at)
|
|
1270
|
-
VALUES (?, ?, ?, ?)
|
|
1271
|
-
`);
|
|
1272
|
-
db.transaction(() => {
|
|
1273
|
-
for (const { contentHash, embedding, modelId } of entries) {
|
|
1274
|
-
stmt.run(contentHash, embeddingToBlob(embedding), modelId, now);
|
|
1275
|
-
}
|
|
1276
|
-
})();
|
|
1277
|
-
}
|
|
1278
|
-
/** Insert a freshly minted canary set (all rows active, one shared set id). */
|
|
1279
|
-
export function insertCanaries(db, canarySetId, canaries, now) {
|
|
1280
|
-
if (canaries.length === 0)
|
|
1281
|
-
return;
|
|
1282
|
-
const ts = now ?? new Date().toISOString();
|
|
1283
|
-
const stmt = db.prepare(`
|
|
1284
|
-
INSERT INTO canary_queries (canary_set_id, anchor_ref, query, source, active, created_at)
|
|
1285
|
-
VALUES (?, ?, ?, ?, 1, ?)
|
|
1286
|
-
`);
|
|
1287
|
-
db.transaction(() => {
|
|
1288
|
-
for (const c of canaries) {
|
|
1289
|
-
stmt.run(canarySetId, c.anchorRef, c.query, c.source ?? "auto", ts);
|
|
1290
|
-
}
|
|
1291
|
-
})();
|
|
1292
|
-
}
|
|
1293
|
-
/** Load the active canary set (empty array = never minted). */
|
|
1294
|
-
export function getActiveCanaries(db) {
|
|
1295
|
-
// Scope to the NEWEST active set: if an interrupted refresh (or a bug) ever
|
|
1296
|
-
// leaves two sets active, mixing their rows would silently corrupt the
|
|
1297
|
-
// recall/entropy trend baselines. The newest set wins; stale-active rows are
|
|
1298
|
-
// simply never returned.
|
|
1299
|
-
return db
|
|
1300
|
-
.prepare(`SELECT * FROM canary_queries
|
|
1301
|
-
WHERE active = 1 AND canary_set_id = (
|
|
1302
|
-
SELECT canary_set_id FROM canary_queries WHERE active = 1
|
|
1303
|
-
ORDER BY created_at DESC, id DESC LIMIT 1
|
|
1304
|
-
)
|
|
1305
|
-
ORDER BY id`)
|
|
1306
|
-
.all();
|
|
1307
|
-
}
|
|
1308
|
-
/** Load one canary set's rows by its exact set id (any active state), insertion order. */
|
|
1309
|
-
export function getCanariesBySetId(db, canarySetId) {
|
|
1310
|
-
return db
|
|
1311
|
-
.prepare(`SELECT * FROM canary_queries WHERE canary_set_id = ? ORDER BY id`)
|
|
1312
|
-
.all(canarySetId);
|
|
1313
|
-
}
|
|
1314
|
-
/** List every distinct canary_set_id that still has active rows. */
|
|
1315
|
-
export function listActiveCanarySetIds(db) {
|
|
1316
|
-
const rows = db.prepare(`SELECT DISTINCT canary_set_id FROM canary_queries WHERE active = 1`).all();
|
|
1317
|
-
return rows.map((r) => r.canary_set_id);
|
|
1318
|
-
}
|
|
1319
|
-
/**
|
|
1320
|
-
* Deactivate every canary row in a set. Rows are RETAINED (active = 0) so
|
|
1321
|
-
* historical improve_cycle_metrics rows keyed on the old canary_set_id stay
|
|
1322
|
-
* interpretable; only `akm improve canary --refresh` calls this.
|
|
1323
|
-
*/
|
|
1324
|
-
export function deactivateCanarySet(db, canarySetId) {
|
|
1325
|
-
const result = db
|
|
1326
|
-
.prepare(`UPDATE canary_queries SET active = 0 WHERE canary_set_id = ? AND active = 1`)
|
|
1327
|
-
.run(canarySetId);
|
|
1328
|
-
const changes = result.changes ?? 0;
|
|
1329
|
-
return typeof changes === "bigint" ? Number(changes) : changes;
|
|
1330
|
-
}
|
|
1331
|
-
/** Persist one qualifying cycle's store-health snapshot. */
|
|
1332
|
-
export function insertCycleMetrics(db, row) {
|
|
1333
|
-
db.prepare(`
|
|
1334
|
-
INSERT INTO improve_cycle_metrics
|
|
1335
|
-
(run_id, ts, pass, canary_set_id, mean_recall, mean_ndcg, mean_mrr,
|
|
1336
|
-
canary_ranks_json, store_total, store_by_type_json, distinct_content_ratio,
|
|
1337
|
-
mean_bigram_diversity, over_generation_count, accepted_actions,
|
|
1338
|
-
merge_floor_violations, alerts_json)
|
|
1339
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1340
|
-
`).run(row.run_id, row.ts, row.pass, row.canary_set_id, row.mean_recall, row.mean_ndcg, row.mean_mrr, row.canary_ranks_json, row.store_total, row.store_by_type_json, row.distinct_content_ratio, row.mean_bigram_diversity, row.over_generation_count, row.accepted_actions, row.merge_floor_violations, row.alerts_json);
|
|
1341
|
-
}
|
|
1342
|
-
/**
|
|
1343
|
-
* Load the most recent cycle rows for one canary set, OLDEST-first (the alert
|
|
1344
|
-
* evaluator's window order). Scoped by canary_set_id so trends never compare
|
|
1345
|
-
* across canary re-mints.
|
|
1346
|
-
*/
|
|
1347
|
-
export function queryRecentCycleMetrics(db, canarySetId, limit) {
|
|
1348
|
-
const rows = db
|
|
1349
|
-
.prepare(`SELECT run_id, ts, pass, canary_set_id, mean_recall, mean_ndcg, mean_mrr,
|
|
1350
|
-
canary_ranks_json, store_total, store_by_type_json, distinct_content_ratio,
|
|
1351
|
-
mean_bigram_diversity, over_generation_count, accepted_actions,
|
|
1352
|
-
merge_floor_violations, alerts_json
|
|
1353
|
-
FROM improve_cycle_metrics WHERE canary_set_id = ?
|
|
1354
|
-
ORDER BY ts DESC, id DESC LIMIT ?`)
|
|
1355
|
-
.all(canarySetId, Math.max(0, limit));
|
|
1356
|
-
return rows.reverse();
|
|
1357
|
-
}
|
|
1358
|
-
/** Load the single most recent cycle row across all canary sets (health surface). */
|
|
1359
|
-
export function getLatestCycleMetrics(db) {
|
|
1360
|
-
const row = db
|
|
1361
|
-
.prepare(`SELECT run_id, ts, pass, canary_set_id, mean_recall, mean_ndcg, mean_mrr,
|
|
1362
|
-
canary_ranks_json, store_total, store_by_type_json, distinct_content_ratio,
|
|
1363
|
-
mean_bigram_diversity, over_generation_count, accepted_actions,
|
|
1364
|
-
merge_floor_violations, alerts_json
|
|
1365
|
-
FROM improve_cycle_metrics ORDER BY ts DESC, id DESC LIMIT 1`)
|
|
1366
|
-
.get();
|
|
1367
|
-
return row == null ? undefined : row;
|
|
1368
|
-
}
|
|
1369
|
-
/**
|
|
1370
|
-
* Delete cycle rows older than `retentionDays` (default 365 — owner-approved;
|
|
1371
|
-
* a slow collapse needs a longer trend window than the 90-day events log).
|
|
1372
|
-
* Returns the purged row count. canary_queries rows are never purged.
|
|
1373
|
-
*/
|
|
1374
|
-
export function purgeOldCycleMetrics(db, retentionDays = 365) {
|
|
1375
|
-
if (!Number.isFinite(retentionDays) || retentionDays <= 0)
|
|
1376
|
-
return 0;
|
|
1377
|
-
const cutoff = new Date(Date.now() - retentionDays * 86_400_000).toISOString();
|
|
1378
|
-
const result = db.prepare("DELETE FROM improve_cycle_metrics WHERE ts < ?").run(cutoff);
|
|
1379
|
-
const changes = result.changes ?? 0;
|
|
1380
|
-
return typeof changes === "bigint" ? Number(changes) : changes;
|
|
1381
|
-
}
|