akm-cli 0.9.0-beta.4 → 0.9.0-beta.41

package/dist/core/state-db.js

@@ -53,6 +53,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { openDatabase } from "../storage/database.js";
 import { runMigrations as runSqliteMigrations } from "../storage/engines/sqlite-migrations.js";
+import { applyStandardPragmas } from "../storage/sqlite-pragmas.js";
 import { classifyImproveAction } from "./improve-types.js";
 import { getDataDir } from "./paths.js";
 import { error } from "./warn.js";
@@ -101,9 +102,7 @@ export function openStateDatabase(dbPath) {
     }
     const db = openDatabase(resolvedPath);
     // PRAGMAs must run before any DDL or DML.
-    db.exec("PRAGMA journal_mode = WAL");
-    db.exec("PRAGMA foreign_keys = ON");
-    db.exec("PRAGMA busy_timeout = 30000");
+    applyStandardPragmas(db, { dataDir: dir });
     runMigrations(db);
     return db;
 }
@@ -193,7 +192,7 @@ const MIGRATIONS = [
       --   metadata_json TEXT      — JSON object for future proposal fields.
       --                             Current fields stored here: sourceRun,
       --                             review, confidence, gateDecision (#577),
-      --                             backupContent.
+      --                             backupContent, eligibilitySource.
       --
       -- ADD COLUMN extension points (future migrations):
       --   ALTER TABLE proposals ADD COLUMN source_run TEXT DEFAULT NULL;
@@ -488,6 +487,335 @@ const MIGRATIONS = [
       );
     `,
     },
+    // ── Migration 006 — pending proposal lookup index ──────────────────────────
+    //
+    // Supports the transaction-scoped dedup / queue-mutation hardening added in
+    // 0.9.x. The queue now acquires an IMMEDIATE write transaction before it
+    // reads pending proposals, so the hot path is a stash-scoped `status='pending'
+    // AND ref=?` probe followed by an update/insert. This composite index keeps
+    // that lookup index-covered under contention.
+    {
+        id: "006-proposals-pending-ref-source",
+        up: `
+      CREATE INDEX IF NOT EXISTS idx_proposals_stash_status_ref_source
+        ON proposals(stash_dir, status, ref, source);
+    `,
+    },
+    // ── Migration 007 — consolidation_judged ────────────────────────────────────
+    //
+    // Judged-state cache for nightly consolidation (#581). Lets one consolidation
+    // run cover the FULL memory corpus cheaply by SKIPPING memories already judged
+    // with unchanged content, instead of narrowing to a recent time-window slice
+    // (which leaves a near-duplicate backlog the corpus can never clear).
+    //
+    // The consolidate LLM judging loop UPSERTs a row for every memory it saw in a
+    // successfully-judged chunk; the next run hashes each candidate's current
+    // content and skips it when the hash equals the cached `content_hash`
+    // (judged-unchanged → no re-judge). A memory whose content changed produces a
+    // new hash and is re-judged. This converts coverage from O(window) to
+    // O(changed/new). DEFAULT OFF — gated behind
+    // `processes.consolidate.judgedCache.enabled`; when the feature is off this
+    // table is never read or written and behaviour is byte-identical to today.
+    //
+    // Indexed (query) columns:
+    //   entry_key    TEXT PK   — `memory:<name>` ref; one row per judged memory.
+    //
+    // Non-indexed columns:
+    //   content_hash TEXT      — sha256 of the frontmatter-stripped, trimmed body.
+    //   judged_at    TEXT      — ISO-8601 UTC; when the memory was last judged.
+    //   outcome      TEXT      — coarse outcome of the last judge ("actioned" |
+    //                            "no_action"); observability only, never gates.
+    //
+    // TTL: no automatic deletion. Rows for memories deleted on disk become
+    // harmless dead entries (their entry_key never recurs); operators can prune
+    // with `DELETE FROM consolidation_judged WHERE judged_at < ?` if desired.
+    {
+        id: "007-consolidation-judged",
+        up: `
+      CREATE TABLE IF NOT EXISTS consolidation_judged (
+        entry_key    TEXT    PRIMARY KEY,
+        content_hash TEXT    NOT NULL,
+        judged_at    TEXT    NOT NULL,
+        outcome      TEXT    NOT NULL
+      );
+    `,
+    },
+    // ── Migration 008 — body_embeddings ─────────────────────────────────────────
+    //
+    // cacheHash-keyed body-embedding cache (WS-3a). Stores the embedding of the
+    // case-preserving stripped body so the dedup pre-pass and the consolidation
+    // clustering step share one computed vector per unique body, eliminating
+    // redundant embedding calls across runs.
+    //
+    // Design:
+    //   - PK is the `cacheHash` (sha256 of the stripped, case-preserving body).
+    //   - `embedding` is a raw BLOB storing a Float32 array (384 floats × 4 B =
+    //     1 536 B per entry for the default bge-small-en-v1.5 model; ~20 MB at
+    //     13 k memories). This matches the native wire format and avoids JSON
+    //     round-trip overhead.
+    //   - `model_id` is MANDATORY. On mismatch (model changed) the entire table
+    //     is dropped and rebuilt — stale vectors from the wrong metric space would
+    //     produce silent cosine errors.
+    //   - `created_at` is an INTEGER Unix ms timestamp for lazy orphan purges.
+    //
+    // Writes: one bulk `WHERE content_hash IN (…)` lookup → embed only misses →
+    // upsert all results in one transaction per run.
+    //
+    // TTL: no automatic row deletion. Orphaned rows for bodies no longer in the
+    // stash stay until an operator prunes them. The table is ~1.5 KB per row
+    // (~20 MB at 13 k memories — acceptable).
+    {
+        id: "008-body-embeddings",
+        up: `
+      CREATE TABLE IF NOT EXISTS body_embeddings (
+        content_hash TEXT    PRIMARY KEY,
+        embedding    BLOB    NOT NULL,
+        model_id     TEXT    NOT NULL,
+        created_at   INTEGER NOT NULL
+      );
+    `,
+    },
+    // ── Migration 009 — asset_salience (WS-1 salience vector) ───────────────────
+    //
+    // Per-asset salience vector persisted in state.db (canonical store).
+    //
+    // Three independently-stored, independently-decayable sub-scores:
+    //   encoding_salience  — intrinsic importance (Gap 1; v1 = type-weight stub).
+    //   outcome_salience   — differential usefulness (WS-2; 0 until that lands).
+    //   retrieval_salience — frequency × recency (the decayable term).
+    //
+    // Plus the scalar projection for ranking:
+    //   rank_score = (w_e·encoding + w_o·outcome + w_r·retrieval) × sizePenalty,
+    //   normalized [0,1]. Every selector reads rank_score; individual sub-scores
+    //   are available for telemetry and per-dimension thresholding.
+    //
+    // Plasticity column:
+    //   consecutive_no_ops INTEGER — number of consecutive improve cycles where
+    //     this asset produced a no-op (reflect/distill produced no change).
+    //     Dampens CONSOLIDATION-SELECTION only — intentionally NOT applied to
+    //     rank_score (stable assets stay retrievable but skip LLM merge passes).
+    //
+    // updated_at is an INTEGER Unix-ms timestamp for recency queries.
+    //
+    // The canonical store is state.db, not frontmatter.  An optional frontmatter
+    // mirror of the stable encodingSalience is allowed for portability (#608).
+    //
+    // TTL: rows are overwritten on every run; orphaned rows for deleted assets
+    // accumulate harmlessly until an operator prunes them.
+    {
+        id: "009-asset-salience",
+        up: `
+      CREATE TABLE IF NOT EXISTS asset_salience (
+        asset_ref          TEXT    PRIMARY KEY,
+        encoding_salience  REAL    NOT NULL DEFAULT 0.5,
+        outcome_salience   REAL    NOT NULL DEFAULT 0.0,
+        retrieval_salience REAL    NOT NULL DEFAULT 0.0,
+        rank_score         REAL    NOT NULL DEFAULT 0.0,
+        consecutive_no_ops INTEGER NOT NULL DEFAULT 0,
+        updated_at         INTEGER NOT NULL DEFAULT 0
+      );
+
+      -- Hot path: sort / filter by rank_score for selector queries.
+      CREATE INDEX IF NOT EXISTS idx_asset_salience_rank
+        ON asset_salience(rank_score DESC);
+    `,
+    },
+    // ── Migration 010 — asset_outcome (WS-2 outcome loop) ───────────────────────
+    //
+    // Per-asset outcome loop persisted in state.db (S2 seam, WS-2).
+    //
+    // Stores the differential "was this retrieval useful" signal so the salience
+    // vector's `outcomeSalience` sub-score (WS-1 `W_OUTCOME` term) is non-zero.
+    //
+    // Columns:
+    //   asset_ref TEXT PK             — `type:name` asset ref (FK to asset_salience).
+    //   last_retrieved_at INTEGER     — Unix-ms of the most recent retrieval.
+    //   retrieval_count INTEGER       — total retrieval count from the index DB.
+    //   expected_retrieval_rate REAL  — EMA-smoothed expected count per cycle.
+    //   negative_feedback_count INTEGER — cumulative negative-feedback events.
+    //   accepted_change_count INTEGER — cumulative accepted proposals.
+    //   review_pressure INTEGER       — #613 pressure counter: repeated low-satisfaction
+    //                                   retrievals increment it; feeds outcomeSalience.
+    //   outcome_score REAL            — differential outcome signal (can be negative).
+    //   updated_at INTEGER            — Unix-ms timestamp of last update.
+    //
+    // Design:
+    //   - outcome_score is differential (prediction-error shaped), NOT a raw count,
+    //     so it rewards assets that are retrieved MORE than their rolling mean AND
+    //     accepted for change when retrieved. See outcome-loop.ts for the formula.
+    //   - review_pressure (#613): repeated negative-feedback retrievals raise it,
+    //     non-negative cycles decay it. Never mutates asset content directly.
+    //   - warm_start: seeded from utility EMA at row creation, clipped to [0, 0.3]
+    //     so the first negative delta does not cause a spurious rank inversion.
+    //   - Orphaned rows (deleted assets) accumulate harmlessly; operators can prune
+    //     with `DELETE FROM asset_outcome WHERE updated_at < ?` if desired.
+    {
+        id: "010-asset-outcome",
+        up: `
+      CREATE TABLE IF NOT EXISTS asset_outcome (
+        asset_ref                TEXT    PRIMARY KEY,
+        last_retrieved_at        INTEGER NOT NULL DEFAULT 0,
+        retrieval_count          INTEGER NOT NULL DEFAULT 0,
+        expected_retrieval_rate  REAL    NOT NULL DEFAULT 0.0,
+        negative_feedback_count  INTEGER NOT NULL DEFAULT 0,
+        accepted_change_count    INTEGER NOT NULL DEFAULT 0,
+        review_pressure          INTEGER NOT NULL DEFAULT 0,
+        outcome_score            REAL    NOT NULL DEFAULT 0.0,
+        updated_at               INTEGER NOT NULL DEFAULT 0
+      );
+
+      -- Hot path: sort assets by review_pressure DESC for #613 admission.
+      CREATE INDEX IF NOT EXISTS idx_asset_outcome_review_pressure
+        ON asset_outcome(review_pressure DESC);
+
+      -- Secondary: sort by outcome_score DESC for outcomeSalience reads.
+      CREATE INDEX IF NOT EXISTS idx_asset_outcome_score
+        ON asset_outcome(outcome_score DESC);
+    `,
+    },
+    // ── Migration 011 — asset_salience: homeostatic_demoted_at column ─────────────
+    //
+    // WS-3b step 0a (homeostatic demotion). Records the last time `retrievalSalience`
+    // was demoted for this asset so:
+    //   (a) Each run can identify assets that have been demoted but not yet
+    //       re-retrieved (they stay in the demoted state until a retrieval
+    //       re-promotes them via `upsertAssetSalience`).
+    //   (b) The homeostatic pass can log "N assets demoted this run".
+    //
+    // NULL = never demoted (or was re-promoted after last demotion, since a fresh
+    // `upsertAssetSalience` call clears the flag by updating retrieval_salience
+    // from live data rather than the demoted value — the column is informational,
+    // not the canonical source of the salience value).
+    {
+        id: "011-asset-salience-homeostatic-demoted-at",
+        up: `
+      ALTER TABLE asset_salience ADD COLUMN homeostatic_demoted_at INTEGER DEFAULT NULL;
+    `,
+    },
+    // ── Migration 012 — improve_gate_thresholds (WS-4 per-phase threshold store) ─
+    //
+    // Persists the auto-tuned accept-gate threshold PER PHASE so that each phase
+    // (reflect, distill, extract, consolidate) maintains its own calibrated
+    // threshold rather than sharing a single global `options.autoAccept`.
+    //
+    // Schema:
+    //   phase TEXT PK        — phase label, e.g. "reflect", "distill", "extract",
+    //                          "consolidate".
+    //   threshold INTEGER    — tuned threshold (0-100), matches the integer
+    //                          scale used everywhere else in the gate pipeline.
+    //   updated_at INTEGER   — Unix milliseconds of the last update.
+    //
+    // `makeGateConfig` reads the stored threshold for its phase (falling back to
+    // the caller-supplied `globalThreshold` when no row exists yet). WS-4's
+    // `persistPhaseThreshold` writes it after each auto-tune step.
+    {
+        id: "012-improve-gate-thresholds",
+        up: `
+      CREATE TABLE IF NOT EXISTS improve_gate_thresholds (
+        phase       TEXT    NOT NULL PRIMARY KEY,
+        threshold   INTEGER NOT NULL,
+        updated_at  INTEGER NOT NULL
+      );
+    `,
+    },
+    // ── Migration 013 — extract_sessions_seen: content_hash column (#602) ────────
+    //
+    // Replaces the brittle timestamp-based incrementality (`session_ended_at`,
+    // compared against the live session metadata) with a content hash. The old
+    // clock/timestamp logic caused the Jun 11-12 double-extract + over-throttle
+    // incident (clock skew / out-of-order endedAt both double-processed AND
+    // over-suppressed sessions). The hash makes the skip decision byte-exact and
+    // clock-independent.
+    //
+    // Additive ADD COLUMN (migration-safe; mirrors migration 011's style). All
+    // pre-existing rows read back `content_hash = NULL`, which the skip logic
+    // treats as "seen before content-hash tracking existed → process once to
+    // backfill", after which the row gets a real hash and becomes hash-stable.
+    // Never mutate migration 004 (the original table) — this column is appended.
+    {
+        id: "013-extract-sessions-content-hash",
+        up: `
+      ALTER TABLE extract_sessions_seen ADD COLUMN content_hash TEXT DEFAULT NULL;
+    `,
+    },
+    // ── Migration 014 — recombine_hypotheses (#625 confirmation count) ───────────
+    //
+    // Second-pass promotion ledger for the recombine pass. The first pass (#609)
+    // only ever emits `type: hypothesis` proposals; this table tracks how many
+    // CONSECUTIVE runs re-induced the SAME generalization (keyed by the
+    // deterministic `deriveRecombineLessonRef` value — a hash of the sorted
+    // member entryKeys). Once `consecutive_count >= confirmThreshold`, the run
+    // promotes the generalization to a `type: lesson` proposal (through the same
+    // proposal queue + quality gate). A hypothesis NOT re-induced in a run has its
+    // consecutive streak reset (decay-to-zero), so confirmation is per exact
+    // member-set and conservative.
+    //
+    // Indexed columns:
+    //   hypothesis_ref TEXT PK — the `lesson:recombined/<slug>-<hash>` ref; one
+    //                            row per re-inducible generalization. The ref is
+    //                            the promotion TARGET (a lesson in both the
+    //                            hypothesis and promoted states), so the ref never
+    //                            encodes the proposal type.
+    //   last_seen_at   TEXT (idx) — for forensic / pruning queries.
+    //
+    // Non-indexed columns:
+    //   signature          TEXT — the cluster's shared relatedness signal (tag /
+    //                             entity) at induction time; forensics only.
+    //   member_key         TEXT — sorted member entryKeys joined; the membership
+    //                             fingerprint behind the ref hash. Stored so a
+    //                             membership change (which yields a DIFFERENT ref)
+    //                             is auditable.
+    //   consecutive_count  INTEGER — current confirmation streak (reset on decay
+    //                                and on promotion).
+    //   first_seen_at      TEXT — ISO-8601 UTC of the first induction.
+    //   last_run           TEXT — sourceRun token of the last induction; the
+    //                             same-run idempotency guard.
+    //   promoted_at        TEXT — non-null once promoted; guards against
+    //                             double-promoting on every subsequent run.
+    //   metadata_json      TEXT — reserved for future forensics; defaults to '{}'.
+    {
+        id: "014-recombine-hypotheses",
+        up: `
+      CREATE TABLE IF NOT EXISTS recombine_hypotheses (
+        hypothesis_ref    TEXT PRIMARY KEY,
+        signature         TEXT NOT NULL,
+        member_key        TEXT NOT NULL,
+        consecutive_count INTEGER NOT NULL DEFAULT 0,
+        first_seen_at     TEXT NOT NULL,
+        last_seen_at      TEXT NOT NULL,
+        last_run          TEXT,
+        promoted_at       TEXT,
+        metadata_json     TEXT NOT NULL DEFAULT '{}'
+      );
+      CREATE INDEX IF NOT EXISTS idx_recombine_hypotheses_last_seen
+        ON recombine_hypotheses(last_seen_at);
+    `,
+    },
+    // ── Migration 015 — asset_salience: encoding_source provenance column (#644) ──
+    //
+    // Records HOW the stored `encoding_salience` was derived so an improve run's
+    // type-weight fallback can no longer clobber a real content-derived score:
+    //
+    //   "content"   — written by the distill path from `scoreEncodingSalience`
+    //                 (novelty·0.40 + magnitude·0.35 + predictionError·0.25).
+    //   "type-stub" — written by `computeSalience`'s `DEFAULT_TYPE_ENCODING_WEIGHTS`
+    //                 fallback (no content-based score available for this ref yet).
+    //   NULL        — legacy row written before this migration; provenance unknown.
+    //                 Treated as a stub by readers UNLESS its stored value differs
+    //                 from the pure type-weight (best-effort heuristic for old data).
+    //
+    // Why a column rather than inference: before #644, every improve run overwrote
+    // the distill-written content score with the type-weight stub, so the stored
+    // value alone cannot distinguish a real score from a stub. The provenance flag
+    // is the single source of truth going forward; `upsertAssetSalience` refuses to
+    // lower a "content" row to a "type-stub" so the high-salience gate (#608) keys
+    // on genuine novelty/magnitude/prediction-error, not the asset type.
+    {
+        id: "015-asset-salience-encoding-source",
+        up: `
+      ALTER TABLE asset_salience ADD COLUMN encoding_source TEXT DEFAULT NULL;
+    `,
+    },
 ];
 /**
  * Apply every pending migration in a single transaction per migration.
@@ -562,6 +890,9 @@ export function proposalRowToProposal(row) {
         ...(typeof meta.confidence === "number" ? { confidence: meta.confidence } : {}),
         ...(meta.gateDecision !== undefined ? { gateDecision: meta.gateDecision } : {}),
         ...(typeof meta.backupContent === "string" ? { backupContent: meta.backupContent } : {}),
+        ...(typeof meta.eligibilitySource === "string"
+            ? { eligibilitySource: meta.eligibilitySource }
+            : {}),
     };
 }
 /**
@@ -581,6 +912,8 @@ export function proposalToRowValues(proposal, stashDir) {
         metaObj.gateDecision = proposal.gateDecision;
     if (proposal.backupContent !== undefined)
         metaObj.backupContent = proposal.backupContent;
+    if (proposal.eligibilitySource !== undefined)
+        metaObj.eligibilitySource = proposal.eligibilitySource;
     return {
         id: proposal.id,
         stash_dir: stashDir,
@@ -723,6 +1056,57 @@ export function listStateProposals(db, options = {}) {
         .all(...params);
     return rows.map(proposalRowToProposal);
 }
+/**
+ * Read every proposal's `gateDecision` record across all stashes (#612).
+ *
+ * Calibration reads the auto-accept gate's per-proposal decisions regardless of
+ * the proposal's current lifecycle status — a proposal that was auto-accepted
+ * is now `accepted`, an auto-rejected one stays `pending`, so filtering by
+ * status would drop half the join. Rows without a `gateDecision` (created
+ * before #577, or never gated) are skipped. The result is ordered by
+ * `decidedAt ASC` for deterministic downstream aggregation, falling back to
+ * `created_at` ordering from the SQL layer for rows with equal/missing
+ * timestamps.
+ */
+export function listProposalGateDecisions(db) {
+    const rows = db.prepare("SELECT metadata_json FROM proposals ORDER BY created_at ASC, rowid ASC").all();
+    const decisions = [];
+    for (const row of rows) {
+        let meta;
+        try {
+            meta = JSON.parse(row.metadata_json);
+        }
+        catch {
+            continue;
+        }
+        const decision = meta.gateDecision;
+        if (decision && typeof decision === "object" && typeof decision.outcome === "string") {
+            decisions.push(decision);
+        }
+    }
+    decisions.sort((a, b) => new Date(a.decidedAt).getTime() - new Date(b.decidedAt).getTime());
+    return decisions;
+}
+// ── WS-4: Per-phase gate threshold store (Migration 012) ─────────────────────
+/**
+ * Read the persisted auto-tuned threshold for a gate phase.
+ *
+ * Returns `undefined` when no row exists yet (first run, or the phase has
+ * never been tuned). The caller falls back to the global `options.autoAccept`
+ * in that case.
+ */
+export function getPhaseThreshold(db, phase) {
+    const row = db.prepare("SELECT threshold FROM improve_gate_thresholds WHERE phase = ?").get(phase);
+    return row?.threshold;
+}
+/**
+ * Persist the auto-tuned threshold for a gate phase.
+ * Uses INSERT OR REPLACE so the call is idempotent (upsert semantics).
+ */
+export function persistPhaseThreshold(db, phase, threshold) {
+    db.prepare(`INSERT OR REPLACE INTO improve_gate_thresholds (phase, threshold, updated_at)
+     VALUES (?, ?, ?)`).run(phase, Math.round(threshold), Date.now());
+}
 /**
  * Look up a single proposal by id, optionally scoped to one stash root.
  * Returns undefined when not found.
@@ -785,6 +1169,32 @@ export function insertProposalIfAbsent(db, proposal, stashDir) {
     const changes = result.changes ?? 0;
     return Number(changes) > 0;
 }
+/**
+ * Run `fn` inside a `BEGIN IMMEDIATE` transaction.
+ *
+ * `db.transaction()` is DEFERRED by default on both Bun and better-sqlite3,
+ * which means two writers can both perform stale preflight reads and only race
+ * when they finally attempt the write. Proposal creation and queue mutation
+ * need the write lock BEFORE those reads so concurrent processes serialize on
+ * the live queue state rather than clobbering each other.
+ */
+export function withImmediateTransaction(db, fn) {
+    db.exec("BEGIN IMMEDIATE");
+    try {
+        const result = fn();
+        db.exec("COMMIT");
+        return result;
+    }
+    catch (err) {
+        try {
+            db.exec("ROLLBACK");
+        }
+        catch {
+            // Ignore rollback failures so the original error is preserved.
+        }
+        throw err;
+    }
+}
 // ── task_history table helpers ───────────────────────────────────────────────
 /**
  * Upsert a task history row.
@@ -1082,9 +1492,11 @@ export function purgeOldImproveRuns(db, retentionDays = 90) {
 }
 /**
  * Record (or update) one session's extract outcome. INSERT-OR-REPLACE so the
- * row reflects the most recent run; downstream skip-logic compares
- * `session_ended_at` against the live session metadata to decide if anything
- * new arrived since `processed_at`.
+ * row reflects the most recent run. The `content_hash` persisted here is what
+ * the NEXT run compares against (#602): a byte-identical session is skipped, a
+ * changed session is re-processed, and a NULL-backfill row becomes hash-stable
+ * after its one reprocess. `session_ended_at` is still written for
+ * telemetry/forensics but is no longer the skip authority.
  */
 export function upsertExtractedSession(db, input) {
     const endedAtIso = typeof input.sessionEndedAt === "number" && Number.isFinite(input.sessionEndedAt)
@@ -1093,9 +1505,10 @@ export function upsertExtractedSession(db, input) {
     db.prepare(`
     INSERT OR REPLACE INTO extract_sessions_seen
       (harness, session_id, processed_at, session_ended_at, outcome,
-       candidate_count, proposal_count, rationale, source_run, metadata_json)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-  `).run(input.harness, input.sessionId, input.processedAt, endedAtIso, input.outcome, input.candidateCount, input.proposalCount, input.rationale ?? null, input.sourceRun ?? null, JSON.stringify(input.metadata ?? {}));
+       candidate_count, proposal_count, rationale, source_run, metadata_json,
+       content_hash)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(input.harness, input.sessionId, input.processedAt, endedAtIso, input.outcome, input.candidateCount, input.proposalCount, input.rationale ?? null, input.sourceRun ?? null, JSON.stringify(input.metadata ?? {}), input.contentHash);
 }
 /**
  * Fetch a single session's last extract record, or `undefined` when the
@@ -1133,31 +1546,279 @@ export function getExtractedSessionsMap(db, harness, sessionIds) {
     return out;
 }
 /**
- * Decide whether a session should be skipped because the extractor has
- * already processed it AND nothing has changed since. The "anything new since
- * last extract?" rule is: the live `sessionEndedAtMs` is strictly later than
- * the recorded `session_ended_at`. Same-or-earlier endedAt means we'd be
- * re-processing the exact same content for no gain.
+ * The most recent extract-run time for a harness — `MAX(processed_at)` across
+ * its ledger rows, as ms epoch — or `null` when the harness has never been
+ * extracted. Used to default the discovery window to "since the last run" so an
+ * intermittently-online host that was off for days still rediscovers sessions
+ * that ended during the gap (the content-hash ledger keeps the widened window
+ * free of redundant LLM cost).
+ */
+export function getLastExtractRunAt(db, harness) {
+    const row = db
+        .prepare("SELECT MAX(processed_at) AS last FROM extract_sessions_seen WHERE harness = ?")
+        .get(harness);
+    if (!row?.last)
+        return null;
+    const ms = Date.parse(row.last);
+    return Number.isFinite(ms) ? ms : null;
+}
+/**
+ * Decide whether a session should be skipped because the extractor has already
+ * processed BYTE-IDENTICAL content (#602). The skip authority is the content
+ * hash, NOT `session_ended_at` — this is clock-independent, so it is immune to
+ * the clock-skew / out-of-order-endedAt problems that caused the Jun 11-12
+ * double-extract + over-throttle incident.
  *
- * Returns:
- *   - `false` — no prior row, or session has new content since last extract.
- *     The caller should process it.
- *   - `true`  — the session was already processed and hasn't been updated.
- *     The caller should skip.
+ * Rules:
+ *   - no prior row              → `false` (never seen → process; AC3).
+ *   - prior.content_hash == null → `false` (legacy / hash-less row → process
+ *     exactly once to backfill the hash, then it becomes hash-stable; AC4).
+ *   - hashes equal              → `true`  (unchanged content → skip; AC1).
+ *   - hashes differ             → `false` (changed content → re-process; AC2).
  */
-export function shouldSkipAlreadyExtractedSession(prior, liveSessionEndedAtMs) {
+export function shouldSkipAlreadyExtractedSession(prior, currentContentHash) {
     if (!prior)
         return false;
-    // No live timestamp → can't tell if anything's new. Be conservative and
-    // skip — the operator can pass --force later if we add it.
-    if (typeof liveSessionEndedAtMs !== "number" || !Number.isFinite(liveSessionEndedAtMs)) {
-        return true;
-    }
-    const priorMs = prior.session_ended_at ? Date.parse(prior.session_ended_at) : Number.NaN;
-    if (!Number.isFinite(priorMs))
+    if (prior.content_hash == null)
         return false;
-    // Re-process when there's new content; skip when the session is unchanged.
-    return liveSessionEndedAtMs <= priorMs;
+    return prior.content_hash === currentContentHash;
+}
+/**
+ * Bulk-fetch the judged-state cache for a set of entry keys in one query.
+ * Returns a Map keyed by entry_key so the consolidate pool-selection loop can
+ * do O(1) "has this memory been judged at this content hash?" lookups.
+ * Empty input → empty map (no query issued).
+ */
+export function getConsolidationJudgedMap(db, entryKeys) {
+    const out = new Map();
+    if (entryKeys.length === 0)
+        return out;
+    // SQLite has a ~999 param ceiling; chunk if a caller ever exceeds that.
+    const CHUNK = 500;
+    for (let i = 0; i < entryKeys.length; i += CHUNK) {
+        const chunk = entryKeys.slice(i, i + CHUNK);
+        const placeholders = chunk.map(() => "?").join(",");
+        const rows = db
+            .prepare(`SELECT * FROM consolidation_judged WHERE entry_key IN (${placeholders})`)
+            .all(...chunk);
+        for (const row of rows)
+            out.set(row.entry_key, row);
+    }
+    return out;
+}
+/**
+ * Record (or update) the judged state for one memory. INSERT-OR-REPLACE so the
+ * row always reflects the most recent judge of that entry_key. Called once per
+ * memory the consolidate LLM saw in a successfully-judged chunk.
+ */
+export function upsertConsolidationJudged(db, input) {
+    db.prepare(`
+    INSERT OR REPLACE INTO consolidation_judged
+      (entry_key, content_hash, judged_at, outcome)
+    VALUES (?, ?, ?, ?)
+  `).run(input.entryKey, input.contentHash, input.judgedAt, input.outcome);
+}
+/**
+ * Record an induction of a recombine hypothesis and return the new consecutive
+ * count. INSERT … ON CONFLICT increments the streak, but the `last_run` guard
+ * makes a repeated call within the SAME run idempotent (no double-increment if
+ * the same ref appears twice in one run). On insert the streak starts at 1.
+ */
+export function recordRecombineInduction(db, input) {
+    const row = db
+        .prepare(`
+      INSERT INTO recombine_hypotheses
+        (hypothesis_ref, signature, member_key, consecutive_count, first_seen_at, last_seen_at, last_run)
+      VALUES (?, ?, ?, 1, ?, ?, ?)
+      ON CONFLICT(hypothesis_ref) DO UPDATE SET
+        consecutive_count = consecutive_count + (CASE WHEN last_run IS excluded.last_run THEN 0 ELSE 1 END),
+        last_seen_at = excluded.last_seen_at,
+        last_run = excluded.last_run,
+        signature = excluded.signature,
+        member_key = excluded.member_key
+      RETURNING consecutive_count
+    `)
+        .get(input.hypothesisRef, input.signature, input.memberKey, input.seenAt, input.seenAt, input.run);
+    return row?.consecutive_count ?? 0;
+}
+/**
+ * #633 — find an existing pending (non-promoted) hypothesis row whose cluster
+ * is the SAME generalization as a newly-induced one, matched by SIGNATURE plus
+ * a Jaccard membership-overlap test, rather than an exact member-set hash.
+ *
+ * In a growing stash any added/removed memory changes the exact member set, so
+ * the ref hash (and member_key) shift every run → a fresh row at count=1 → the
+ * streak never reaches `confirmThreshold` and nothing ever promotes. Matching
+ * on overlap lets a drifting-but-stable cluster keep accumulating under one row.
+ *
+ * Returns the matched row with the HIGHEST overlap (ties broken by most-recent
+ * `last_seen_at`), or `undefined` when none clears `minOverlap`. Already-promoted
+ * rows are ignored so a confirmed lesson is not reopened by a later induction.
+ *
+ * @param memberKey  the candidate cluster's membership fingerprint
+ *                   (sorted member entryKeys joined by `|`).
+ * @param minOverlap Jaccard threshold in [0,1]; a candidate matches when
+ *                   |A∩B| / |A∪B| >= minOverlap.
+ */
+export function findMatchingRecombineHypothesis(db, input) {
+    const candidateMembers = new Set(input.memberKey.split("|").filter((m) => m.length > 0));
+    if (candidateMembers.size === 0)
+        return undefined;
+    const rows = db
+        .prepare("SELECT * FROM recombine_hypotheses WHERE signature = ? AND promoted_at IS NULL ORDER BY last_seen_at DESC")
+        .all(input.signature);
+    let best;
+    let bestOverlap = -1;
+    for (const row of rows) {
+        const rowMembers = row.member_key.split("|").filter((m) => m.length > 0);
+        if (rowMembers.length === 0)
+            continue;
+        let intersection = 0;
+        for (const m of rowMembers) {
+            if (candidateMembers.has(m))
+                intersection += 1;
+        }
+        const union = candidateMembers.size + rowMembers.length - intersection;
+        const overlap = union === 0 ? 0 : intersection / union;
+        // rows are ordered last_seen_at DESC, so a strict `>` keeps the most-recent
+        // row on ties.
+        if (overlap >= input.minOverlap && overlap > bestOverlap) {
+            best = row;
+            bestOverlap = overlap;
+        }
+    }
+    return best;
+}
+/**
+ * Fetch a single recombine hypothesis row, or `undefined` when the ref has
+ * never been induced. Normalizes bun:sqlite null → undefined like
+ * {@link getExtractedSession}.
+ */
+export function getRecombineHypothesis(db, hypothesisRef) {
+    const row = db
+        .prepare("SELECT * FROM recombine_hypotheses WHERE hypothesis_ref = ?")
+        .get(hypothesisRef);
+    return row ?? undefined;
+}
+/**
+ * Mark a hypothesis promoted: stamp `promoted_at` and reset the consecutive
+ * count to 0, so it must re-accumulate a full confirmation streak before it can
+ * promote again. The `promoted_at` non-null state is the double-promotion guard.
+ */
+export function markRecombineHypothesisPromoted(db, hypothesisRef, promotedAt) {
+    db.prepare("UPDATE recombine_hypotheses SET promoted_at = ?, consecutive_count = 0 WHERE hypothesis_ref = ?").run(promotedAt, hypothesisRef);
+}
+/**
+ * Decay-to-zero every NON-promoted hypothesis NOT re-induced in the current run.
+ *
+ * A generalization that stops being supported by the corpus has lost its
+ * confirmation streak, so we hard-reset `consecutive_count` to 0 (the
+ * alternative — `count - 1` floored at 0 — tolerates a single noisy run but
+ * blurs the "consecutive" semantics; hard-reset is the conservative choice).
+ *
+ * Only rows whose `hypothesis_ref` is NOT in `seenRefs` AND whose `last_run` is
+ * NOT the current run are decayed. Already-promoted rows are left alone.
+ * Returns the number of rows reset.
+ */
+export function decayUnseenRecombineHypotheses(db, currentRun, seenRefs) {
+    // Reset every eligible row, then exclude the seen refs in chunks to respect
+    // the ~999 SQLite param ceiling. With no seen refs we reset all non-promoted
+    // rows from prior runs in a single statement.
+    if (seenRefs.length === 0) {
+        const res = db
+            .prepare("UPDATE recombine_hypotheses SET consecutive_count = 0 WHERE promoted_at IS NULL AND (last_run IS NULL OR last_run != ?) AND consecutive_count != 0")
+            .run(currentRun);
+        return Number(res.changes);
+    }
+    // A single NOT IN keeps the exclusion atomic (a chunked NOT IN would let a ref
+    // excluded by one chunk still be reset by another chunk's statement). The
+    // recombine pass caps re-induced clusters at `maxClustersPerRun` (a handful),
+    // so `seenRefs` is far under SQLite's ~999 param ceiling in practice; we cap
+    // defensively and fall back to resetting all when somehow exceeded.
+    if (seenRefs.length > 900) {
+        const res = db
+            .prepare("UPDATE recombine_hypotheses SET consecutive_count = 0 WHERE promoted_at IS NULL AND (last_run IS NULL OR last_run != ?) AND consecutive_count != 0")
+            .run(currentRun);
+        return Number(res.changes);
+    }
+    const placeholders = seenRefs.map(() => "?").join(",");
+    const res = db
+        .prepare(`UPDATE recombine_hypotheses SET consecutive_count = 0
+       WHERE promoted_at IS NULL
+         AND (last_run IS NULL OR last_run != ?)
+         AND consecutive_count != 0
+         AND hypothesis_ref NOT IN (${placeholders})`)
+        .run(currentRun, ...seenRefs);
+    return Number(res.changes);
+}
+/**
+ * Convert a `number[]` embedding vector to the `Float32Array` byte
+ * representation stored in the `body_embeddings.embedding` BLOB column.
+ */
+export function embeddingToBlob(vec) {
+    const f32 = new Float32Array(vec);
+    return new Uint8Array(f32.buffer);
+}
+/**
+ * Convert the raw `Uint8Array` bytes from the `body_embeddings.embedding`
+ * BLOB column back to a `number[]` embedding vector.
+ */
+export function blobToEmbedding(blob) {
+    // SQLite BLOB columns are returned as Uint8Array; re-interpret as Float32.
+    const f32 = new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
+    return Array.from(f32);
+}
+/**
+ * Bulk-fetch cached body embeddings for a set of content hashes.
+ * Returns a Map keyed by `content_hash` (embedding decoded to `number[]`).
+ * Empty input → empty map (no query issued).
+ *
+ * If the stored `model_id` does not match `expectedModelId` the entire table
+ * is cleared (drop-all on model mismatch) and an empty map is returned so
+ * callers re-embed everything on this run.
+ */
+export function getBodyEmbeddings(db, contentHashes, expectedModelId) {
+    const out = new Map();
+    if (contentHashes.length === 0)
+        return out;
+    // Model-id mismatch: vectors are in the wrong metric space — drop all rows.
+    const firstRow = db.prepare("SELECT model_id FROM body_embeddings LIMIT 1").get();
+    if (firstRow && firstRow.model_id !== expectedModelId) {
+        db.exec("DELETE FROM body_embeddings");
+        return out;
+    }
+    // SQLite has a ~999 param ceiling; chunk if needed.
+    const CHUNK = 500;
+    for (let i = 0; i < contentHashes.length; i += CHUNK) {
+        const chunk = contentHashes.slice(i, i + CHUNK);
+        const placeholders = chunk.map(() => "?").join(",");
+        const rows = db
+            .prepare(`SELECT content_hash, embedding FROM body_embeddings WHERE content_hash IN (${placeholders})`)
+            .all(...chunk);
+        for (const row of rows) {
+            out.set(row.content_hash, blobToEmbedding(row.embedding));
+        }
+    }
+    return out;
+}
+/**
+ * Upsert body-embedding rows in a single transaction.
+ * Each entry maps a `cacheHash` → `number[]` vector. `model_id` is stored
+ * so a future model change can trigger a drop-all purge.
+ */
+export function upsertBodyEmbeddings(db, entries) {
+    if (entries.length === 0)
+        return;
+    const now = Date.now();
+    const stmt = db.prepare(`
+    INSERT OR REPLACE INTO body_embeddings (content_hash, embedding, model_id, created_at)
+    VALUES (?, ?, ?, ?)
+  `);
+    db.transaction(() => {
+        for (const { contentHash, embedding, modelId } of entries) {
+            stmt.run(contentHash, embeddingToBlob(embedding), modelId, now);
+        }
+    })();
 }
 // ── registry_index_cache (goes in index.db, not state.db) ───────────────────
 /**