akm-cli 0.9.0-beta.9 → 0.9.0-rc.0

package/dist/commands/improve/salience.js

@@ -0,0 +1,455 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * WS-1 — Unified SALIENCE model (S1 seam).
+ *
+ * Replaces the three competing attention formulas that existed across:
+ *   - proactive-maintenance.ts:186 — `importance × log(1+freq) × recencyDecay / log10(size)`
+ *   - feedback-valence.ts:111      — `combinedEligibilityScore = utility·0.7 + valence·0.3`
+ *   - getUtilityScoresByIds        — MemRL utility EMA (#386)
+ *
+ * ## Salience vector (three independently-stored, independently-decayable sub-scores)
+ *
+ * | Sub-score         | Brain analogy          | Source                          |
+ * |-------------------|------------------------|---------------------------------|
+ * | `encodingSalience`| Amygdala tagging (Gap 1)| Set at extract; v1 = type weight|
+ * | `outcomeSalience` | Dopaminergic outcome   | WS-2 (0 until that lands)       |
+ * | `retrievalSalience`| Hippocampal frequency+recency | usage_events + utility_scores |
+ *
+ * ## Projection
+ *
+ * `rankScore = (w_e·encoding + w_o·outcome + w_r·retrieval) × sizePenalty`, normalized [0,1].
+ *
+ * **WS-2 default-ON (R1 loop closure):**
+ * `w_o = 0.15` is applied by default now that `outcome_score` saturates at
+ * `OUTCOME_SCORE_MAX` (G2). Operators can opt out via
+ * `improve.salience.outcomeWeightEnabled: false`, which restores the WS-1
+ * parity weights `w_e=0.30, w_r=0.70, w_o=0`. `outcomeSalience` is populated
+ * from `asset_outcome.outcome_score` regardless of the flag.
+ *
+ * ## Plasticity
+ *
+ * `consecutive_no_ops` (INTEGER column in `asset_salience`) dampens CONSOLIDATION-
+ * SELECTION only — it is intentionally NOT applied to `rankScore`, so a stable
+ * asset stays fully retrievable while no longer consuming repeated LLM merge
+ * attempts. See `getConsecutiveNoOps` / `recordNoOp` / `resetConsecutiveNoOps`.
+ *
+ * ## Canonical store
+ *
+ * The three sub-scores live in `state.db :: asset_salience` (canonical).
+ * An optional frontmatter mirror of the stable `encodingSalience` is allowed for
+ * portability (issue #608 may write it there), but state.db is the source of truth
+ * for ranking. This prevents the frontmatter-vs-state.db split that issue #608
+ * would otherwise create (#608 pull-forward decision from Part VI).
+ *
+ * @module salience
+ */
+import { makeAssetRef } from "../../core/asset/asset-ref.js";
+import { getAllEntries, getUtilityScoresByIds } from "../../indexer/db/db.js";
+import { WARM_START_CAP } from "./outcome-loop.js";
+// ── One day in ms ─────────────────────────────────────────────────────────────
+const DAY_MS = 86_400_000;
+// ── Recency decay half-life (mirrors the proactive-maintenance prototype) ─────
+const RECENCY_HALFLIFE_DAYS = 21;
+// ── Recency-floor half-life (R4 — SHY-style continuous downscaling) ──────────
+//
+// The recency floor itself decays on this (much longer) half-life so an
+// unreviewed-forever asset keeps drifting down instead of parking at the 0.1
+// floor. This replaces the deleted homeostatic demotion pass (which was
+// default-off and self-undoing — every salience recompute clobbered it);
+// folding the decay into the always-applied recency term makes it persist by
+// construction. At 180 days the floor halves; a 1-year-stale asset sits at
+// ~0.025 instead of 0.1.
+const RECENCY_FLOOR_HALFLIFE_DAYS = 180;
+// Absolute epsilon under the decaying floor. Keeps the frequency term ordinal
+// for assets whose last-use timestamp is unknown (utility_scores has no
+// last_used_at) — without it their retrieval salience collapses to exactly 0
+// and frequency ordering is lost for maintenance selection.
+const RECENCY_EPSILON = 0.01;
+// ── Size proxy floor (avoids log10(0)) ────────────────────────────────────────
+const SIZE_FLOOR_BYTES = 200;
+// ── Projection weights ────────────────────────────────────────────────────────
+//
+// These constants are the DEFAULT ranking weights (R1 loop closure). Operators
+// can opt back out to the WS-1 parity weights (w_e=0.30, w_r=0.70, w_o=0) via
+// `improve.salience.outcomeWeightEnabled: false`.
+//
+// WS-2 split (w_e=0.25, w_o=0.15, w_r=0.60, sum = 1.0):
+// [exp] Expert recommendation: encoding should be moderate so a type-importance
+// stub does not completely dominate; retrieval should be strong since it directly
+// measures use; outcome provides a quality signal proportional to usefulness.
+//
+// Re-tune via the Part-V measurement protocol if the throughput/quality gate
+// shows regression after enabling the outcome weight.
+export const W_ENCODING = 0.25; // WS-2 target encoding weight (w_e)
+export const W_OUTCOME = 0.15; // WS-2 target outcome weight (w_o)
+export const W_RETRIEVAL = 0.6; // WS-2 target retrieval weight (w_r)
+// Compile-time guard: weights must sum to 1.0 (±ε). The TS initializer runs
+// at module load, not build time, so this acts as a startup assertion.
+if (Math.abs(W_ENCODING + W_OUTCOME + W_RETRIEVAL - 1.0) > 1e-9) {
+    throw new Error(`salience.ts: W_ENCODING + W_OUTCOME + W_RETRIEVAL must equal 1.0 (got ${W_ENCODING + W_OUTCOME + W_RETRIEVAL})`);
+}
+// ── WS-1 parity weights ───────────────────────────────────────────────────────
+//
+// These constants reflect the WS-1 parity weights used when the operator
+// explicitly opts out (`outcomeWeightEnabled: false`). They preserve the
+// WS-1 two-way split (w_e=0.30, w_r=0.70) with w_o=0 so outcome does not
+// affect rankScore in the opt-out mode.
+//
+// Named here (rather than inline literals in the else branch) so a future
+// re-tune has a single source of truth and the sum-to-1 guard below catches
+// any accidental mis-edit.
+export const W_ENCODING_PARITY = 0.3; // WS-1 parity encoding weight
+export const W_OUTCOME_PARITY = 0; // WS-1 parity outcome weight (0 = disabled)
+export const W_RETRIEVAL_PARITY = 0.7; // WS-1 parity retrieval weight
+// Startup guard: parity triple must also sum to 1.0 (±ε).
+if (Math.abs(W_ENCODING_PARITY + W_OUTCOME_PARITY + W_RETRIEVAL_PARITY - 1.0) > 1e-9) {
+    throw new Error(`salience.ts: W_ENCODING_PARITY + W_OUTCOME_PARITY + W_RETRIEVAL_PARITY must equal 1.0 (got ${W_ENCODING_PARITY + W_OUTCOME_PARITY + W_RETRIEVAL_PARITY})`);
+}
+// ── Type-importance fallback weights (#608 landed) ────────────────────────────
+//
+// The real encoding salience estimator is `scoreEncodingSalience` in
+// `encoding-salience.ts` (#608). These weights are the fallback used when
+// `SalienceInputs.encodingSalience` is absent (pre-#608 assets without a
+// frontmatter `salience:` field or a state.db row seeded by distill).
+export const DEFAULT_TYPE_ENCODING_WEIGHTS = Object.freeze({
+    skill: 0.9,
+    agent: 0.9,
+    command: 0.8,
+    workflow: 0.8,
+    lesson: 0.75,
+    knowledge: 0.7,
+    script: 0.6,
+    memory: 0.5,
+});
+/** Default encoding salience for types not in the table above. */
+export const DEFAULT_ENCODING_SALIENCE = 0.5;
+// ── Core computation ─────────────────────────────────────────────────────────
+/**
+ * Compute the salience vector for one asset.
+ *
+ * Pure function — no I/O. All inputs are pre-fetched by the caller.
+ */
+export function computeSalience(inputs) {
+    const now = inputs.now ?? Date.now();
+    // ── Encoding salience ────────────────────────────────────────────────────────
+    //
+    // When `inputs.encodingSalience` is provided (computed by `scoreEncodingSalience`
+    // in encoding-salience.ts at extract/distill time, #608), use it directly.
+    // Fall back to the type-importance stub only when the caller has not yet
+    // computed a content-based score (e.g. on older assets before the first
+    // staleness refresh runs).
+    const encodingSource = inputs.encodingSalience !== undefined ? "content" : "type-stub";
+    const encoding = inputs.encodingSalience !== undefined
+        ? Math.min(1, Math.max(0, inputs.encodingSalience))
+        : (DEFAULT_TYPE_ENCODING_WEIGHTS[inputs.type] ?? DEFAULT_ENCODING_SALIENCE);
+    // ── Outcome salience (WS-2 active) ────────────────────────────────────────
+    //
+    // When `inputs.outcomeSalience` is provided (WS-2 has populated asset_outcome
+    // for this ref), use it directly — it has already been normalised by
+    // `outcomeScoreToSalience` in outcome-loop.ts (value in [DIVERSITY_FLOOR, 1]).
+    //
+    // When absent (new asset, no WS-2 row yet): fall back to the warm-start seed
+    // from `utilityScore` clipped to [0, WARM_START_CAP], matching the seed
+    // value that `updateAssetOutcome` writes on first row creation. This ensures
+    // `outcomeSalience` is non-zero at launch for assets with utility history
+    // (avoiding the starvation problem described in the plan §WS-2 warm start).
+    let outcome;
+    if (inputs.outcomeSalience !== undefined) {
+        // Direct pass-through — caller already normalised via outcomeScoreToSalience.
+        outcome = Math.min(1, Math.max(0, inputs.outcomeSalience));
+    }
+    else {
+        // Warm-start fallback: clip utility to [0, WARM_START_CAP] so the
+        // outcomeSalience term contributes a modest non-zero baseline.
+        outcome = Math.min(WARM_START_CAP, Math.max(0, inputs.utilityScore ?? 0));
+    }
+    // ── Retrieval salience ─────────────────────────────────────────────────────
+    //
+    // Formula: log(1 + freq) × recencyDecay
+    //   log(1+freq): sub-linear frequency term (same as proactive-maintenance prototype).
+    //   recencyDecay: max(ε, 0.1·0.5^(useAgeDays/180) + 0.5^(useAgeDays/21)) —
+    //     the fast term halves every 21 days; the 0.1 floor itself halves every
+    //     180 days (R4: SHY-style continuous downscaling — an unreviewed-forever
+    //     asset keeps drifting down instead of parking at the floor). The ε=0.01
+    //     epsilon keeps the frequency term ordinal for unknown-last-use assets.
+    //     lastUseMs=0/undefined → useAgeDays=9999 → recencyDecay=ε.
+    //
+    // The recency term is MANDATORY (plan requirement §WS-1 step 2). Without it
+    // retrievalSalience degenerates to a non-decaying frequency count. This
+    // always-applied decay replaces the deleted homeostatic demotion pass.
+    const lastUseMs = inputs.lastUseMs ?? 0;
+    const useAgeDays = lastUseMs > 0 ? (now - lastUseMs) / DAY_MS : 9999;
+    const recencyDecay = Math.max(RECENCY_EPSILON, 0.1 * 0.5 ** (useAgeDays / RECENCY_FLOOR_HALFLIFE_DAYS) + 0.5 ** (useAgeDays / RECENCY_HALFLIFE_DAYS));
+    const rawRetrieval = Math.log(1 + inputs.retrievalFreq) * recencyDecay;
+    // ── Size penalty ─────────────────────────────────────────────────────────────
+    // 1/log10(size): larger assets are slightly deprioritized (same as proactive prototype).
+    const sizeProxy = Math.max(SIZE_FLOOR_BYTES, inputs.sizeBytes ?? 0);
+    const sizePenalty = 1 / Math.log10(sizeProxy);
+    // ── Projection → rankScore ────────────────────────────────────────────────
+    //
+    // Raw projection may be > 1 (log retrieval terms can exceed 1 for high freq + fresh use).
+    // Normalize by the theoretical maximum of the retrieval component:
+    //   max retrievalRaw = log(1 + Infinity) × (0.1 + 1.0) = Infinity, so we
+    //   cap instead — rankScore is clamped to [0,1] after applying the size penalty.
+    //
+    // Normalization approach: we scale the combined linear sum to [0,1] by clamping,
+    // after applying the size penalty. The encoding term is already in [0,1]; the
+    // retrieval term is open-ended but bounded in practice by log(1+N)×1.1 where N
+    // is the retrieval count. We normalize `retrieval` to [0,1] using a soft cap:
+    //   retrieval_normalized = rawRetrieval / (rawRetrieval + 1)
+    // which asymptotes to 1 and equals 0.5 at rawRetrieval=1. This is the same
+    // formula used for MemRL utility updates.
+    const retrieval = rawRetrieval / (rawRetrieval + 1);
+    // ── Weight selection (R1 — outcome loop closed by default) ───────────────
+    //
+    // When `outcomeWeightEnabled` is true/absent (DEFAULT ON since the G2
+    // saturation cap landed): use WS-2 weights (w_e=0.25, w_o=0.15, w_r=0.60)
+    // so the prediction-error outcome signal actually shapes rankScore — this
+    // is the R1 loop-closure from docs/design/improve-self-learning-analysis.md.
+    //
+    // When `outcomeWeightEnabled` is explicitly false (operator opt-out via
+    // `improve.salience.outcomeWeightEnabled: false`): fall back to the WS-1
+    // parity weights (w_e=0.30, w_r=0.70, w_o=0). The `outcome` sub-score is
+    // still computed and stored for observability in that mode.
+    let we;
+    let wo;
+    let wr;
+    if (inputs.outcomeWeightEnabled !== false) {
+        // WS-2 active (default): three-way split.
+        we = W_ENCODING; // 0.25
+        wo = W_OUTCOME; // 0.15
+        wr = W_RETRIEVAL; // 0.60
+    }
+    else {
+        // WS-1 parity (opt-out): w_o=0, redistribute to WS-1 proportions.
+        // Original WS-1 split was w_e=0.30, w_r=0.70.
+        we = W_ENCODING_PARITY;
+        wo = W_OUTCOME_PARITY;
+        wr = W_RETRIEVAL_PARITY;
+    }
+    const rawRankScore = (we * encoding + wo * outcome + wr * retrieval) * sizePenalty;
+    const rankScore = Math.min(1, Math.max(0, rawRankScore));
+    return { encoding, outcome, retrieval, rankScore, encodingSource };
+}
+/**
+ * Does this row carry a genuine content-derived `encoding_salience` (#644)?
+ *
+ * Returns true when the provenance flag is `"content"`. For legacy rows
+ * (`encoding_source === null`, written before migration 015) we apply a
+ * conservative heuristic: treat the stored value as content-derived only when it
+ * does NOT equal the pure type-weight stub for the asset's type — because before
+ * the #644 fix every run overwrote real scores with the stub, a value that still
+ * differs from the stub must have been content-written and never re-clobbered.
+ * When the type cannot be determined (no `type` given) a null-provenance row is
+ * treated as a stub (the safe default).
+ */
+export function isContentEncodingRow(row, type) {
+    if (row.encoding_source === "content")
+        return true;
+    if (row.encoding_source === "type-stub")
+        return false;
+    // Legacy NULL provenance: differ-from-stub heuristic.
+    if (!type)
+        return false;
+    const stub = DEFAULT_TYPE_ENCODING_WEIGHTS[type] ?? DEFAULT_ENCODING_SALIENCE;
+    return Math.abs(row.encoding_salience - stub) > 1e-9;
+}
+/**
+ * Upsert salience scores for one asset into state.db.
+ *
+ * Idempotent: safe to call every run; updates the outcome / retrieval / rank
+ * columns on conflict.
+ *
+ * #644 — encoding provenance guard: the `encoding_salience` + `encoding_source`
+ * columns are NOT lowered from a real content-derived score to a type-weight
+ * stub. When the stored row is `encoding_source = 'content'` and the incoming
+ * vector is a `type-stub` fallback, the stored encoding score and its provenance
+ * are preserved (only the other sub-scores and `rank_score` advance). A `content`
+ * write always wins; a `type-stub` write only seeds a row that has no content
+ * score yet. This stops the improve loop's type-weight fallback re-asserting the
+ * stub over a distill-written score on every run.
+ *
+ * NOTE: when the guard preserves the stored encoding score, the incoming
+ * `vector.rankScore` (computed from the stub encoding) is still written. Callers
+ * that want the rank_score to reflect the preserved content score should pass the
+ * stored content score back in as `inputs.encodingSalience` to `computeSalience`
+ * — which the improve loop does. The guard here is the defensive backstop.
+ */
+export function upsertAssetSalience(db, ref, vector, now) {
+    const ts = now ?? Date.now();
+    db.prepare(`INSERT INTO asset_salience
+       (asset_ref, encoding_salience, outcome_salience, retrieval_salience, rank_score, consecutive_no_ops, updated_at, encoding_source)
+     VALUES (?, ?, ?, ?, ?, 0, ?, ?)
+     ON CONFLICT(asset_ref) DO UPDATE SET
+       -- #644: never lower a real content-derived score to a type-weight stub.
+       -- Keep the stored encoding score + provenance when the stored row is
+       -- 'content' and the incoming write is a 'type-stub' fallback.
+       encoding_salience  = CASE
+         WHEN asset_salience.encoding_source = 'content' AND excluded.encoding_source = 'type-stub'
+           THEN asset_salience.encoding_salience
+           ELSE excluded.encoding_salience
+       END,
+       encoding_source    = CASE
+         WHEN asset_salience.encoding_source = 'content' AND excluded.encoding_source = 'type-stub'
+           THEN asset_salience.encoding_source
+           ELSE excluded.encoding_source
+       END,
+       outcome_salience   = excluded.outcome_salience,
+       retrieval_salience = excluded.retrieval_salience,
+       rank_score         = excluded.rank_score,
+       updated_at         = excluded.updated_at`).run(ref, vector.encoding, vector.outcome, vector.retrieval, vector.rankScore, ts, vector.encodingSource ?? "type-stub");
+}
+/**
+ * Load the salience row for one asset, or undefined if not yet computed.
+ */
+export function getAssetSalience(db, ref) {
+    const row = db
+        .prepare(`SELECT asset_ref, encoding_salience, outcome_salience, retrieval_salience,
+              rank_score, consecutive_no_ops, updated_at, encoding_source
+       FROM asset_salience WHERE asset_ref = ?`)
+        .get(ref);
+    // Bun SQLite returns null (not undefined) when no row found.
+    return row == null ? undefined : row;
+}
+/**
+ * Load ALL rank scores from the asset_salience table (full-stash query).
+ *
+ * Used by the forgetting-safety report (plan §WS-1 step 7) to compute stash-wide
+ * rank positions rather than pool-relative positions. Returns an empty Map when the
+ * table is empty (first WS-1 run = no pre-existing rows).
+ *
+ * Order is unspecified; callers must sort before assigning 1-indexed positions.
+ */
+export function getAllRankScores(db) {
+    const rows = db.prepare("SELECT asset_ref, rank_score FROM asset_salience").all();
+    const result = new Map();
+    for (const row of rows) {
+        result.set(row.asset_ref, row.rank_score);
+    }
+    return result;
+}
+// ── Plasticity helpers ────────────────────────────────────────────────────────
+/**
+ * Increment `consecutive_no_ops` for an asset. Called after a no-op reflect/distill.
+ * Has NO effect on `rank_score` — the plasticity counter only dampens consolidation
+ * selection, not retrieval ranking. See plan §WS-1 step 8.
+ *
+ * Invariant: recordNoOp must never originate rank_score semantics. If the asset has
+ * no salience row yet (persistence's best-effort try/catch may have swallowed an
+ * error), we do nothing — a no-op counter is meaningless without a rank_score row,
+ * and a synthetic INSERT would fabricate a rank_score=0 entry that could produce
+ * false catastrophic-forgetting signals in buildRankChangeReport.
+ */
+export function recordNoOp(db, ref) {
+    db.prepare(`UPDATE asset_salience SET consecutive_no_ops = consecutive_no_ops + 1, updated_at = ? WHERE asset_ref = ?`).run(Date.now(), ref);
+    // If changes === 0 the asset has no salience row yet — leave the table unchanged.
+}
+/**
+ * Reset `consecutive_no_ops` to 0 when an asset produces an accepted change.
+ * Call after a successful proposal acceptance or detected mutation.
+ */
+export function resetConsecutiveNoOps(db, ref) {
+    db.prepare(`UPDATE asset_salience SET consecutive_no_ops = 0, updated_at = ? WHERE asset_ref = ?`).run(Date.now(), ref);
+}
+/**
+ * Return the `consecutive_no_ops` count for one asset. 0 when unknown.
+ */
+export function getConsecutiveNoOps(db, ref) {
+    const row = db.prepare(`SELECT consecutive_no_ops FROM asset_salience WHERE asset_ref = ?`).get(ref);
+    return row?.consecutive_no_ops ?? 0;
+}
+// ── Consolidation-selection dampener constants ────────────────────────────────
+//
+// Assets with consecutive_no_ops >= THRESHOLD are deprioritised in the
+// SELECTION ORDER only. The persisted rank_score is intentionally left
+// unchanged so stable assets remain fully retrievable by other callers.
+//
+// Tuning guidance:
+//   THRESHOLD — how many consecutive no-op runs before dampening kicks in.
+//               3 means "skipped three times in a row", which signals the
+//               LLM consistently has nothing to say about this asset.
+//   FACTOR    — multiplicative penalty on the effective selection score.
+//               0.5 halves the apparent score so a dampened asset sorts
+//               after any peer with >= half its rankScore.
+export const SALIENCE_NO_OP_DAMPEN_THRESHOLD = 3;
+export const SALIENCE_NO_OP_DAMPEN_FACTOR = 0.5;
+/**
+ * Emit the forgetting-safety rank-change distribution report.
+ *
+ * Compares the provided `newRanks` (Map<ref, position (1-indexed)>) against
+ * the provided `oldRanks` and flags refs that were in the old top-200 but
+ * are now below position 500 as "forgetting candidates".
+ *
+ * Caller is responsible for computing old/new rank positions before and after
+ * the WS-1 formula cutover. Called once at cutover, not every run.
+ *
+ * @param oldRanks - Map<ref, 1-indexed rank position> under the OLD formula.
+ * @param newRanks - Map<ref, 1-indexed rank position> under the NEW formula.
+ * @param oldTopN  - Assets in old top-N to guard (default: 200).
+ * @param forgettingThreshold - New rank position below which a fall is flagged (default: 500).
+ */
+export function buildRankChangeReport(oldRanks, newRanks, oldTopN = 200, forgettingThreshold = 500) {
+    const allChanges = [];
+    const forgettingCandidates = [];
+    for (const [ref, oldRank] of oldRanks) {
+        const newRank = newRanks.get(ref);
+        if (newRank === undefined)
+            continue; // ref not in new ranking
+        const rankDelta = newRank - oldRank; // positive = fell in rank
+        allChanges.push({ ref, oldRank, newRank, rankDelta });
+        if (oldRank <= oldTopN && newRank > forgettingThreshold) {
+            forgettingCandidates.push({ ref, oldRank, newRank, rankDelta });
+        }
+    }
+    // Sort by magnitude of rank drop (most dramatic first).
+    forgettingCandidates.sort((a, b) => b.rankDelta - a.rankDelta);
+    return { forgettingCandidates, allChanges };
+}
+// ── Last-use timestamp lookup helper ─────────────────────────────────────────
+//
+// Wraps the index DB query to retrieve the last-retrieval timestamp per ref,
+// so callers do not need to import the raw db helpers directly. Returns a Map
+// keyed by the same ref strings passed in.
+//
+// Source: `utility_scores.last_used_at` (ISO-8601 string) joined to entries
+// via entry_id. WS-2 may later supersede this with `asset_outcome.last_retrieved_at`.
+/**
+ * Build a Map<ref, lastUseMs> from the index database's utility_scores table.
+ *
+ * Returns only refs that have a non-null `last_used_at`. Refs absent from the
+ * map should be treated as never retrieved (lastUseMs = 0).
+ *
+ * @param indexDb - An open read-capable index database connection.
+ * @param refs    - The set of asset refs to look up.
+ */
+export function getLastUseMsByRef(indexDb, refs) {
+    const result = new Map();
+    if (refs.length === 0)
+        return result;
+    const refSet = new Set(refs);
+    const allEntries = getAllEntries(indexDb);
+    const idToRef = new Map();
+    for (const indexed of allEntries) {
+        const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
+        if (refSet.has(ref))
+            idToRef.set(indexed.id, ref);
+    }
+    const ids = [...idToRef.keys()];
+    if (ids.length === 0)
+        return result;
+    const { global: scores } = getUtilityScoresByIds(indexDb, ids);
+    for (const [id, row] of scores) {
+        const ref = idToRef.get(id);
+        if (!ref)
+            continue;
+        const lastUsedAt = row.lastUsedAt;
+        if (!lastUsedAt)
+            continue;
+        const ms = typeof lastUsedAt === "number" ? lastUsedAt : Date.parse(lastUsedAt);
+        if (ms > 0)
+            result.set(ref, ms);
+    }
+    return result;
+}

package/dist/commands/improve/schema-similarity-gate.js

@@ -0,0 +1,168 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * WS-3b Step 0b — Schema-similarity intake gate.
+ *
+ * At intake, if a new candidate's body embedding is within ε of an existing
+ * derived-layer lesson/knowledge node, mark `schema-consistent` and lower its
+ * priority; only schema-inconsistent/contradicting candidates get full
+ * `encodingSalience`. One embedding lookup via body_embeddings cache; relieves
+ * dedup pressure before it accumulates.
+ *
+ * @module schema-similarity-gate
+ */
+import { warn } from "../../core/warn.js";
+import { closeDatabase, openExistingDatabase } from "../../indexer/db/db.js";
+/** Default epsilon for schema-similarity gate (looser than dedup's 0.97). */
+export const DEFAULT_SCHEMA_SIMILARITY_EPSILON = 0.85;
+/** Default multiplicative confidence penalty applied to schema-consistent candidates. */
+export const DEFAULT_SCHEMA_CONFIDENCE_PENALTY = 0.5;
+/**
+ * Check whether a candidate body embedding is schema-consistent with an existing
+ * derived-layer lesson/knowledge node. Returns `true` when the candidate is
+ * within ε of ANY existing derived node (i.e. it's likely covering ground the
+ * derived layer already knows about, so give it lower priority).
+ *
+ * One embedding lookup via the body_embeddings cache; no LLM call.
+ * Fails open: returns `false` (not schema-consistent) on any error so the
+ * candidate is not silently dropped.
+ *
+ * @param candidateEmbedding - Float32 embedding vector for the candidate body.
+ * @param existingDerivedEmbeddings - Pre-loaded embeddings for existing derived assets.
+ * @param config - Schema-similarity gate config.
+ */
+export function isSchemaConsistent(candidateEmbedding, existingDerivedEmbeddings, config) {
+    if (!config.enabled || existingDerivedEmbeddings.length === 0) {
+        return { consistent: false };
+    }
+    const epsilon = config.epsilon ?? DEFAULT_SCHEMA_SIMILARITY_EPSILON;
+    let bestSim = -Infinity;
+    let bestRef;
+    for (const { ref, embedding } of existingDerivedEmbeddings) {
+        // cosine similarity: dot(a,b) / (|a| * |b|)
+        let dot = 0;
+        let magA = 0;
+        let magB = 0;
+        for (let i = 0; i < candidateEmbedding.length; i++) {
+            const a = candidateEmbedding[i] ?? 0;
+            const b = embedding[i] ?? 0;
+            dot += a * b;
+            magA += a * a;
+            magB += b * b;
+        }
+        const sim = magA === 0 || magB === 0 ? 0 : dot / (Math.sqrt(magA) * Math.sqrt(magB));
+        if (sim > bestSim) {
+            bestSim = sim;
+            bestRef = ref;
+        }
+    }
+    if (bestSim >= epsilon) {
+        return { consistent: true, matchedRef: bestRef, similarity: bestSim };
+    }
+    return { consistent: false };
+}
+/**
+ * WS-3b Step-0b: apply the schema-similarity intake gate to one extract
+ * candidate. Pure/deterministic given `embedText`, so it is directly unit
+ * testable without the full extract→LLM harness.
+ *
+ * Returns the (possibly penalised) effective confidence plus a `penalised` flag
+ * and an optional human-readable `warning`. Parity guarantees:
+ *  - `ctx === null` (gate disabled / default-off)  → no change, never embeds.
+ *  - empty `derivedEmbeddings`                      → no change, never embeds.
+ *  - candidate type not lesson/knowledge            → no change, never embeds.
+ *  - embed throws                                   → fail open (no change), warns.
+ */
+export async function applySchemaSimilarityPenalty(candidate, ctx, embedText) {
+    const baseConfidence = typeof candidate.confidence === "number" ? candidate.confidence : undefined;
+    if (ctx === null || ctx.derivedEmbeddings.length === 0) {
+        return { effectiveConfidence: baseConfidence, penalised: false };
+    }
+    if (candidate.type !== "lesson" && candidate.type !== "knowledge") {
+        return { effectiveConfidence: baseConfidence, penalised: false };
+    }
+    try {
+        const candidateVec = await embedText(candidate.body);
+        const check = isSchemaConsistent(candidateVec, ctx.derivedEmbeddings, ctx.config);
+        if (check.consistent) {
+            const penalty = ctx.config.confidencePenalty ?? DEFAULT_SCHEMA_CONFIDENCE_PENALTY;
+            return {
+                effectiveConfidence: (baseConfidence ?? 1.0) * penalty,
+                penalised: true,
+                warning: `[extract] schema-consistent candidate ${candidate.type}:${candidate.name} ` +
+                    `(sim=${check.similarity?.toFixed(3)} vs ${check.matchedRef}) — confidence penalised ×${penalty}`,
+            };
+        }
+        return { effectiveConfidence: baseConfidence, penalised: false };
+    }
+    catch (embedErr) {
+        // Fail open: embed errors must never abort extraction.
+        return {
+            effectiveConfidence: baseConfidence,
+            penalised: false,
+            warning: `[extract] schema-similarity embed failed for ${candidate.type}:${candidate.name} — skipping gate: ` +
+                (embedErr instanceof Error ? embedErr.message : String(embedErr)),
+        };
+    }
+}
+/**
+ * Load persisted body embeddings for all indexed **derived-layer**
+ * (lesson + knowledge) entries from index.db. Returns an empty array when
+ * the DB is unavailable, empty, or the embeddings table has no entries for
+ * those types — the caller treats an empty array as "gate inactive".
+ *
+ * FAIL-OPEN: any error emits a debug warning and returns an empty array.
+ * This ensures the extract pass never fails because of a missing index.
+ *
+ * The returned entries are keyed by `entry_key` (e.g. "lesson:foo",
+ * "knowledge:bar"). Only entries whose embedding dimension matches the first
+ * observed dimension are included (mixed-dim BLOBs are silently skipped).
+ *
+ * @param dbPath - Optional path override for index.db (for testing).
+ */
+export function loadDerivedLayerEmbeddings(dbPath) {
+    let db;
+    try {
+        db = openExistingDatabase(dbPath);
+        const rows = db
+            .prepare(`SELECT e.entry_key, emb.embedding
+         FROM entries e
+         JOIN embeddings emb ON emb.id = e.id
+         WHERE e.entry_type IN ('lesson', 'knowledge')`)
+            .all();
+        if (rows.length === 0)
+            return [];
+        let expectedDim;
+        const result = [];
+        for (const row of rows) {
+            const buf = row.embedding;
+            if (!buf || buf.byteLength === 0 || buf.byteLength % 4 !== 0)
+                continue;
+            const dim = buf.byteLength / 4;
+            if (expectedDim === undefined)
+                expectedDim = dim;
+            if (dim !== expectedDim)
+                continue;
+            const aligned = new ArrayBuffer(buf.byteLength);
+            new Uint8Array(aligned).set(buf);
+            const f32 = new Float32Array(aligned);
+            result.push({ ref: row.entry_key, embedding: Array.from(f32) });
+        }
+        return result;
+    }
+    catch (err) {
+        warn("[schema-similarity-gate] loadDerivedLayerEmbeddings: failed to load from index.db — gate inactive:", err instanceof Error ? err.message : String(err));
+        return [];
+    }
+    finally {
+        if (db) {
+            try {
+                closeDatabase(db);
+            }
+            catch {
+                // ignore close errors
+            }
+        }
+    }
+}

package/dist/commands/improve/shared.js

@@ -0,0 +1,51 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { getDefaultLlmConfig, getImproveProcessConfig } from "../../core/config/config.js";
+import { warn } from "../../core/warn.js";
+import { resolveImproveProcessRunnerFromProfile, runnerIsLlm } from "../../integrations/agent/runner.js";
+import { chatCompletion } from "../../llm/client.js";
+/** Normalize an unknown thrown value to a human-readable message string. */
+export function errMessage(e) {
+    return e instanceof Error ? e.message : String(e);
+}
+/**
+ * Slugify an asset ref for use in eval-case / rejection filenames: lowercase,
+ * non-alphanumerics collapsed to `-`, capped at 60 characters.
+ */
+export function refSlug(ref) {
+    return ref
+        .replace(/[^a-z0-9]/gi, "-")
+        .toLowerCase()
+        .slice(0, 60);
+}
+/**
+ * Resolve the production LLM seam for an improve process (`recombine` /
+ * `procedural`). Returns a function that issues one bounded chatCompletion per
+ * call, or `undefined` when no LLM is configured (the pass then makes no
+ * calls). Previously copied verbatim in recombine.ts and procedural.ts.
+ *
+ * When `opts.activeProfile` is supplied, its per-process runner override wins
+ * over the `default` profile so `akm improve --profile <name>` selects the
+ * profile's model; absent falls back to `default`.
+ */
+export function resolveImproveLlmFn(config, opts) {
+    const processConfig = getImproveProcessConfig(config, opts.processKey, opts.activeProfile);
+    const runnerSpec = resolveImproveProcessRunnerFromProfile(processConfig, config);
+    const llmConfig = runnerSpec && runnerIsLlm(runnerSpec) ? runnerSpec.connection : getDefaultLlmConfig(config);
+    if (!llmConfig)
+        return undefined;
+    return async (prompt) => {
+        const messages = [
+            { role: "system", content: opts.systemPrompt },
+            { role: "user", content: prompt },
+        ];
+        try {
+            return await chatCompletion(llmConfig, messages, { signal: opts.signal, enableThinking: false });
+        }
+        catch (e) {
+            warn(`${opts.tag} LLM call failed: ${String(e)}`);
+            return null;
+        }
+    };
+}