akm-cli 0.9.0-rc.9 → 0.9.0

package/dist/storage/repositories/index-entry-mapper.js

@@ -10,22 +10,20 @@
  * than crashing the caller.
  *
  * Relocated from `src/indexer/db/entry-mapper.ts` (WI-5a): it now imports the
- * shared shapes from the leaf types module rather than from `db.ts`, so it no
- * longer participates in the indexer-db import cycle.
+ * shared shapes from the leaf types module `./index-entry-types.ts` rather
+ * than reaching across the storage↔indexer boundary into `db.ts`, so it no
+ * longer participates in the `db.ts` / `entry-mapper.ts` / `schema.ts` import
+ * cycle that used to pin them together.
  */
 import { warn } from "../../core/warn.js";
 /**
  * Canonical column list for reading a full indexed entry from the `entries`
  * table, in the order {@link rowToIndexedEntry} expects.
  *
- * Chunk-5 flip (Checkpoint A): the durable identity columns `concept_id` and
- * `bundle_id` are surfaced here so the state.db salience/outcome dual-read arms
- * can build the fully-qualified `<bundle>//<concept-id>` new-grammar key from a
- * mapped entry with ZERO extra queries (they are the last two columns and both
- * nullable — a NULL-provenance write-back straggler simply yields `undefined`,
- * and the reader falls through to its inline legacy `type:name` arm).
+ * Durable identity columns are surfaced here so every mapped entry carries
+ * current indexed provenance.
  */
-export const ENTRY_COLUMNS = "id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text, concept_id, bundle_id";
+export const ENTRY_COLUMNS = "id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text, item_ref, concept_id, bundle_id, adapter_id";
 /**
  * Map one raw `entries` row to a {@link DbIndexedEntry}, parsing `entry_json`.
  * Returns `null` (and warns, tagged with `context`) when the JSON is corrupt so
@@ -40,6 +38,10 @@ export function rowToIndexedEntry(row, context) {
         warn(`[db] ${context}: skipping entry id=${row.id} — corrupt entry_json`);
         return null;
     }
+    if (!row.item_ref || !row.concept_id || !row.bundle_id || !row.adapter_id) {
+        warn(`[db] ${context}: skipping entry id=${row.id} — missing indexed provenance`);
+        return null;
+    }
     return {
         id: row.id,
         entryKey: row.entry_key,
@@ -48,7 +50,9 @@ export function rowToIndexedEntry(row, context) {
         stashDir: row.stash_dir,
         entry,
         searchText: row.search_text,
-        conceptId: row.concept_id ?? undefined,
-        bundleId: row.bundle_id ?? undefined,
+        itemRef: row.item_ref,
+        conceptId: row.concept_id,
+        bundleId: row.bundle_id,
+        adapterId: row.adapter_id,
     };
 }

package/dist/storage/repositories/index-fts-repository.js

@@ -5,7 +5,7 @@
  * `index.db` FTS5 search + rebuild repository.
  *
  * Owns the `entries_fts` full-text query path and the incremental/full FTS
- * rebuild. Extracted verbatim from `src/indexer/db/db.ts` (WI-5a).
+ * rebuild.
  */
 import { warn } from "../../core/warn.js";
 import { buildPrefixQuery, sanitizeFtsQuery } from "../../indexer/search/fts-query.js";
@@ -52,7 +52,7 @@ function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
     }
     const sql = `
     SELECT e.id, e.file_path AS filePath, e.entry_json, e.search_text AS searchText,
-           e.item_ref AS itemRef,
+           e.item_ref AS itemRef, e.bundle_id AS bundleId, e.concept_id AS conceptId, e.adapter_id AS adapterId,
            bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
     FROM entries_fts f
     JOIN entries e ON e.id = f.entry_id
@@ -81,6 +81,9 @@ function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
                 searchText: row.searchText,
                 bm25Score: row.bm25Score,
                 itemRef: row.itemRef,
+                bundleId: row.bundleId,
+                conceptId: row.conceptId,
+                adapterId: row.adapterId,
             });
         }
         return results;

package/dist/storage/repositories/index-llm-cache-repository.js

@@ -6,7 +6,6 @@
  *
  * Owns the raw SQL for `llm_enrichment_cache` — the body-hash-keyed cache that
  * lets `akm index --enrich` skip the LLM call when a file's body is unchanged.
- * Extracted verbatim from `src/indexer/db/db.ts` (WI-5a).
  */
 import { bestEffort } from "../../core/best-effort.js";
 import { sha256Hex } from "../../runtime.js";

package/dist/storage/repositories/index-meta-repository.js

@@ -5,9 +5,8 @@
  * `index.db` meta + per-directory index-state repository.
  *
  * Owns the raw SQL for `index_meta` (the key/value stamp table) and
- * `index_dir_state` (incremental-index bookkeeping). Extracted verbatim from
- * `src/indexer/db/db.ts` (WI-5a) so the storage layer, not the indexer god-file,
- * owns these primitives.
+ * `index_dir_state` (incremental-index bookkeeping). Lives in the storage
+ * layer, not the indexer, so the storage layer owns these primitives.
  */
 import path from "node:path";
 // ── Meta helpers ────────────────────────────────────────────────────────────

package/dist/storage/repositories/index-schema.js

@@ -2,14 +2,13 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
- * index.db schema, version stamps, and targeted migrations — relocated from
- * `src/indexer/db/schema.ts` (WI-5a) into the storage layer. This isolates the
- * one genuinely risky area (schema evolution) from the CRUD/FTS/vector queries.
+ * index.db schema, version stamps, and targeted migrations, kept in the
+ * storage layer. This isolates the one genuinely risky area (schema
+ * evolution) from the CRUD/FTS/vector queries.
  *
  * The meta accessors, embedding purge, and vec-availability probe that
- * `ensureSchema` leans on now live in the sibling `index-meta-repository` /
- * `index-vec-repository` modules; importing them from there (rather than from
- * the old `db.ts` hub) is what lets this module leave the import cycle.
+ * `ensureSchema` leans on live in the sibling `index-meta-repository` /
+ * `index-vec-repository` modules.
  */
 import { bestEffort } from "../../core/best-effort.js";
 import { warn } from "../../core/warn.js";
@@ -529,27 +528,13 @@ function ensureBundleRefColumns(db) {
  * durable identity — spec §11.4). A pre-v19 DB has a NON-unique
  * `idx_entries_item_ref`, so we DROP-then-CREATE (a `CREATE UNIQUE INDEX IF NOT
  * EXISTS` under the same name would no-op against the existing non-unique index).
- * SQLite treats NULLs as distinct in a UNIQUE index, so NULL-item_ref write-back
- * stragglers do not collide.
- *
- * A partially-migrated index could still hold two rows with the same non-NULL
- * item_ref; the UNIQUE build then throws. The index is a regenerable derived
- * cache, so we fall back to the plain lookup index (reads stay fast) and warn —
- * a full `akm index` rebuild repopulates one row per item_ref and the next open
- * upgrades cleanly.
+ * SQLite treats NULLs as distinct in a UNIQUE index. Duplicate durable identities
+ * are an invalid index and fail the schema open rather than enabling a dual-key
+ * fallback.
  */
 function ensureUniqueItemRefIndex(db) {
-    let uniqueOk = false;
-    bestEffort(() => {
-        db.exec("DROP INDEX IF EXISTS idx_entries_item_ref");
-        db.exec("CREATE UNIQUE INDEX idx_entries_item_ref ON entries(item_ref)");
-        uniqueOk = true;
-    }, "entries.item_ref UNIQUE index — column added by ensureBundleRefColumns above");
-    if (!uniqueOk) {
-        warn("[akm] entries.item_ref UNIQUE index could not be built (duplicate item_ref on a partially-migrated index) — " +
-            "keeping the plain lookup index; a full `akm index` rebuild restores uniqueness.");
-        bestEffort(() => db.exec("CREATE INDEX IF NOT EXISTS idx_entries_item_ref ON entries(item_ref)"), "entries.item_ref lookup index fallback");
-    }
+    db.exec("DROP INDEX IF EXISTS idx_entries_item_ref");
+    db.exec("CREATE UNIQUE INDEX idx_entries_item_ref ON entries(item_ref)");
 }
 /**
  * Returns true when a table exists in the current database.

package/dist/storage/repositories/index-utility-repository.js

@@ -7,7 +7,6 @@
  * Owns the raw SQL for `utility_scores` / `utility_scores_scoped` and the
  * retrieval-frequency counting over `usage_events`. The bounded-step EMA policy
  * itself lives in `indexer/feedback/utility-policy`; this repo only reads/writes.
- * Extracted verbatim from `src/indexer/db/db.ts` (WI-5a).
  */
 import path from "node:path";
 import { conceptIdFromTypeName, parseRefInput } from "../../core/asset/resolve-ref.js";
@@ -106,11 +105,7 @@ export function upsertUtilityScore(db, entryId, data) {
 }
 /**
  * Reduce a stored ref to its bare form, dropping any `origin//` / `bundle//`
- * prefix — the SAME strip the counting SQL does (`substr` after the first
- * `//`). The bare form is `type:name` for a legacy row and `conceptId` for a
- * 0.9.0 re-keyed row (Chunk-5 flip F4c). Matching the SQL strip exactly keeps
- * the JS row-grouping and the SQL `IN (…)` filter in lockstep across both
- * spellings.
+ * prefix, matching the counting SQL's `substr` after the first `//`.
  */
 function bareRef(ref) {
     const boundary = ref.indexOf("//");
@@ -124,13 +119,8 @@ function bareRef(ref) {
  */
 function bareRefCandidates(ref) {
     const bare = bareRef(ref.trim());
-    try {
-        const parsed = parseRefInput(bare);
-        return [conceptIdFromTypeName(parsed.type, parsed.name)];
-    }
-    catch {
-        return [bare];
-    }
+    const parsed = parseRefInput(bare);
+    return [conceptIdFromTypeName(parsed.type, parsed.name)];
 }
 /**
  * Count retrieval events for the given entry refs.
@@ -141,16 +131,13 @@ function bareRefCandidates(ref) {
  * proactive-maintenance selector to rank zero-feedback assets by retrieval
  * frequency.
  *
- * Unscoped callers retain normalization-aware legacy matching. Source-scoped
- * callers instead require either an event linked to an entry in the selected
- * stash root or a detached ref qualified with the selected source name. Bare
- * detached events are accepted only when `includeLegacyBare` is explicitly set
- * for the historical local stash.
+ * Source-scoped callers require either an event linked to an entry in the
+ * selected stash root or a detached ref qualified with the selected source
+ * name.
  *
  * `curate` events are included: their per-item rows are written with
  * entry_ref populated (see logCurateEvent), so curation is a real retrieval
- * signal here. Legacy summary-only curate rows with a NULL entry_ref simply
- * contribute nothing.
+ * signal here. Summary-only rows with a NULL entry_ref contribute nothing.
  *
  * Machine/eval/unattributed/extension events are excluded: this count feeds
  * salience and ranking, so only explicitly attributed `user` traffic counts.
@@ -161,9 +148,8 @@ export function getRetrievalCounts(indexDb, stateDb, refs, options = {}) {
     if (options.sourceName || options.stashDir) {
         return getSourceScopedRetrievalCounts(indexDb, stateDb, refs, options);
     }
-    // Map each candidate bare form (both spellings) back to the input ref(s) that
-    // produced it so we can re-key DB results (grouped by bare form) onto the
-    // caller's ref strings — spelling-agnostic across the §11.4 re-key.
+    // Map each conceptId back to the input refs that produced it so DB results
+    // can be re-keyed onto the caller's strings.
     const bareToInputs = new Map();
     for (const ref of refs) {
         for (const bare of bareRefCandidates(ref)) {
@@ -185,7 +171,7 @@ export function getRetrievalCounts(indexDb, stateDb, refs, options = {}) {
         // everything up to and including the last `//` separator. SQLite has no
         // rfind, but stored origins never themselves contain `//`, so a stash ref
         // has exactly one `//` and `substr(... instr ...)` is exact; bare refs have
-        // no `//` and pass through unchanged.
+        // no `//` and are excluded above.
         // usage_events lives in state.db (Chunk-8 WI-8.3) — this global count needs
         // no entries join, so it reads state.db directly.
         const rows = stateDb
@@ -199,7 +185,8 @@ export function getRetrievalCounts(indexDb, stateDb, refs, options = {}) {
          FROM usage_events
          WHERE event_type IN ('search','show','curate')
            AND entry_ref IS NOT NULL
-            AND source = 'user'
+           AND instr(entry_ref, '//') > 0
+           AND source = 'user'
            AND CASE
                  WHEN instr(entry_ref, '//') > 0
                    THEN substr(entry_ref, instr(entry_ref, '//') + 2)
@@ -253,7 +240,8 @@ function getSourceScopedRetrievalCounts(indexDb, stateDb, refs, options) {
            FROM usage_events ue
           WHERE ue.event_type IN ('search','show','curate')
             AND ue.entry_ref IS NOT NULL
-              AND ue.source = 'user'
+            AND instr(ue.entry_ref, '//') > 0
+            AND ue.source = 'user'
             AND CASE
                   WHEN instr(ue.entry_ref, '//') > 0
                     THEN substr(ue.entry_ref, instr(ue.entry_ref, '//') + 2)
@@ -268,8 +256,7 @@ function getSourceScopedRetrievalCounts(indexDb, stateDb, refs, options) {
                 : false;
             const detached = row.entry_id === null || selectedRoot === undefined;
             const qualifiedForSource = detached && options.sourceName !== undefined && row.entry_ref === `${options.sourceName}//${bare}`;
-            const acceptedLegacyBare = detached && options.includeLegacyBare === true && row.entry_ref === bare;
-            if (!linkedToSelectedRoot && !qualifiedForSource && !acceptedLegacyBare)
+            if (!linkedToSelectedRoot && !qualifiedForSource)
                 continue;
             countsByBare.set(bare, (countsByBare.get(bare) ?? 0) + 1);
         }

package/dist/storage/repositories/index-vec-repository.js

@@ -6,7 +6,6 @@
  *
  * Owns the sqlite-vec extension load/availability probe, the BLOB `embeddings`
  * table, the `entries_vec` virtual table, and the JS-cosine fallback path.
- * Extracted verbatim from `src/indexer/db/db.ts` (WI-5a).
  */
 import { createRequire } from "node:module";
 import { bestEffort } from "../../core/best-effort.js";
@@ -59,6 +58,12 @@ export function setVecFastPathReady(db, ready) {
 export function isVecFastPathReady(db) {
     return getMeta(db, VEC_FAST_PATH_READY_META) !== "0";
 }
+/** Remove both vector representations for an entry whose embedding input changed. */
+export function deleteEntryVectors(db, id) {
+    db.prepare("DELETE FROM embeddings WHERE id = ?").run(id);
+    if (isVecAvailable(db))
+        db.prepare("DELETE FROM entries_vec WHERE id = ?").run(id);
+}
 const VEC_DOCS_URL = "https://github.com/itlackey/akm/blob/main/docs/reference/configuration.md#sqlite-vec-extension";
 const VEC_FALLBACK_THRESHOLD = 10_000;
 // Per-database warning state: tracks which databases have already emitted the

package/dist/storage/repositories/outcome-repository.js

@@ -0,0 +1,119 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Upsert one `asset_outcome` row. Extracted verbatim from the tail of
+ * `updateAssetOutcome` in outcome-loop.ts — same SQL text, same bind order;
+ * the business logic that computes these values (warm-start seed vs.
+ * differential update) stays in outcome-loop.ts and calls this function.
+ *
+ * `review_pressure` is intentionally omitted from both the INSERT column list
+ * and the ON CONFLICT SET clause: the column's DEFAULT 0 seeds fresh rows, and
+ * omitting it from SET leaves an existing row's value untouched on update
+ * (never written going forward). The column itself is dropped in migration 018.
+ */
+export function upsertAssetOutcome(db, values) {
+    db.prepare(`INSERT INTO asset_outcome
+       (asset_ref, last_retrieved_at, retrieval_count, expected_retrieval_rate,
+        negative_feedback_count, accepted_change_count,
+        outcome_score, updated_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT(asset_ref) DO UPDATE SET
+       last_retrieved_at      = excluded.last_retrieved_at,
+       retrieval_count        = excluded.retrieval_count,
+       expected_retrieval_rate= excluded.expected_retrieval_rate,
+       negative_feedback_count= excluded.negative_feedback_count,
+       accepted_change_count  = excluded.accepted_change_count,
+       outcome_score          = excluded.outcome_score,
+       updated_at             = excluded.updated_at`).run(values.ref, values.lastRetrievedAt, values.retrievalCount, values.expectedRetrievalRate, values.negativeFeedbackCount, values.acceptedChangeCount, values.outcomeScore, values.updatedAt);
+}
+/**
+ * Load the outcome row for one asset, or `undefined` if not yet written.
+ */
+export function getAssetOutcome(db, ref) {
+    const row = db
+        .prepare(`SELECT asset_ref, last_retrieved_at, retrieval_count, expected_retrieval_rate,
+              negative_feedback_count, accepted_change_count,
+              outcome_score, updated_at
+       FROM asset_outcome WHERE asset_ref = ?`)
+        .get(ref);
+    return row == null ? undefined : row;
+}
+/**
+ * Load ALL asset_outcome rows. Used for the proxy-adequacy tripwire computation.
+ */
+export function getAllAssetOutcomes(db) {
+    return db
+        .prepare(`SELECT asset_ref, last_retrieved_at, retrieval_count, expected_retrieval_rate,
+              negative_feedback_count, accepted_change_count,
+              outcome_score, updated_at
+       FROM asset_outcome ORDER BY asset_ref`)
+        .all();
+}
+/**
+ * Build a Map<ref, outcome_score> for a set of refs in one query.
+ * Used by `salience.ts` to populate `outcomeSalience`.
+ */
+export function getOutcomeScoresByRef(db, refs) {
+    const result = new Map();
+    if (refs.length === 0)
+        return result;
+    const CHUNK = 500;
+    for (let i = 0; i < refs.length; i += CHUNK) {
+        const chunk = refs.slice(i, i + CHUNK);
+        const placeholders = chunk.map(() => "?").join(",");
+        const rows = db
+            .prepare(`SELECT asset_ref, outcome_score FROM asset_outcome WHERE asset_ref IN (${placeholders})`)
+            .all(...chunk);
+        for (const row of rows) {
+            result.set(row.asset_ref, row.outcome_score);
+        }
+    }
+    return result;
+}
+/** List every `asset_outcome` ref with its current `missing_since` marker. */
+export function listAssetOutcomeMissingState(db) {
+    return db.prepare(`SELECT asset_ref, missing_since FROM asset_outcome`).all();
+}
+/**
+ * Stamp `missing_since = now` for the given refs, but ONLY where it is
+ * currently NULL — a ref already stamped keeps its original timestamp so the
+ * grace window measures from first sighting, not last. Refs with no row at
+ * all are silently skipped (0 rows affected). Returns the number of rows
+ * actually stamped.
+ */
+export function stampAssetOutcomeMissing(db, refs, now) {
+    if (refs.length === 0)
+        return 0;
+    const stmt = db.prepare(`UPDATE asset_outcome SET missing_since = ? WHERE asset_ref = ? AND missing_since IS NULL`);
+    let changed = 0;
+    for (const ref of refs)
+        changed += Number(stmt.run(now, ref).changes);
+    return changed;
+}
+/**
+ * Clear `missing_since` for refs that resolved again. Returns the number of
+ * rows actually cleared.
+ */
+export function clearAssetOutcomeMissing(db, refs) {
+    if (refs.length === 0)
+        return 0;
+    const stmt = db.prepare(`UPDATE asset_outcome SET missing_since = NULL WHERE asset_ref = ? AND missing_since IS NOT NULL`);
+    let changed = 0;
+    for (const ref of refs)
+        changed += Number(stmt.run(ref).changes);
+    return changed;
+}
+/**
+ * Delete `asset_outcome` rows whose `missing_since` is older than
+ * `cutoffMs`. A single DELETE — SQLite runs it as one implicit transaction.
+ * Returns the number of rows deleted.
+ */
+export function deleteAssetOutcomeMissingBefore(db, cutoffMs) {
+    return Number(db.prepare(`DELETE FROM asset_outcome WHERE missing_since IS NOT NULL AND missing_since < ?`).run(cutoffMs).changes);
+}
+/** Count `asset_outcome` rows currently marked missing (the live backlog size). */
+export function countAssetOutcomeMissing(db) {
+    const row = db.prepare(`SELECT COUNT(*) AS n FROM asset_outcome WHERE missing_since IS NOT NULL`).get();
+    return row.n;
+}