akm-cli 0.9.0-rc.0 → 0.9.0-rc.13

package/dist/{indexer/db/schema.js → storage/repositories/index-schema.js}

@@ -2,18 +2,18 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
- * index.db schema, version stamps, and targeted migrations — extracted from
- * indexer/db/db.ts. This isolates the one genuinely risky area (schema
- * evolution) from the CRUD/FTS/vector queries that stay in db.ts.
+ * index.db schema, version stamps, and targeted migrations, kept in the
+ * storage layer. This isolates the one genuinely risky area (schema
+ * evolution) from the CRUD/FTS/vector queries.
  *
  * The meta accessors, embedding purge, and vec-availability probe that
- * `ensureSchema` leans on remain in db.ts (they are part of the database
- * lifecycle) and are imported back here.
+ * `ensureSchema` leans on live in the sibling `index-meta-repository` /
+ * `index-vec-repository` modules.
  */
 import { bestEffort } from "../../core/best-effort.js";
 import { warn } from "../../core/warn.js";
-import { ensureUsageEventsSchema } from "../usage/usage-events.js";
-import { getMeta, isVecAvailable, purgeEmbeddings, setMeta } from "./db.js";
+import { getMeta, setMeta } from "./index-meta-repository.js";
+import { isVecAvailable, purgeEmbeddings } from "./index-vec-repository.js";
 // ── Constants ───────────────────────────────────────────────────────────────
 // NOTE: schema changes are additive. DB_VERSION is a forensic stamp only — it
 // no longer gates any destructive path (the old nuclear drop-and-rebuild was
@@ -21,7 +21,54 @@ import { getMeta, isVecAvailable, purgeEmbeddings, setMeta } from "./db.js";
 // older/partial DB forward without dropping data). Graph re-keying uses a
 // TARGETED, graph-only migration (migrateGraphFilesSchema) — the model for any
 // incompatible change: migrate in place, never wipe the whole index.
-export const DB_VERSION = 17;
+//
+// v17→v18 (Chunk-5 Step 2, spec §14.4): the `entries` table gains the durable
+// bundle-adapter identity/provenance columns — `item_ref` (`<bundle>//<concept
+// -id>` canonical stored spelling), `bundle_id`/`component_id`/`concept_id`/
+// `adapter_id` provenance, `type` (open token), and `content_hash`/`document
+// _json`. They land ADDITIVELY ALONGSIDE the legacy `entry_key`/`dir_path`/
+// `stash_dir`/`entry_json`/`entry_type` columns (dev-time transitional shape):
+// the writer populates the identity/provenance columns while every reader still
+// keys on the legacy columns, so the battery stays green while the reader
+// repoint + ref-grammar flip land incrementally. The legacy columns + this
+// coexistence are removed once every reader is repointed onto `item_ref`
+// (spec §3.3 — single clean shape, no dual read-path). The index is a
+// regenerable derived cache, so an `akm index` rebuild repopulates the new
+// columns on any DB opened at an older version.
+//
+// v18→v19 (Chunk-5 flip F4c, spec §11.4): `entries.item_ref` becomes THE durable
+// identity — its lookup index is upgraded to UNIQUE (every indexed row now
+// carries item_ref; NULLs are distinct in a SQLite UNIQUE index, so write-back
+// stragglers coexist). The durable state keyed off refs (`usage_events.entry_ref`)
+// is re-keyed onto item_ref by the one-time §11.4 migration cutover
+// (020-three-db-cutover) with orphan quarantine in `legacy_state`. The index is
+// regenerable, so a rebuild is an acceptable fallback if the UNIQUE upgrade finds
+// a duplicate on a partially-migrated DB.
+//
+// v19→v20 (Chunk-8 WI-8.3, three-DB merge): index.db STOPS owning `usage_events`
+// + `legacy_state`. Both are durable, non-regenerable state and now live in
+// state.db (folded by state migration 020; the cutover rescues the old index.db
+// rows across). index.db is a regenerable cache, so dropping their DDL from the
+// schema path needs no migration — a rebuild simply no longer re-creates them.
+// STANDARD REBUILD NOTE: after this bump, any index.db opened at v19 is rebuilt
+// from the stash on the next `akm index`; usage_events/legacy_state are read
+// exclusively from state.db thereafter.
+//
+// Reader/writer repoint progress (spec §3.3 "single clean shape"): the `entries`
+// upsert (index-entries-repository `getUpsertStmts`) uses the UNIQUE `item_ref`
+// as its PRIMARY conflict target, and the graph-boost related-ref reader
+// (`listRelatedPathsForFile`) resolves the user-facing ref from
+// `concept_id`/`item_ref` instead of stripping `entry_key`. The legacy
+// `entry_key`/`dir_path`/`stash_dir`/`entry_type`/`entry_json` columns are NOT
+// yet removable: they retain live consumers outside this module — `entry_key`
+// (mv-cli re-key, usage-event legacy resolution, index-entry-mapper, the LLM
+// cache), `entry_json` (the row payload every reader decodes), `entry_type`
+// (FTS + workflow loader), `stash_dir`/`dir_path` (scan/delete/utility scoping).
+// The upsert therefore keeps `entry_key` as a NULL-item_ref-safe SECOND conflict
+// target (the LLM metadata-enhance re-upsert still writes existing rows with a
+// NULL item_ref) and degrades to it entirely when the item_ref index is the
+// non-unique fallback. Both are deletable once every remaining reader repoints.
+export const DB_VERSION = 20;
 export const EMBEDDING_DIM = 384;
 // #624-P1: graph_files re-keyed to (stash_root, file_path, body_hash). Bumped 3→4
 // as a marker; the actual migration is the targeted drop in migrateGraphFilesSchema.
@@ -62,6 +109,102 @@ const REGISTRY_INDEX_CACHE_DDL = `
   CREATE INDEX IF NOT EXISTS idx_registry_cache_fetched
     ON registry_index_cache(fetched_at);
 `;
+/**
+ * Create the graph-extraction tables (`graph_meta`/`graph_files`/`graph_file_entities`/
+ * `graph_file_relations`/`graph_extraction_queue`). Extracted verbatim from
+ * {@link ensureSchema} (called at the same point, between `migrateGraphFilesSchema`
+ * and `migrateGraphDataFromLegacy`) — a pure, behavior-identical decomposition.
+ */
+function ensureGraphTables(db) {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS graph_meta (
+      stash_root          TEXT PRIMARY KEY,
+      schema_version      INTEGER NOT NULL,
+      generated_at        TEXT NOT NULL,
+      considered_files    INTEGER NOT NULL DEFAULT 0,
+      extracted_files     INTEGER NOT NULL DEFAULT 0,
+      entity_count        INTEGER NOT NULL DEFAULT 0,
+      relation_count      INTEGER NOT NULL DEFAULT 0,
+      extraction_coverage REAL NOT NULL DEFAULT 0,
+      density             REAL NOT NULL DEFAULT 0,
+      extractor_id        TEXT,
+      extraction_run_id   TEXT,
+      model               TEXT,
+      prompt_version      TEXT,
+      batch_size          INTEGER,
+      cache_hits          INTEGER NOT NULL DEFAULT 0,
+      cache_misses        INTEGER NOT NULL DEFAULT 0,
+      truncation_count    INTEGER NOT NULL DEFAULT 0,
+      failure_count       INTEGER NOT NULL DEFAULT 0
+    );
+
+    CREATE TABLE IF NOT EXISTS graph_files (
+      stash_root        TEXT NOT NULL,
+      file_path         TEXT NOT NULL,
+      file_order        INTEGER NOT NULL,
+      file_type         TEXT NOT NULL,
+      body_hash         TEXT NOT NULL,
+      confidence        REAL,
+      status            TEXT NOT NULL DEFAULT 'extracted',
+      reason            TEXT,
+      extraction_run_id TEXT,
+      PRIMARY KEY (stash_root, file_path, body_hash)
+    );
+
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_graph_files_path
+      ON graph_files(stash_root, file_path);
+
+    CREATE INDEX IF NOT EXISTS idx_graph_files_stash_order
+      ON graph_files(stash_root, file_order);
+
+    CREATE TABLE IF NOT EXISTS graph_file_entities (
+      stash_root   TEXT NOT NULL,
+      file_path    TEXT NOT NULL,
+      body_hash    TEXT NOT NULL,
+      entity_order INTEGER NOT NULL,
+      entity_norm  TEXT NOT NULL,
+      entity       TEXT NOT NULL,
+      PRIMARY KEY (stash_root, file_path, body_hash, entity_order),
+      FOREIGN KEY (stash_root, file_path, body_hash)
+        REFERENCES graph_files(stash_root, file_path, body_hash) ON DELETE CASCADE
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_graph_file_entities_entity_norm
+      ON graph_file_entities(stash_root, entity_norm);
+
+    CREATE TABLE IF NOT EXISTS graph_file_relations (
+      stash_root     TEXT NOT NULL,
+      file_path      TEXT NOT NULL,
+      body_hash      TEXT NOT NULL,
+      relation_order INTEGER NOT NULL,
+      from_entity_norm TEXT NOT NULL,
+      from_entity    TEXT NOT NULL,
+      to_entity_norm TEXT NOT NULL,
+      to_entity      TEXT NOT NULL,
+      relation_type  TEXT,
+      confidence     REAL,
+      PRIMARY KEY (stash_root, file_path, body_hash, relation_order),
+      FOREIGN KEY (stash_root, file_path, body_hash)
+        REFERENCES graph_files(stash_root, file_path, body_hash) ON DELETE CASCADE
+    );
+
+    -- #624-P3: lazy graph-extraction queue. Standalone table (NO FK to
+    -- graph_files — a queued file by definition has no graph row yet).
+    -- Idempotent on (stash_root, file_path); drained highest-priority-first.
+    -- CREATE TABLE IF NOT EXISTS is the forward migration (no DB_VERSION bump).
+    CREATE TABLE IF NOT EXISTS graph_extraction_queue (
+      stash_root TEXT NOT NULL,
+      file_path  TEXT NOT NULL,
+      body_hash  TEXT NOT NULL,
+      queued_at  TEXT NOT NULL DEFAULT (datetime('now')),
+      priority   INTEGER NOT NULL DEFAULT 0,
+      PRIMARY KEY (stash_root, file_path)
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_graph_extraction_queue_drain
+      ON graph_extraction_queue(stash_root, priority DESC, queued_at);
+  `);
+}
 export function ensureSchema(db, embeddingDim) {
     // Create meta table first so we can check version
     db.exec(`
@@ -90,13 +233,35 @@ export function ensureSchema(db, embeddingDim) {
       entry_json  TEXT NOT NULL,
       search_text TEXT NOT NULL,
       entry_type  TEXT NOT NULL,
-      derived_from TEXT
+      derived_from TEXT,
+      -- Chunk-5 Step 2 / DB v18 (spec 14.4): bundle-adapter identity + provenance,
+      -- ADDITIVE alongside the legacy columns above. item_ref is the durable
+      -- <bundle>//<concept-id> spelling; nullable during the transition so a
+      -- pre-repoint reader path never trips a NOT NULL on a partially-migrated row.
+      item_ref     TEXT,
+      bundle_id    TEXT,
+      component_id TEXT,
+      concept_id   TEXT,
+      adapter_id   TEXT,
+      type         TEXT,
+      content_hash TEXT,
+      document_json TEXT
     );
 
     CREATE INDEX IF NOT EXISTS idx_entries_dir ON entries(dir_path);
     CREATE INDEX IF NOT EXISTS idx_entries_type ON entries(entry_type);
     CREATE INDEX IF NOT EXISTS idx_entries_file_path ON entries(file_path);
   `);
+    // v18: backfill the bundle-adapter identity/provenance columns on databases
+    // created against a pre-v18 binary (partial schema) — same PRAGMA-then-ALTER
+    // guard pattern as `ensureDerivedFromColumn`. Runs BEFORE the item_ref index
+    // so the CREATE INDEX below never references a not-yet-added column.
+    ensureBundleRefColumns(db);
+    // v19 (F4c, spec §11.4): item_ref is THE durable identity — its index is
+    // UNIQUE. Every indexed row now carries item_ref; SQLite treats NULLs as
+    // distinct in a UNIQUE index, so NULL-item_ref write-back stragglers coexist.
+    // A pre-v19 DB carries a NON-unique `idx_entries_item_ref`, so drop-then-create.
+    ensureUniqueItemRefIndex(db);
     // Phase 5A / DB v17: backfill `derived_from` column + index on databases
     // that were created at v17 fresh OR carry a partial v17 schema (a DB whose
     // `index_meta.version` was bumped to 17 but whose `entries` table still
@@ -152,7 +317,9 @@ export function ensureSchema(db, embeddingDim) {
       );
     `);
     }
-    // Usage events table — created by ensureUsageEventsSchema() at runtime.
+    // usage_events + legacy_state moved to state.db (Chunk-8 WI-8.3, DB_VERSION
+    // v20). index.db no longer creates them; usage_events writers/readers open
+    // state.db. utility_scores (a regenerable index.db cache) stays here.
     // Utility scores table (aggregated per-entry utility metrics)
     db.exec(`
     CREATE TABLE IF NOT EXISTS utility_scores (
@@ -229,94 +396,7 @@ export function ensureSchema(db, embeddingDim) {
     // migrateGraphDataFromLegacy() copies the data across so the graph is PRESERVED
     // (not re-extracted).
     migrateGraphFilesSchema(db);
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS graph_meta (
-      stash_root          TEXT PRIMARY KEY,
-      schema_version      INTEGER NOT NULL,
-      generated_at        TEXT NOT NULL,
-      considered_files    INTEGER NOT NULL DEFAULT 0,
-      extracted_files     INTEGER NOT NULL DEFAULT 0,
-      entity_count        INTEGER NOT NULL DEFAULT 0,
-      relation_count      INTEGER NOT NULL DEFAULT 0,
-      extraction_coverage REAL NOT NULL DEFAULT 0,
-      density             REAL NOT NULL DEFAULT 0,
-      extractor_id        TEXT,
-      extraction_run_id   TEXT,
-      model               TEXT,
-      prompt_version      TEXT,
-      batch_size          INTEGER,
-      cache_hits          INTEGER NOT NULL DEFAULT 0,
-      cache_misses        INTEGER NOT NULL DEFAULT 0,
-      truncation_count    INTEGER NOT NULL DEFAULT 0,
-      failure_count       INTEGER NOT NULL DEFAULT 0
-    );
-
-    CREATE TABLE IF NOT EXISTS graph_files (
-      stash_root        TEXT NOT NULL,
-      file_path         TEXT NOT NULL,
-      file_order        INTEGER NOT NULL,
-      file_type         TEXT NOT NULL,
-      body_hash         TEXT NOT NULL,
-      confidence        REAL,
-      status            TEXT NOT NULL DEFAULT 'extracted',
-      reason            TEXT,
-      extraction_run_id TEXT,
-      PRIMARY KEY (stash_root, file_path, body_hash)
-    );
-
-    CREATE UNIQUE INDEX IF NOT EXISTS idx_graph_files_path
-      ON graph_files(stash_root, file_path);
-
-    CREATE INDEX IF NOT EXISTS idx_graph_files_stash_order
-      ON graph_files(stash_root, file_order);
-
-    CREATE TABLE IF NOT EXISTS graph_file_entities (
-      stash_root   TEXT NOT NULL,
-      file_path    TEXT NOT NULL,
-      body_hash    TEXT NOT NULL,
-      entity_order INTEGER NOT NULL,
-      entity_norm  TEXT NOT NULL,
-      entity       TEXT NOT NULL,
-      PRIMARY KEY (stash_root, file_path, body_hash, entity_order),
-      FOREIGN KEY (stash_root, file_path, body_hash)
-        REFERENCES graph_files(stash_root, file_path, body_hash) ON DELETE CASCADE
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_graph_file_entities_entity_norm
-      ON graph_file_entities(stash_root, entity_norm);
-
-    CREATE TABLE IF NOT EXISTS graph_file_relations (
-      stash_root     TEXT NOT NULL,
-      file_path      TEXT NOT NULL,
-      body_hash      TEXT NOT NULL,
-      relation_order INTEGER NOT NULL,
-      from_entity_norm TEXT NOT NULL,
-      from_entity    TEXT NOT NULL,
-      to_entity_norm TEXT NOT NULL,
-      to_entity      TEXT NOT NULL,
-      relation_type  TEXT,
-      confidence     REAL,
-      PRIMARY KEY (stash_root, file_path, body_hash, relation_order),
-      FOREIGN KEY (stash_root, file_path, body_hash)
-        REFERENCES graph_files(stash_root, file_path, body_hash) ON DELETE CASCADE
-    );
-
-    -- #624-P3: lazy graph-extraction queue. Standalone table (NO FK to
-    -- graph_files — a queued file by definition has no graph row yet).
-    -- Idempotent on (stash_root, file_path); drained highest-priority-first.
-    -- CREATE TABLE IF NOT EXISTS is the forward migration (no DB_VERSION bump).
-    CREATE TABLE IF NOT EXISTS graph_extraction_queue (
-      stash_root TEXT NOT NULL,
-      file_path  TEXT NOT NULL,
-      body_hash  TEXT NOT NULL,
-      queued_at  TEXT NOT NULL DEFAULT (datetime('now')),
-      priority   INTEGER NOT NULL DEFAULT 0,
-      PRIMARY KEY (stash_root, file_path)
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_graph_extraction_queue_drain
-      ON graph_extraction_queue(stash_root, priority DESC, queued_at);
-  `);
+    ensureGraphTables(db);
     // #624-P1 migration step 2: copy any renamed-aside legacy graph data into the
     // new-shape tables (just created above), then drop the legacy tables. No-op
     // unless migrateGraphFilesSchema renamed a legacy graph_files this open.
@@ -385,8 +465,8 @@ export function ensureSchema(db, embeddingDim) {
             setMeta(db, "embeddingDim", String(embeddingDim));
         }
     }
-    // Usage telemetry table
-    ensureUsageEventsSchema(db);
+    // Usage telemetry (usage_events) lives in state.db since Chunk-8 WI-8.3 —
+    // no longer created here.
     // Registry index cache table — caches remote registry index documents so
     // `akm search` does not hit the network on every invocation.
     db.exec(REGISTRY_INDEX_CACHE_DDL);
@@ -410,6 +490,52 @@ function ensureDerivedFromColumn(db) {
         db.exec("CREATE INDEX IF NOT EXISTS idx_entries_derived_from ON entries(derived_from)");
     }, "entries table may not exist on a brand-new DB before CREATE — caller is responsible");
 }
+/**
+ * Chunk-5 Step 2 / DB v18 schema guard.
+ *
+ * Ensures the bundle-adapter identity/provenance columns exist on the open
+ * `entries` table. Called from `ensureSchema()` after the entries CREATE so a
+ * legacy database (created against a pre-v18 binary) gains the new columns
+ * without a rebuild. All columns are nullable and added ADDITIVELY — the
+ * writer populates `item_ref`/`bundle_id`/`component_id`/`concept_id`/
+ * `adapter_id`/`type` while readers still key on the legacy columns. Idempotent:
+ * a `PRAGMA table_info` lookup gates each ALTER.
+ */
+function ensureBundleRefColumns(db) {
+    bestEffort(() => {
+        const cols = db.prepare("PRAGMA table_info(entries)").all();
+        const have = new Set(cols.map((c) => c.name));
+        const additive = [
+            ["item_ref", "TEXT"],
+            ["bundle_id", "TEXT"],
+            ["component_id", "TEXT"],
+            ["concept_id", "TEXT"],
+            ["adapter_id", "TEXT"],
+            ["type", "TEXT"],
+            ["content_hash", "TEXT"],
+            ["document_json", "TEXT"],
+        ];
+        for (const [name, sqlType] of additive) {
+            if (!have.has(name))
+                db.exec(`ALTER TABLE entries ADD COLUMN ${name} ${sqlType}`);
+        }
+    }, "entries table may not exist on a brand-new DB before CREATE — caller is responsible");
+}
+/**
+ * Chunk-5 flip F4c / DB v19 schema guard.
+ *
+ * Upgrade `entries.item_ref`'s lookup index to UNIQUE (item_ref is now THE
+ * durable identity — spec §11.4). A pre-v19 DB has a NON-unique
+ * `idx_entries_item_ref`, so we DROP-then-CREATE (a `CREATE UNIQUE INDEX IF NOT
+ * EXISTS` under the same name would no-op against the existing non-unique index).
+ * SQLite treats NULLs as distinct in a UNIQUE index. Duplicate durable identities
+ * are an invalid index and fail the schema open rather than enabling a dual-key
+ * fallback.
+ */
+function ensureUniqueItemRefIndex(db) {
+    db.exec("DROP INDEX IF EXISTS idx_entries_item_ref");
+    db.exec("CREATE UNIQUE INDEX idx_entries_item_ref ON entries(item_ref)");
+}
 /**
  * Returns true when a table exists in the current database.
  */

package/dist/storage/repositories/index-sql.js

@@ -0,0 +1,12 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Shared SQL constants for the `index.db` storage repositories.
+ */
+/**
+ * SQLite parameter chunk size — chosen well below SQLITE_MAX_VARIABLE_NUMBER
+ * (default 999 on most builds) so multi-row `IN (?, ?, ...)` queries stay
+ * within bounds. Shared by the entries / fts / utility / llm-cache repos.
+ */
+export const SQLITE_CHUNK_SIZE = 500;