akm-cli 0.9.1 → 0.9.2-alpha.2

package/dist/storage/repositories/index-schema.js

@@ -1,77 +1,26 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
-/**
- * index.db schema, version stamps, and targeted migrations, kept in the
- * storage layer. This isolates the one genuinely risky area (schema
- * evolution) from the CRUD/FTS/vector queries.
- *
- * The meta accessors, embedding purge, and vec-availability probe that
- * `ensureSchema` leans on live in the sibling `index-meta-repository` /
- * `index-vec-repository` modules.
- */
-import { bestEffort } from "../../core/best-effort.js";
-import { warn } from "../../core/warn.js";
+import { CANONICAL_ENTRY_SCHEMA_SQL, CANONICAL_INDEX_DB_VERSION, isCanonicalIndexGeneration, } from "./index-entry-schema.js";
 import { getMeta, setMeta } from "./index-meta-repository.js";
 import { isVecAvailable, purgeEmbeddings } from "./index-vec-repository.js";
 // ── Constants ───────────────────────────────────────────────────────────────
-// NOTE: schema changes are additive. DB_VERSION is a forensic stamp only — it
-// no longer gates any destructive path (the old nuclear drop-and-rebuild was
-// removed; index.db's idempotent CREATE … IF NOT EXISTS schema converges any
-// older/partial DB forward without dropping data). Graph re-keying uses a
-// TARGETED, graph-only migration (migrateGraphFilesSchema) — the model for any
-// incompatible change: migrate in place, never wipe the whole index.
-//
-// v17→v18 (Chunk-5 Step 2, spec §14.4): the `entries` table gains the durable
-// bundle-adapter identity/provenance columns — `item_ref` (`<bundle>//<concept
-// -id>` canonical stored spelling), `bundle_id`/`component_id`/`concept_id`/
-// `adapter_id` provenance, `type` (open token), and `content_hash`/`document
-// _json`. They land ADDITIVELY ALONGSIDE the legacy `entry_key`/`dir_path`/
-// `stash_dir`/`entry_json`/`entry_type` columns (dev-time transitional shape):
-// the writer populates the identity/provenance columns while every reader still
-// keys on the legacy columns, so the battery stays green while the reader
-// repoint + ref-grammar flip land incrementally. The legacy columns + this
-// coexistence are removed once every reader is repointed onto `item_ref`
-// (spec §3.3 — single clean shape, no dual read-path). The index is a
-// regenerable derived cache, so an `akm index` rebuild repopulates the new
-// columns on any DB opened at an older version.
+// index.db is a regenerable cache. Incompatible entry-schema changes advance
+// this generation and discard only derived index tables; durable state remains
+// in state.db. Current readers and writers therefore target exactly one schema
+// and never carry live compatibility SQL for previous generations.
 //
-// v18→v19 (Chunk-5 flip F4c, spec §11.4): `entries.item_ref` becomes THE durable
-// identity — its lookup index is upgraded to UNIQUE (every indexed row now
-// carries item_ref; NULLs are distinct in a SQLite UNIQUE index, so write-back
-// stragglers coexist). The durable state keyed off refs (`usage_events.entry_ref`)
-// is re-keyed onto item_ref by the one-time §11.4 migration cutover
-// (020-three-db-cutover) with orphan quarantine in `legacy_state`. The index is
-// regenerable, so a rebuild is an acceptable fallback if the UNIQUE upgrade finds
-// a duplicate on a partially-migrated DB.
+// v20→v21: remove the transitional entry_key/dir_path/stash_dir/entry_json/
+// entry_type columns. item_ref is the sole conflict key; document_json is the
+// sole stored document projection; bundle provenance and file_path provide the
+// current identity and materialized read path.
 //
-// v19→v20 (Chunk-8 WI-8.3, three-DB merge): index.db STOPS owning `usage_events`
-// + `legacy_state`. Both are durable, non-regenerable state and now live in
-// state.db (folded by state migration 020; the cutover rescues the old index.db
-// rows across). index.db is a regenerable cache, so dropping their DDL from the
-// schema path needs no migration — a rebuild simply no longer re-creates them.
-// STANDARD REBUILD NOTE: after this bump, any index.db opened at v19 is rebuilt
-// from the stash on the next `akm index`; usage_events/legacy_state are read
-// exclusively from state.db thereafter.
-//
-// Reader/writer repoint progress (spec §3.3 "single clean shape"): the `entries`
-// upsert (index-entries-repository `getUpsertStmts`) uses the UNIQUE `item_ref`
-// as its PRIMARY conflict target, and the graph-boost related-ref reader
-// (`listRelatedPathsForFile`) resolves the user-facing ref from
-// `concept_id`/`item_ref` instead of stripping `entry_key`. The legacy
-// `entry_key`/`dir_path`/`stash_dir`/`entry_type`/`entry_json` columns are NOT
-// yet removable: they retain live consumers outside this module — `entry_key`
-// (mv-cli re-key, usage-event legacy resolution, index-entry-mapper, the LLM
-// cache), `entry_json` (the row payload every reader decodes), `entry_type`
-// (FTS + workflow loader), `stash_dir`/`dir_path` (scan/delete/utility scoping).
-// The upsert therefore keeps `entry_key` as a NULL-item_ref-safe SECOND conflict
-// target (the LLM metadata-enhance re-upsert still writes existing rows with a
-// NULL item_ref) and degrades to it entirely when the item_ref index is the
-// non-unique fallback. Both are deletable once every remaining reader repoints.
-export const DB_VERSION = 20;
+// v21→v22: entry mutations publish FTS synchronously and no dirty queue exists.
+// Discard the old derived generation so stale FTS rows and caller-managed dirty
+// state cannot cross the mutation-authority boundary.
+export const DB_VERSION = CANONICAL_INDEX_DB_VERSION;
 export const EMBEDDING_DIM = 384;
-// #624-P1: graph_files re-keyed to (stash_root, file_path, body_hash). Bumped 3→4
-// as a marker; the actual migration is the targeted drop in migrateGraphFilesSchema.
+// #624-P1: graph_files is keyed to (stash_root, file_path, body_hash).
 export const GRAPH_SCHEMA_VERSION = 4;
 // ── Schema ──────────────────────────────────────────────────────────────────
 /**
@@ -111,9 +60,7 @@ const REGISTRY_INDEX_CACHE_DDL = `
 `;
 /**
  * Create the graph-extraction tables (`graph_meta`/`graph_files`/`graph_file_entities`/
- * `graph_file_relations`/`graph_extraction_queue`). Extracted verbatim from
- * {@link ensureSchema} (called at the same point, between `migrateGraphFilesSchema`
- * and `migrateGraphDataFromLegacy`) — a pure, behavior-identical decomposition.
+ * `graph_file_relations`/`graph_extraction_queue`).
  */
 function ensureGraphTables(db) {
     db.exec(`
@@ -205,6 +152,48 @@ function ensureGraphTables(db) {
       ON graph_extraction_queue(stash_root, priority DESC, queued_at);
   `);
 }
+/**
+ * Cross the incompatible entry-schema boundary by discarding the derived index
+ * generation. No row conversion or dual-schema compatibility is attempted:
+ * the next index run rebuilds entries, FTS, embeddings, utility aggregates,
+ * graph extraction, and enrichment caches from current sources/state.
+ */
+function rebuildIncompatibleIndexGeneration(db) {
+    const version = getMeta(db, "version");
+    const hasEntries = tableExists(db, "entries");
+    if (!hasEntries && version === undefined)
+        return;
+    if (isCanonicalIndexGeneration(db))
+        return;
+    let vecResetPending = false;
+    try {
+        db.exec("DROP TABLE IF EXISTS entries_vec");
+    }
+    catch {
+        // A vec0 table cannot be dropped while sqlite-vec is unavailable. It does
+        // not reference entries, so leave a marker and drop it on the first later
+        // open where the extension is available.
+        vecResetPending = true;
+    }
+    db.transaction(() => {
+        db.exec("DROP TABLE IF EXISTS graph_file_relations");
+        db.exec("DROP TABLE IF EXISTS graph_file_entities");
+        db.exec("DROP TABLE IF EXISTS graph_files");
+        db.exec("DROP TABLE IF EXISTS graph_extraction_queue");
+        db.exec("DROP TABLE IF EXISTS graph_meta");
+        db.exec("DROP TABLE IF EXISTS entries_fts_dirty");
+        db.exec("DROP TABLE IF EXISTS entries_fts");
+        db.exec("DROP TABLE IF EXISTS embeddings");
+        db.exec("DROP TABLE IF EXISTS utility_scores_scoped");
+        db.exec("DROP TABLE IF EXISTS utility_scores");
+        db.exec("DROP TABLE IF EXISTS llm_enrichment_cache");
+        db.exec("DROP TABLE IF EXISTS index_dir_state");
+        db.exec("DROP TABLE IF EXISTS entries");
+        db.exec("DELETE FROM index_meta");
+    })();
+    if (vecResetPending)
+        setMeta(db, "vecResetPending", "1");
+}
 export function ensureSchema(db, embeddingDim) {
     // Create meta table first so we can check version
     db.exec(`
@@ -213,87 +202,14 @@ export function ensureSchema(db, embeddingDim) {
       value TEXT NOT NULL
     );
   `);
-    // index.db is a fully regenerable derived cache, so its schema is built
-    // idempotently below: every table is CREATE … IF NOT EXISTS and column
-    // additions go through guarded ALTERs (ensureDerivedFromColumn) and targeted
-    // migrations (migrateGraphFilesSchema / migrateGraphDataFromLegacy). Opening a
-    // database with an older or partial schema converges it forward WITHOUT ever
-    // dropping data — there is intentionally no "nuclear drop the whole index on a
-    // DB_VERSION mismatch" path (a destructive design the regenerable index never
-    // needed, and whose pre-drop data-dir backup it required). A genuinely
-    // incompatible change is handled by an additive/targeted migration; the few
-    // derived tables that ever must be rebuilt are regenerated by `akm index`.
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS entries (
-      id          INTEGER PRIMARY KEY AUTOINCREMENT,
-      entry_key   TEXT NOT NULL UNIQUE,
-      dir_path    TEXT NOT NULL,
-      file_path   TEXT NOT NULL,
-      stash_dir   TEXT NOT NULL,
-      entry_json  TEXT NOT NULL,
-      search_text TEXT NOT NULL,
-      entry_type  TEXT NOT NULL,
-      derived_from TEXT,
-      -- Chunk-5 Step 2 / DB v18 (spec 14.4): bundle-adapter identity + provenance,
-      -- ADDITIVE alongside the legacy columns above. item_ref is the durable
-      -- <bundle>//<concept-id> spelling; nullable during the transition so a
-      -- pre-repoint reader path never trips a NOT NULL on a partially-migrated row.
-      item_ref     TEXT,
-      bundle_id    TEXT,
-      component_id TEXT,
-      concept_id   TEXT,
-      adapter_id   TEXT,
-      type         TEXT,
-      content_hash TEXT,
-      document_json TEXT
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_entries_dir ON entries(dir_path);
-    CREATE INDEX IF NOT EXISTS idx_entries_type ON entries(entry_type);
-    CREATE INDEX IF NOT EXISTS idx_entries_file_path ON entries(file_path);
-  `);
-    // v18: backfill the bundle-adapter identity/provenance columns on databases
-    // created against a pre-v18 binary (partial schema) — same PRAGMA-then-ALTER
-    // guard pattern as `ensureDerivedFromColumn`. Runs BEFORE the item_ref index
-    // so the CREATE INDEX below never references a not-yet-added column.
-    ensureBundleRefColumns(db);
-    // v19 (F4c, spec §11.4): item_ref is THE durable identity — its index is
-    // UNIQUE. Every indexed row now carries item_ref; SQLite treats NULLs as
-    // distinct in a UNIQUE index, so NULL-item_ref write-back stragglers coexist.
-    // A pre-v19 DB carries a NON-unique `idx_entries_item_ref`, so drop-then-create.
-    ensureUniqueItemRefIndex(db);
-    // Phase 5A / DB v17: backfill `derived_from` column + index on databases
-    // that were created at v17 fresh OR carry a partial v17 schema (a DB whose
-    // `index_meta.version` was bumped to 17 but whose `entries` table still
-    // lacks the column — this happens when a previous v17 binary opened a
-    // pre-v17 DB without taking the upgrade path because no version mismatch
-    // was seen at boot). The PRAGMA-then-ALTER guard runs unconditionally so
-    // both fresh and partial schemas converge. The CREATE INDEX for
-    // `derived_from` MUST run after this helper so we never reference a
-    // column that has not yet been added on partial schemas.
-    ensureDerivedFromColumn(db);
-    // Validated WorkflowDocument JSON, one row per indexed workflow entry.
-    // Pure index data — fully rebuilt on each `akm index`. ON DELETE CASCADE
-    // means clearing entries (full rebuild or per-dir delete) drops these too.
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS workflow_documents (
-      entry_id        INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE,
-      schema_version  INTEGER NOT NULL,
-      document_json   TEXT NOT NULL,
-      source_path     TEXT NOT NULL,
-      source_hash     TEXT NOT NULL,
-      updated_at      TEXT NOT NULL
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_workflow_documents_source_path
-      ON workflow_documents(source_path);
-  `);
-    // Set version immediately after table creation so a crash before the end of
-    // ensureSchema() does not leave the database in a versionless state on next open.
-    const versionAfterCreate = getMeta(db, "version");
-    if (!versionAfterCreate) {
-        setMeta(db, "version", String(DB_VERSION));
-    }
+    rebuildIncompatibleIndexGeneration(db);
+    db.exec(CANONICAL_ENTRY_SCHEMA_SQL);
+    // Workflow source is compiled directly into source IR at each command
+    // boundary. The former workflow_documents cache duplicated that IR in a
+    // second persisted representation and was never used by current execution.
+    // index.db is derived state, so remove the obsolete table on every open.
+    db.exec("DROP TABLE IF EXISTS workflow_documents");
+    setMeta(db, "version", String(DB_VERSION));
     // BLOB-based embedding storage (always available, no sqlite-vec needed)
     db.exec(`
     CREATE TABLE IF NOT EXISTS embeddings (
@@ -317,9 +233,8 @@ export function ensureSchema(db, embeddingDim) {
       );
     `);
     }
-    // usage_events + legacy_state moved to state.db (Chunk-8 WI-8.3, DB_VERSION
-    // v20). index.db no longer creates them; usage_events writers/readers open
-    // state.db. utility_scores (a regenerable index.db cache) stays here.
+    // usage_events lives in state.db. utility_scores remains a regenerable
+    // index.db cache.
     // Utility scores table (aggregated per-entry utility metrics)
     db.exec(`
     CREATE TABLE IF NOT EXISTS utility_scores (
@@ -360,7 +275,7 @@ export function ensureSchema(db, embeddingDim) {
     // result for each asset so that subsequent `akm index --enrich` runs can
     // skip the LLM call when the body hasn't changed. The cache is keyed by
     // a stable asset_ref string (e.g. the absolute file path for graph/memory
-    // passes, or `entryKey:passId` for the metadata-enhance pass).
+    // passes, or `itemRef:passId` for the metadata-enhance pass).
     // Entries are cleaned up when assets are removed or --re-enrich is used.
     db.exec(`
     CREATE TABLE IF NOT EXISTS llm_enrichment_cache (
@@ -389,27 +304,13 @@ export function ensureSchema(db, embeddingDim) {
     // body_hash) and declare a composite FK -> graph_files ON DELETE CASCADE so
     // child rows are removed when a graph_files row is replaced.
     //
-    // #624-P1 targeted migration: an existing DB may still hold the OLD graph_files
-    // (entry_id PK). SQLite can't ALTER a primary key, so we RENAME the 3 graph
-    // tables aside (→ *_legacy) here — ONLY the graph tables, never the index/
-    // embeddings — then the CREATE block below builds the new shape, then
-    // migrateGraphDataFromLegacy() copies the data across so the graph is PRESERVED
-    // (not re-extracted).
-    migrateGraphFilesSchema(db);
     ensureGraphTables(db);
-    // #624-P1 migration step 2: copy any renamed-aside legacy graph data into the
-    // new-shape tables (just created above), then drop the legacy tables. No-op
-    // unless migrateGraphFilesSchema renamed a legacy graph_files this open.
-    migrateGraphDataFromLegacy(db);
-    // FTS-dirty queue. Created here (not lazily on first upsert) so the
-    // per-entry write path doesn't issue a CREATE TABLE IF NOT EXISTS on
-    // every call — that DDL would fire thousands of times during a full
-    // index. See `markFtsDirty` and `rebuildFts({ incremental: true })`.
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS entries_fts_dirty (
-      entry_id INTEGER PRIMARY KEY
-    );
-  `);
+    // If a generation rebuild could not drop a vec0 table while the extension
+    // was unavailable, finish that reset as soon as vec0 can be loaded again.
+    if (isVecAvailable(db) && getMeta(db, "vecResetPending") === "1") {
+        db.exec("DROP TABLE IF EXISTS entries_vec");
+        setMeta(db, "vecResetPending", "0");
+    }
     // sqlite-vec table
     //
     // Dimension contract:
@@ -471,86 +372,6 @@ export function ensureSchema(db, embeddingDim) {
     // `akm search` does not hit the network on every invocation.
     db.exec(REGISTRY_INDEX_CACHE_DDL);
 }
-/**
- * Phase 5A / DB v17 schema guard.
- *
- * Ensures the `entries.derived_from` column + index exist on the open
- * connection. Called from `ensureSchema()` after the entries CREATE so that
- * legacy databases (created against a pre-v17 binary) still gain the new column
- * without data loss. Idempotent: a `PRAGMA table_info` lookup gates the ALTER.
- */
-function ensureDerivedFromColumn(db) {
-    bestEffort(() => {
-        const cols = db.prepare("PRAGMA table_info(entries)").all();
-        const hasColumn = cols.some((c) => c.name === "derived_from");
-        if (!hasColumn) {
-            db.exec("ALTER TABLE entries ADD COLUMN derived_from TEXT");
-        }
-        // Index creation is idempotent on its own; safe to call unconditionally.
-        db.exec("CREATE INDEX IF NOT EXISTS idx_entries_derived_from ON entries(derived_from)");
-    }, "entries table may not exist on a brand-new DB before CREATE — caller is responsible");
-}
-/**
- * Chunk-5 Step 2 / DB v18 schema guard.
- *
- * Ensures the bundle-adapter identity/provenance columns exist on the open
- * `entries` table. Called from `ensureSchema()` after the entries CREATE so a
- * legacy database (created against a pre-v18 binary) gains the new columns
- * without a rebuild. All columns are nullable and added ADDITIVELY — the
- * writer populates `item_ref`/`bundle_id`/`component_id`/`concept_id`/
- * `adapter_id`/`type` while readers still key on the legacy columns. Idempotent:
- * a `PRAGMA table_info` lookup gates each ALTER.
- */
-function ensureBundleRefColumns(db) {
-    bestEffort(() => {
-        const cols = db.prepare("PRAGMA table_info(entries)").all();
-        const have = new Set(cols.map((c) => c.name));
-        const additive = [
-            ["item_ref", "TEXT"],
-            ["bundle_id", "TEXT"],
-            ["component_id", "TEXT"],
-            ["concept_id", "TEXT"],
-            ["adapter_id", "TEXT"],
-            ["type", "TEXT"],
-            ["content_hash", "TEXT"],
-            ["document_json", "TEXT"],
-        ];
-        for (const [name, sqlType] of additive) {
-            if (!have.has(name))
-                db.exec(`ALTER TABLE entries ADD COLUMN ${name} ${sqlType}`);
-        }
-    }, "entries table may not exist on a brand-new DB before CREATE — caller is responsible");
-}
-/**
- * Chunk-5 flip F4c / DB v19 schema guard.
- *
- * Upgrade `entries.item_ref`'s lookup index to UNIQUE (item_ref is now THE
- * durable identity — spec §11.4). A pre-v19 DB has a NON-unique
- * `idx_entries_item_ref`, so we DROP-then-CREATE (a `CREATE UNIQUE INDEX IF NOT
- * EXISTS` under the same name would no-op against the existing non-unique index).
- * SQLite treats NULLs as distinct in a UNIQUE index. Duplicate durable identities
- * are an invalid index and fail the schema open rather than enabling a dual-key
- * fallback.
- */
-function ensureUniqueItemRefIndex(db) {
-    // Probe before mutating. This ran unconditionally on EVERY open as two
-    // separate autocommit statements, so there was always a window in which the
-    // index did not exist — a concurrent open (registry-cache search, indexer,
-    // improve) could DROP between the other's DROP and CREATE and then fail with
-    // "index idx_entries_item_ref already exists", or serve a query with no index
-    // at all. A DB whose index is already UNIQUE needs no work.
-    const existing = db
-        .prepare("SELECT sql FROM sqlite_master WHERE type = 'index' AND name = 'idx_entries_item_ref'")
-        .get();
-    if (existing?.sql && /\bUNIQUE\b/i.test(existing.sql))
-        return;
-    // Pre-v19 (non-unique index) or absent: convert atomically so a racing open
-    // sees either the old index or the new one, never neither.
-    db.transaction(() => {
-        db.exec("DROP INDEX IF EXISTS idx_entries_item_ref");
-        db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_entries_item_ref ON entries(item_ref)");
-    })();
-}
 /**
  * Returns true when a table exists in the current database.
  */
@@ -558,100 +379,3 @@ function tableExists(db, name) {
     const row = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=? LIMIT 1").get(name);
     return row !== undefined && row !== null;
 }
-/**
- * #624-P1 targeted graph-schema migration — STEP 1 of 2 (rename).
- *
- * graph_files was re-keyed from `entry_id INTEGER PRIMARY KEY REFERENCES
- * entries(id)` to a self-contained `(stash_root, file_path, body_hash)` PK.
- * SQLite cannot ALTER a primary key, so an existing DB carrying the OLD shape
- * has its 3 graph tables RENAMED to `*_legacy` here; ensureSchema's CREATE block
- * then builds the new-shape tables, and {@link migrateGraphDataFromLegacy} COPIES
- * the data across before dropping the legacy tables. The graph is preserved —
- * NOT re-extracted (re-extraction is ~19s/file of LLM work).
- *
- * Crucially this is GRAPH-SCOPED: it touches ONLY the graph tables, never the
- * index / embeddings / enrichment cache. So users keep their (expensive)
- * embeddings instead of being forced into a full re-embed by a DB_VERSION bump.
- *
- * Detection: the old schema has an `entry_id` column on graph_files. Fresh DBs
- * (no graph_files yet) and already-migrated DBs (no entry_id column) are no-ops.
- * Idempotent.
- */
-function migrateGraphFilesSchema(db) {
-    bestEffort(() => {
-        const cols = db.prepare("PRAGMA table_info(graph_files)").all();
-        const isLegacyShape = cols.some((c) => c.name === "entry_id");
-        if (!isLegacyShape)
-            return;
-        // A previous interrupted migration may have left *_legacy behind — drop those
-        // husks first so the rename below doesn't collide.
-        db.exec("DROP TABLE IF EXISTS graph_file_relations_legacy");
-        db.exec("DROP TABLE IF EXISTS graph_file_entities_legacy");
-        db.exec("DROP TABLE IF EXISTS graph_files_legacy");
-        // Rename the 3 entry_id-keyed tables aside. graph_meta is unchanged (stash_root
-        // key) so it is left in place. ALTER … RENAME auto-updates child FK refs in
-        // SQLite ≥3.25, which is fine — the legacy children are dropped after the copy.
-        db.exec("ALTER TABLE graph_files RENAME TO graph_files_legacy");
-        if (tableExists(db, "graph_file_entities")) {
-            db.exec("ALTER TABLE graph_file_entities RENAME TO graph_file_entities_legacy");
-        }
-        if (tableExists(db, "graph_file_relations")) {
-            db.exec("ALTER TABLE graph_file_relations RENAME TO graph_file_relations_legacy");
-        }
-    }, "graph_files may not exist on a brand-new DB before CREATE — caller is responsible");
-}
-/**
- * #624-P1 targeted graph-schema migration — STEP 2 of 2 (copy + drop legacy).
- *
- * Runs AFTER the graph CREATE TABLE block, so the new-shape tables exist. Copies
- * every legacy row into the re-keyed tables — the old tables already carry
- * (stash_root, file_path, body_hash) next to entry_id, so the projection is a
- * straight column copy (children JOIN back to graph_files_legacy to resolve the
- * composite key from their entry_id). Then drops the `*_legacy` tables.
- *
- * Best-effort: a copy failure (e.g. a pre-body_hash legacy schema) is tolerated,
- * and the legacy tables are dropped regardless so they never linger. Rows whose
- * body_hash is null/empty can't form the new PK and are skipped (they re-extract).
- */
-function migrateGraphDataFromLegacy(db) {
-    if (!tableExists(db, "graph_files_legacy"))
-        return;
-    let migratedFiles = 0;
-    bestEffort(() => {
-        db.transaction(() => {
-            const res = db
-                .prepare(`INSERT OR IGNORE INTO graph_files
-             (stash_root, file_path, body_hash, file_order, file_type, confidence, status, reason, extraction_run_id)
-           SELECT stash_root, file_path, body_hash, file_order, file_type, confidence, status, reason, extraction_run_id
-             FROM graph_files_legacy
-            WHERE body_hash IS NOT NULL AND body_hash != ''`)
-                .run();
-            migratedFiles = Number(res.changes);
-            if (tableExists(db, "graph_file_entities_legacy")) {
-                db.exec(`INSERT OR IGNORE INTO graph_file_entities
-             (stash_root, file_path, body_hash, entity_order, entity_norm, entity)
-           SELECT gf.stash_root, gf.file_path, gf.body_hash, e.entity_order, e.entity_norm, e.entity
-             FROM graph_file_entities_legacy e
-             JOIN graph_files_legacy gf ON gf.entry_id = e.entry_id
-            WHERE gf.body_hash IS NOT NULL AND gf.body_hash != ''`);
-            }
-            if (tableExists(db, "graph_file_relations_legacy")) {
-                db.exec(`INSERT OR IGNORE INTO graph_file_relations
-             (stash_root, file_path, body_hash, relation_order, from_entity_norm, from_entity, to_entity_norm, to_entity, relation_type, confidence)
-           SELECT gf.stash_root, gf.file_path, gf.body_hash, r.relation_order, r.from_entity_norm, r.from_entity, r.to_entity_norm, r.to_entity, r.relation_type, r.confidence
-             FROM graph_file_relations_legacy r
-             JOIN graph_files_legacy gf ON gf.entry_id = r.entry_id
-            WHERE gf.body_hash IS NOT NULL AND gf.body_hash != ''`);
-            }
-        })();
-    }, "graph data migration is best-effort; legacy tables are dropped regardless below");
-    // Always drop the legacy tables (children first), migrated or not.
-    bestEffort(() => {
-        db.exec("DROP TABLE IF EXISTS graph_file_relations_legacy");
-        db.exec("DROP TABLE IF EXISTS graph_file_entities_legacy");
-        db.exec("DROP TABLE IF EXISTS graph_files_legacy");
-    }, "drop legacy graph tables after migration");
-    if (migratedFiles > 0) {
-        warn(`[akm] graph index re-keyed (#624): migrated ${migratedFiles} extracted file(s) to the new schema — no re-extraction needed. Index + embeddings untouched.`);
-    }
-}

package/dist/storage/repositories/index-utility-repository.js

@@ -8,7 +8,6 @@
  * retrieval-frequency counting over `usage_events`. The bounded-step EMA policy
  * itself lives in `indexer/feedback/utility-policy`; this repo only reads/writes.
  */
-import path from "node:path";
 import { conceptIdFromTypeName, parseRefInput } from "../../core/asset/resolve-ref.js";
 import { computeNextUtility } from "../../indexer/feedback/utility-policy.js";
 import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
@@ -145,7 +144,7 @@ function bareRefCandidates(ref) {
 export function getRetrievalCounts(indexDb, stateDb, refs, options = {}) {
     if (refs.length === 0)
         return new Map();
-    if (options.sourceName || options.stashDir) {
+    if (options.sourceName) {
         return getSourceScopedRetrievalCounts(indexDb, stateDb, refs, options);
     }
     // Map each conceptId back to the input refs that produced it so DB results
@@ -218,20 +217,18 @@ function getSourceScopedRetrievalCounts(indexDb, stateDb, refs, options) {
                 bareToInputs.set(bare, [ref]);
         }
     }
-    // Cross-DB (Chunk-8 WI-8.3): usage_events rows come from state.db; the
-    // per-row `stash_dir` (formerly a LEFT JOIN on entries) is resolved from
-    // index.db by entry_id. Read the usage rows first, then batch-look-up the
-    // stash_dir for their entry_ids and join in JS.
-    const entryStashDir = indexDb.prepare("SELECT stash_dir FROM entries WHERE id = ?");
-    const stashDirFor = (entryId) => {
+    // Cross-DB: usage rows come from state.db; resolve the linked entry's
+    // canonical bundle identity from index.db by entry_id.
+    const entryBundle = indexDb.prepare("SELECT bundle_id FROM entries WHERE id = ?");
+    const bundleFor = (entryId) => {
         if (entryId === null)
             return null;
-        const row = entryStashDir.get(entryId);
-        return row?.stash_dir ?? null;
+        const row = entryBundle.get(entryId);
+        return row?.bundle_id ?? null;
     };
     const countsByBare = new Map();
     const bareForms = [...bareToInputs.keys()];
-    const selectedRoot = options.stashDir ? path.resolve(options.stashDir) : undefined;
+    const selectedBundle = options.sourceName;
     for (let i = 0; i < bareForms.length; i += SQLITE_CHUNK_SIZE) {
         const chunk = bareForms.slice(i, i + SQLITE_CHUNK_SIZE);
         const placeholders = chunk.map(() => "?").join(", ");
@@ -248,15 +245,13 @@ function getSourceScopedRetrievalCounts(indexDb, stateDb, refs, options) {
                   ELSE ue.entry_ref
                 END IN (${placeholders})`)
             .all(...chunk);
-        const rows = rawRows.map((r) => ({ ...r, stash_dir: stashDirFor(r.entry_id) }));
+        const rows = rawRows.map((r) => ({ ...r, bundle_id: bundleFor(r.entry_id) }));
         for (const row of rows) {
             const bare = bareRef(row.entry_ref);
-            const linkedToSelectedRoot = row.entry_id !== null && selectedRoot !== undefined && row.stash_dir !== null
-                ? path.resolve(row.stash_dir) === selectedRoot
-                : false;
-            const detached = row.entry_id === null || selectedRoot === undefined;
+            const linkedToSelectedBundle = row.entry_id !== null && row.bundle_id === selectedBundle;
+            const detached = row.entry_id === null;
             const qualifiedForSource = detached && options.sourceName !== undefined && row.entry_ref === `${options.sourceName}//${bare}`;
-            if (!linkedToSelectedRoot && !qualifiedForSource)
+            if (!linkedToSelectedBundle && !qualifiedForSource)
                 continue;
             countsByBare.set(bare, (countsByBare.get(bare) ?? 0) + 1);
         }

package/dist/storage/repositories/index-vec-repository.js

@@ -12,6 +12,7 @@ import { bestEffort } from "../../core/best-effort.js";
 import { warn } from "../../core/warn.js";
 import { cosineSimilarity } from "../../llm/embedders/types.js";
 import { getMeta, setMeta } from "./index-meta-repository.js";
+import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
 // ── sqlite-vec extension ────────────────────────────────────────────────────
 const vecStatus = new WeakMap();
 /**
@@ -68,6 +69,43 @@ export function isVecFastPathReady(db) {
     // flag, so the read path has to verify the table really exists.
     return hasVecTable(db);
 }
+/**
+ * Verify that the vec fast-path table mirrors the complete durable BLOB set.
+ *
+ * A targeted embedding write preserves the prior readiness decision because
+ * its subset cannot prove an older degraded generation is healed. Global
+ * materialization uses this aggregate check before promoting the persisted
+ * flag; search itself still reads the cheap flag and does not repeat the check
+ * per query.
+ */
+export function isVecFastPathComplete(db) {
+    if (!isVecAvailable(db) || !hasVecTable(db))
+        return false;
+    try {
+        const missingVecRows = db
+            .prepare(`
+        SELECT id FROM embeddings
+        EXCEPT
+        SELECT id FROM entries_vec
+        LIMIT 1
+      `)
+            .all();
+        if (missingVecRows.length > 0)
+            return false;
+        const orphanVecRows = db
+            .prepare(`
+        SELECT id FROM entries_vec
+        EXCEPT
+        SELECT id FROM embeddings
+        LIMIT 1
+      `)
+            .all();
+        return orphanVecRows.length === 0;
+    }
+    catch {
+        return false;
+    }
+}
 const vecTablePresent = new WeakMap();
 /**
  * Whether `entries_vec` exists on this connection, memoized per handle.
@@ -274,13 +312,24 @@ function searchBlobVec(db, queryEmbedding, k) {
  * Return all entries that do not yet have an embedding row.
  * Used by the embedding phase to determine which entries need vectors generated.
  */
-export function getAllEntriesForEmbedding(db) {
-    return db
-        .prepare(`
-      SELECT e.id, e.search_text AS searchText, e.entry_key AS entryKey, e.file_path AS filePath FROM entries e
-      WHERE NOT EXISTS (SELECT 1 FROM embeddings b WHERE b.id = e.id)
-    `)
-        .all();
+export function getAllEntriesForEmbedding(db, entryIds) {
+    const select = `
+      SELECT e.id, e.search_text AS searchText, e.item_ref AS itemRef, e.file_path AS filePath FROM entries e
+    `;
+    const missing = "NOT EXISTS (SELECT 1 FROM embeddings b WHERE b.id = e.id)";
+    if (entryIds === undefined) {
+        return db.prepare(`${select} WHERE ${missing} ORDER BY e.id`).all();
+    }
+    const targets = [...new Set(entryIds)].sort((left, right) => left - right);
+    const rows = [];
+    for (let offset = 0; offset < targets.length; offset += SQLITE_CHUNK_SIZE) {
+        const chunk = targets.slice(offset, offset + SQLITE_CHUNK_SIZE);
+        if (chunk.length === 0)
+            continue;
+        const placeholders = chunk.map(() => "?").join(",");
+        rows.push(...db.prepare(`${select} WHERE e.id IN (${placeholders}) AND ${missing} ORDER BY e.id`).all(...chunk));
+    }
+    return rows;
 }
 export function getEmbeddingCount(db) {
     const row = db.prepare("SELECT COUNT(*) AS cnt FROM embeddings").get();