akm-cli 0.9.16-alpha.1 → 0.9.16

package/dist/storage/repositories/index-entries-repository.js

@@ -14,24 +14,22 @@ import { parseBundleRef } from "../../core/asset/asset-ref.js";
 import { conceptIdFromTypeName } from "../../core/asset/resolve-ref.js";
 import { bestEffort } from "../../core/best-effort.js";
 import { isPathAbsent } from "../../core/path-access.js";
-import { getStateDbPath, withImmediateTransaction, withStateDb } from "../../core/state-db.js";
+import { getStateDbPath, withStateDb } from "../../core/state-db.js";
 import { warn } from "../../core/warn.js";
 import { getMarkdownFragmentContent, hasMarkdownFragmentContent, } from "../../indexer/passes/metadata.js";
 import { buildSearchText } from "../../indexer/search/search-fields.js";
 import { ENTRY_COLUMNS, rowToIndexedEntry } from "./index-entry-mapper.js";
-import { deleteFragmentSource, replaceFragmentSource } from "./index-fts-repository.js";
+import { deleteFtsEntries, replaceFtsEntry } from "./index-fts-repository.js";
 import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
+import { deleteEntryVectors, isVecAvailable } from "./index-vec-repository.js";
 // ── Entry operations ────────────────────────────────────────────────────────
 /**
  * Insert or update one canonical entry and all synchronously derived search
  * state. Returns the stable row id.
  *
- * The entries row and its safe-Markdown fragment source commit as one SQLite
- * transaction. Callers therefore cannot publish an entry and forget the
- * fragment-source write. (Unit derivation — `unit_texts`/`units_fts`/
- * `entry_units` — is the caller's job, driven by reconcile: a changed
- * `search_text` needs no explicit vector invalidation here, since a new
- * document simply derives new content-addressed unit hashes.)
+ * The entries row, FTS projection, and stale-vector invalidation commit as one
+ * SQLite transaction. Callers therefore cannot publish an entry and forget a
+ * second FTS maintenance step.
  */
 export function upsertEntry(db, filePath, entry, searchText, provenance, contentHash) {
     // Hot path during indexing — cache prepared statements per database
@@ -44,37 +42,20 @@ export function upsertEntry(db, filePath, entry, searchText, provenance, content
     // `content_hash` is optional on the LLM-enrichment re-upsert; a missing hash
     // preserves the scan writer's current value.
     const apply = () => {
+        const previous = stmts.findByItemRef.get(provenance.itemRef);
         const result = stmts.upsert.get(provenance.itemRef, provenance.bundleId, provenance.componentId, provenance.conceptId, provenance.adapterId, entry.type, filePath, contentHash ?? null, JSON.stringify(entry), searchText, derivedFrom);
         if (!result)
             throw new Error("upsertEntry: item_ref not found after upsert");
-        replaceFragmentSource(db, result.id, hasMarkdownFragmentContent(entry) ? (getMarkdownFragmentContent(entry) ?? null) : undefined);
+        if (previous?.id === result.id && previous.search_text !== searchText)
+            deleteEntryVectors(db, result.id);
+        replaceFtsEntry(db, result.id, entry, hasMarkdownFragmentContent(entry) ? (getMarkdownFragmentContent(entry) ?? null) : undefined);
         return result.id;
     };
-    // Both production callers (reconcile.ts's `applyChange`, enrich.ts's
-    // `applyEnrichmentToEntry`) always invoke this from inside their OWN outer
-    // `withImmediateTransaction`, so `db.inTransaction` is already true and a
-    // standalone top-level call never happens today — but a caller that DOES
-    // call this standalone (no outer transaction) must still get a proper
-    // `BEGIN IMMEDIATE`, not a bare deferred `db.transaction()` that fails
-    // instantly with SQLITE_BUSY under a competing writer instead of honouring
-    // `busy_timeout` (docs/plans/index-redesign.md rule 5). Route ONLY that
-    // standalone case through `withImmediateTransaction`.
-    //
-    // The nested case deliberately keeps the driver's own `db.transaction()`
-    // instead of `withImmediateTransaction`: the driver lowers a transaction
-    // opened inside an already-open one to a SAVEPOINT, so a caller that
-    // catches this mutation's error and continues its OWN outer (bare)
-    // transaction still cannot commit a partial entries row — the exact
-    // contract tests/integration/indexer/index-mutation-boundary.test.ts pins
-    // (#820). `withImmediateTransaction`'s join-if-open guard runs `fn`
-    // directly with NO isolation boundary of its own (by design, #686 — see
-    // its doc comment), which would let such a caught, partial write escape
-    // through the outer transaction's COMMIT instead of rolling back to a
-    // savepoint. Both real production callers already hold their OWN
-    // immediate transaction before calling this, so this nested branch is a
-    // savepoint under an already-immediate lock, not a fresh deferred BEGIN —
-    // no busy_timeout race to lose here either way.
-    return db.inTransaction ? db.transaction(apply)() : withImmediateTransaction(db, apply, "index");
+    // Always enter the driver's transaction wrapper. Both supported SQLite
+    // drivers lower a transaction opened inside another transaction to a
+    // savepoint, so a caller that catches this mutation's error cannot commit a
+    // partial entries row through its outer transaction.
+    return db.transaction(apply)();
 }
 const upsertStmtsByDb = new WeakMap();
 // item_ref is the sole durable conflict target. `content_hash` COALESCEs so a
@@ -107,6 +88,7 @@ function getUpsertStmts(db) {
       ON CONFLICT(item_ref) DO UPDATE ${UPSERT_SET_CLAUSE}
       RETURNING id
     `),
+        findByItemRef: db.prepare("SELECT id, search_text FROM entries WHERE item_ref = ?"),
     };
     upsertStmtsByDb.set(db, stmts);
     return stmts;
@@ -182,18 +164,16 @@ export function getBaseBeliefStatesForDerivedTwins(db, twinIds) {
 /**
  * Re-key an entries row in place for the opt-in source-maintenance script.
  *
- * The row id is preserved on purpose — `utility_scores` and
- * `utility_scores_scoped` are keyed by `entry_id`, so an UPDATE (rather than
- * a delete + insert under the new `item_ref`) is what keeps the asset's
- * accumulated usage-ranking history attached across a rename.
- * (`asset_salience` / `asset_outcome` live in state.db keyed by `asset_ref`
- * TEXT and are re-keyed separately by `akm mv` — see the state rekey
- * helper.) `document_json.name` (and `filename`, when present) is patched
- * and `search_text` rebuilt so search reflects the new name. Its
- * safe-Markdown fragment source is updated in the same transaction as the
- * canonical identity; the new `search_text` needs no explicit vector
- * invalidation — content-addressed units simply derive new hashes on the
- * next reconcile.
+ * The row id is preserved on purpose — `utility_scores`,
+ * `utility_scores_scoped`, and `embeddings` are keyed by `entry_id`, so an
+ * UPDATE (rather than a delete + insert under the new `item_ref`) is what
+ * keeps the asset's accumulated usage-ranking history attached across a
+ * rename. (`asset_salience` / `asset_outcome` live in state.db keyed by
+ * `asset_ref` TEXT and are re-keyed separately by `akm mv` — see
+ * the state rekey helper.) `document_json.name` (and `filename`, when
+ * present) is patched and `search_text` rebuilt so search reflects the new
+ * name. Its FTS projection and stale vector are updated in the same
+ * transaction as the canonical identity.
  *
  * Bundle-qualified `usage_events.entry_ref` rows for the old conceptId are
  * rewritten to the new item ref. Without this, events keep the old
@@ -207,8 +187,10 @@ export function getBaseBeliefStatesForDerivedTwins(db, twinIds) {
  * A stale row already occupying the new item ref (the caller has verified no
  * FILE exists at the target, so such a row can only be a leftover for a
  * deleted file) is evicted first — through {@link deleteRelatedRows}, so its
- * child rows (utility scores, usage events) go with it. The moved row keeps
- * its id.
+ * child rows (embeddings, entries_vec, utility scores, usage events) go with
+ * it. A bare `DELETE FROM entries` would trip the non-CASCADE `embeddings`
+ * FK under `PRAGMA foreign_keys = ON` and roll back the whole re-key.
+ * The moved row keeps its id.
  *
  * Returns the surviving row id, or `null` when no row matches the old item ref
  * (nothing indexed under the old name — the caller falls open and the next
@@ -250,12 +232,14 @@ export function rekeyEntryInPlace(db, opts) {
         throw new Error(`Refusing to re-key entry ${oldItemRef}: target ref does not match the entry type and name.`);
     }
     const newItemRef = `${opts.sourceName}//${opts.newRef}`;
-    withImmediateTransaction(db, () => {
+    db.transaction(() => {
         const stale = db.prepare("SELECT id FROM entries WHERE item_ref = ?").get(newItemRef);
         if (stale && stale.id !== row.id) {
-            // Full child-row cleanup (utility scores, usage events, fragment
-            // source) BEFORE the entries delete — the FK-less child rows would
-            // otherwise orphan permanently.
+            // Full child-row cleanup (embeddings, entries_vec, utility scores,
+            // usage events, FTS + dirty marks) BEFORE the entries delete: the
+            // `embeddings` FK is non-CASCADE and `foreign_keys = ON`, so a bare
+            // entries delete would throw and roll back the entire re-key; and
+            // without it the FK-less child rows would orphan permanently.
             deleteRelatedRows(db, [{ id: stale.id }]);
             db.prepare("DELETE FROM entries WHERE id = ?").run(stale.id);
         }
@@ -263,11 +247,13 @@ export function rekeyEntryInPlace(db, opts) {
         if (opts.newDerivedFrom !== undefined) {
             db.prepare("UPDATE entries SET derived_from = ? WHERE id = ?").run(opts.newDerivedFrom, row.id);
         }
+        if (row.search_text !== searchText)
+            deleteEntryVectors(db, row.id);
         if (document)
-            replaceFragmentSource(db, row.id, hasMarkdownFragmentContent(document) ? (getMarkdownFragmentContent(document) ?? null) : undefined);
+            replaceFtsEntry(db, row.id, document, hasMarkdownFragmentContent(document) ? (getMarkdownFragmentContent(document) ?? null) : undefined);
         else
-            deleteFragmentSource(db, [row.id]);
-    }, "index");
+            deleteFtsEntries(db, [row.id]);
+    })();
     // Re-point usage history at the new ref. Chunk-8 WI-8.3: usage_events lives in
     // state.db now, so this is a SEPARATE cross-DB transaction (best-effort — the
     // rename itself already committed above; on failure the next full index's
@@ -297,9 +283,9 @@ function rewriteUsageEventRefForMove(opts) {
     };
     try {
         withStateDb((stateDb) => {
-            withImmediateTransaction(stateDb, () => {
+            stateDb.transaction(() => {
                 rename(stateDb, `${opts.sourceName}//${opts.oldRef}`, `${opts.sourceName}//${opts.newRef}`);
-            });
+            })();
         });
     }
     catch (error) {
@@ -386,14 +372,14 @@ function deleteEntryRows(db, rows, options = {}) {
     return rows.map((row) => row.id);
 }
 export function deleteEntriesByDirAndBundle(db, dirPath, bundleId, options = {}) {
-    return withImmediateTransaction(db, () => deleteEntryRows(db, rowsInDirectory(db, dirPath, bundleId), options), "index");
+    return db.transaction(() => deleteEntryRows(db, rowsInDirectory(db, dirPath, bundleId), options))();
 }
 /** Delete every entry and child row belonging to one canonical bundle. */
 export function deleteEntriesByBundle(db, bundleId) {
-    withImmediateTransaction(db, () => {
+    db.transaction(() => {
         const rows = db.prepare("SELECT id FROM entries WHERE bundle_id = ?").all(bundleId);
         deleteEntryRows(db, rows);
-    }, "index");
+    })();
 }
 /**
  * Delete the complete regenerable entry generation through the same child-row
@@ -401,10 +387,10 @@ export function deleteEntriesByBundle(db, bundleId) {
  * usage events so the finalize pass can relink them to the new row ids.
  */
 export function deleteAllEntries(db, options = {}) {
-    return withImmediateTransaction(db, () => {
+    return db.transaction(() => {
         const rows = db.prepare("SELECT id FROM entries").all();
         return deleteEntryRows(db, rows, options);
-    }, "index");
+    })();
 }
 /**
  * Diff-persist orphan delete: remove every entry under `dirPath` whose durable
@@ -412,7 +398,7 @@ export function deleteAllEntries(db, options = {}) {
  *
  * Replaces the old per-dir `deleteEntriesByDir` + full re-insert: the caller
  * upserts the current file set first (ON CONFLICT preserving `entries.id`, so
- * utility / usage stay attached to unchanged rows), then calls this
+ * embeddings / utility / usage stay attached to unchanged rows), then calls this
  * to prune only the departed rows. The net row-state for the directory is identical
  * to delete-then-reinsert; the win is that unchanged rows keep their id.
  *
@@ -420,25 +406,27 @@ export function deleteAllEntries(db, options = {}) {
  * prune one another's rows.
  */
 export function deleteEntriesByDirExceptRefs(db, dirPath, bundleId, keepRefs, options = {}) {
-    return withImmediateTransaction(db, () => {
+    return db.transaction(() => {
         const doomed = rowsInDirectory(db, dirPath, bundleId).filter((row) => !keepRefs.has(row.item_ref));
         return deleteEntryRows(db, doomed, options);
-    }, "index");
+    })();
 }
 function deleteRelatedRows(db, ids, options = {}) {
     if (ids.length === 0)
         return;
     const numericIds = ids.map((r) => r.id);
-    // The safe-Markdown fragment source is part of the canonical mutation
-    // boundary, not a caller-maintained dirty queue. Delete it before the
-    // parent row inside this transaction (redundant with entry_fragments' own
-    // ON DELETE CASCADE, but explicit here alongside the other child-row
-    // cleanup this function owns).
-    deleteFragmentSource(db, numericIds);
+    const vecAvail = isVecAvailable(db);
+    // FTS is part of the canonical mutation boundary, not a caller-maintained
+    // dirty queue. Delete it before the parent row inside this transaction.
+    deleteFtsEntries(db, numericIds);
     // Process in chunks to stay within SQLITE_MAX_VARIABLE_NUMBER
     for (let i = 0; i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
         const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
         const placeholders = chunk.map(() => "?").join(",");
+        bestEffort(() => db.prepare(`DELETE FROM embeddings WHERE id IN (${placeholders})`).run(...chunk), "delete embeddings for entries");
+        if (vecAvail) {
+            bestEffort(() => db.prepare(`DELETE FROM entries_vec WHERE id IN (${placeholders})`).run(...chunk), "delete entries_vec for entries");
+        }
         // Clean up utility scores before deleting entries
         bestEffort(() => db.prepare(`DELETE FROM utility_scores WHERE entry_id IN (${placeholders})`).run(...chunk), "delete utility_scores for entries");
         bestEffort(() => db.prepare(`DELETE FROM utility_scores_scoped WHERE entry_id IN (${placeholders})`).run(...chunk), "delete utility_scores_scoped for entries");
@@ -493,14 +481,15 @@ export function deleteUsageEventsByEntryIds(entryIds) {
 }
 /**
  * Delete entries by their primary key IDs, along with all related rows
- * (entry_fragments, entry_units, utility scores, usage_events).
+ * (embeddings, entries_vec, entries_fts, utility scores, usage_events).
  *
- * Used by `reconcile.ts` to remove entries whose source files are gone.
+ * Used by explicit `--clean` reconciliation before embeddings and final
+ * verification to remove stale entries whose source files no longer exist.
  */
 export function deleteEntriesByIds(db, ids) {
     if (ids.length === 0)
         return;
-    withImmediateTransaction(db, () => {
+    db.transaction(() => {
         const idObjs = ids.map((id) => ({ id }));
         deleteRelatedRows(db, idObjs);
         for (let i = 0; i < ids.length; i += SQLITE_CHUNK_SIZE) {
@@ -508,7 +497,7 @@ export function deleteEntriesByIds(db, ids) {
             const placeholders = chunk.map(() => "?").join(",");
             db.prepare(`DELETE FROM entries WHERE id IN (${placeholders})`).run(...chunk);
         }
-    }, "index");
+    })();
 }
 // ── All entries ─────────────────────────────────────────────────────────────
 function parseEntryRows(rows, context) {
@@ -803,10 +792,11 @@ export function relinkUsageEvents(indexDb, stateDb, options = {}) {
         });
         if (staleLinks.length > 0) {
             const nullOut = stateDb.prepare(`UPDATE ${usageEvents} SET entry_id = NULL WHERE entry_id = ? AND entry_ref IS ?`);
-            withImmediateTransaction(stateDb, () => {
+            const nullTx = stateDb.transaction(() => {
                 for (const { id, ref } of staleLinks)
                     nullOut.run(id, ref);
             });
+            nullTx();
         }
         // Step 2: re-resolve each fully-qualified ref. Bare rows are not current
         // durable identities and remain detached.
@@ -814,7 +804,7 @@ export function relinkUsageEvents(indexDb, stateDb, options = {}) {
             .prepare(`SELECT DISTINCT entry_ref AS ref FROM ${usageEvents} WHERE entry_id IS NULL AND entry_ref IS NOT NULL`)
             .all();
         const update = stateDb.prepare(`UPDATE ${usageEvents} SET entry_id = ? WHERE entry_ref = ? AND entry_id IS NULL`);
-        withImmediateTransaction(stateDb, () => {
+        const relinkTx = stateDb.transaction(() => {
             for (const { ref } of refs) {
                 let id;
                 try {
@@ -829,5 +819,6 @@ export function relinkUsageEvents(indexDb, stateDb, options = {}) {
                     update.run(id, ref);
             }
         });
+        relinkTx();
     }, "usage_events table may not exist yet during entry_id re-resolution");
 }

package/dist/storage/repositories/index-entry-schema.js

@@ -12,18 +12,7 @@
  */
 // v23 adds an isolated fragment FTS population. v22 is the last shipped
 // generation and is intentionally rebuilt rather than migrated in place.
-//
-// v23→v24 (index-redesign B5c): `entries_fts` and `entry_fragments_fts` are
-// dropped from the canonical shape. Lexical search runs entirely over
-// `units_fts` now (index-redesign-contract.md B1/B3) — the card unit already
-// carries name/description/tags/hints, so an entry-level lexical query is a
-// units query grouped by entry, and a fragment-level lexical query is the
-// same `units_fts` table filtered to fragment-kind units. `entry_fragments`
-// (the safe-rendered Markdown source, NOT an FTS index) stays: it is what a
-// matched fragment hit's display metadata is projected from, and what `akm
-// show <ref>#<fragmentId>` resolves an opaque fragment selector through —
-// both are consumers independent of which table search itself queries.
-export const CANONICAL_INDEX_DB_VERSION = 24;
+export const CANONICAL_INDEX_DB_VERSION = 23;
 export const CANONICAL_ENTRY_SCHEMA_SQL = `
   CREATE TABLE IF NOT EXISTS entries (
     id            INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -45,14 +34,30 @@ export const CANONICAL_ENTRY_SCHEMA_SQL = `
   CREATE INDEX IF NOT EXISTS idx_entries_file_path ON entries(file_path);
   CREATE INDEX IF NOT EXISTS idx_entries_derived_from ON entries(derived_from);
 
-  -- The safe-rendered Markdown source a matched fragment hit's display
-  -- metadata is projected from (index-fts-repository.ts's
-  -- getIndexedMarkdownFragment(s)) and that akm show's opaque fragment
-  -- selectors resolve through. Not a search index — units_fts is.
+  -- Keep parent metadata and body fragments in separate FTS populations.
+  -- Combining them changes parent-document IDF and conjunction semantics.
+  CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(
+    entry_id UNINDEXED,
+    name,
+    description,
+    tags,
+    hints,
+    content,
+    tokenize='porter unicode61'
+  );
+
   CREATE TABLE IF NOT EXISTS entry_fragments (
     entry_id INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE,
     safe_markdown TEXT NOT NULL
   );
+
+  CREATE VIRTUAL TABLE IF NOT EXISTS entry_fragments_fts USING fts5(
+    entry_id UNINDEXED,
+    fragment_id UNINDEXED,
+    fragment_ordinal UNINDEXED,
+    content,
+    tokenize='porter unicode61'
+  );
 `;
 const CANONICAL_ENTRY_SCHEMA_FINGERPRINT = {
     tableSql: "CREATE TABLE entries ( id INTEGER PRIMARY KEY AUTOINCREMENT, item_ref TEXT NOT NULL UNIQUE, bundle_id TEXT NOT NULL, component_id TEXT NOT NULL, concept_id TEXT NOT NULL, adapter_id TEXT NOT NULL, type TEXT NOT NULL, file_path TEXT NOT NULL, content_hash TEXT, document_json TEXT NOT NULL, search_text TEXT NOT NULL, derived_from TEXT )",
@@ -205,7 +210,9 @@ const CANONICAL_ENTRY_SCHEMA_FINGERPRINT = {
         },
     ],
     searchSurfaces: {
+        entriesFtsSql: "CREATE VIRTUAL TABLE entries_fts USING fts5( entry_id UNINDEXED, name, description, tags, hints, content, tokenize='porter unicode61' )",
         fragmentSourceSql: "CREATE TABLE entry_fragments ( entry_id INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE, safe_markdown TEXT NOT NULL )",
+        fragmentsFtsSql: "CREATE VIRTUAL TABLE entry_fragments_fts USING fts5( entry_id UNINDEXED, fragment_id UNINDEXED, fragment_ordinal UNINDEXED, content, tokenize='porter unicode61' )",
     },
 };
 function sqlString(value) {
@@ -263,7 +270,9 @@ export function readEntrySchemaFingerprint(db) {
         columns,
         indexes,
         searchSurfaces: {
+            entriesFtsSql: readNamedTableSql(db, "entries_fts"),
             fragmentSourceSql: readNamedTableSql(db, "entry_fragments"),
+            fragmentsFtsSql: readNamedTableSql(db, "entry_fragments_fts"),
         },
     };
 }