akm-cli 0.9.12 → 0.9.14-beta.1

package/dist/storage/repositories/index-connection.js

@@ -126,23 +126,38 @@ export function openExistingDatabase(dbPath) {
     if (classifyPathAccess(resolvedPath).access === "absent") {
         throw new Error(`Index database not found at ${resolvedPath}. Run 'akm index' to build it.`);
     }
-    return openManagedDatabase({
+    const db = openManagedDatabase({
         path: resolvedPath,
         init: (db) => {
             loadVecExtension(db);
-            warnIfNonCanonicalIndexGeneration(db, resolvedPath);
         },
         create: false,
     });
+    try {
+        assertCanonicalIndexGeneration(db, resolvedPath);
+        return db;
+    }
+    catch (error) {
+        db.close();
+        throw error;
+    }
 }
-function warnIfNonCanonicalIndexGeneration(db, resolvedPath) {
+/**
+ * Read callers must never receive a known-incompatible derived index.  The
+ * writable opener owns rebuilding an older generation; a reader can only
+ * report the one action that is safe for the direction of the mismatch.
+ */
+function assertCanonicalIndexGeneration(db, resolvedPath) {
     if (isCanonicalIndexGeneration(db))
         return;
     const classification = classifyIndexGeneration(db);
-    warnOnce(`index-read-noncanonical:${resolvedPath}`, `Index database at ${resolvedPath} does not match this akm's derived schema (stored generation ` +
-        `${classification.storedVersion ?? "unknown"}; this binary understands ${CANONICAL_INDEX_DB_VERSION}). ` +
-        "Reading it as-is; a query that needs a table or column this generation lacks will fail on its own. " +
-        "Run 'akm index' to rebuild it for this binary.");
+    const stored = classification.storedVersion ?? "unknown";
+    if (classification.status === "newer") {
+        throw new ConfigError(`Index database at ${resolvedPath} was built by a newer akm (stored generation ${stored}; ` +
+            `this binary understands ${CANONICAL_INDEX_DB_VERSION}). Upgrade akm to use this index.`, "INDEX_SCHEMA_INCOMPATIBLE", "Upgrade akm to a version that understands this index generation.");
+    }
+    throw new ConfigError(`Index database at ${resolvedPath} is not usable with this akm's derived schema (stored generation ${stored}; ` +
+        `this binary understands ${CANONICAL_INDEX_DB_VERSION}). Run 'akm index' to rebuild it.`, "INDEX_SCHEMA_INCOMPATIBLE", "Run `akm index` to rebuild the derived index from the currently materialized sources.");
 }
 /**
  * Refuse to treat an UNREADABLE index as a missing one (#791).
@@ -204,7 +219,7 @@ export function openReadonlyExistingDatabase(dbPath, options) {
     // connection, so apply just that one.
     try {
         db.exec(`PRAGMA busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`);
-        warnIfNonCanonicalIndexGeneration(db, resolvedPath);
+        assertCanonicalIndexGeneration(db, resolvedPath);
         return db;
     }
     catch (error) {

package/dist/storage/repositories/index-entries-repository.js

@@ -16,6 +16,7 @@ import { bestEffort } from "../../core/best-effort.js";
 import { isPathAbsent } from "../../core/path-access.js";
 import { getStateDbPath, withStateDb } from "../../core/state-db.js";
 import { warn } from "../../core/warn.js";
+import { getMarkdownFragmentContent, hasMarkdownFragmentContent, } from "../../indexer/passes/metadata.js";
 import { buildSearchText } from "../../indexer/search/search-fields.js";
 import { ENTRY_COLUMNS, rowToIndexedEntry } from "./index-entry-mapper.js";
 import { deleteFtsEntries, replaceFtsEntry } from "./index-fts-repository.js";
@@ -47,7 +48,7 @@ export function upsertEntry(db, filePath, entry, searchText, provenance, content
             throw new Error("upsertEntry: item_ref not found after upsert");
         if (previous?.id === result.id && previous.search_text !== searchText)
             deleteEntryVectors(db, result.id);
-        replaceFtsEntry(db, result.id, entry);
+        replaceFtsEntry(db, result.id, entry, hasMarkdownFragmentContent(entry) ? (getMarkdownFragmentContent(entry) ?? null) : undefined);
         return result.id;
     };
     // Always enter the driver's transaction wrapper. Both supported SQLite
@@ -249,7 +250,7 @@ export function rekeyEntryInPlace(db, opts) {
         if (row.search_text !== searchText)
             deleteEntryVectors(db, row.id);
         if (document)
-            replaceFtsEntry(db, row.id, document);
+            replaceFtsEntry(db, row.id, document, hasMarkdownFragmentContent(document) ? (getMarkdownFragmentContent(document) ?? null) : undefined);
         else
             deleteFtsEntries(db, [row.id]);
     })();

package/dist/storage/repositories/index-entry-schema.js

@@ -10,7 +10,9 @@
  * serving preflights, and read-only evaluator tooling cannot drift into
  * separate definitions of "current".
  */
-export const CANONICAL_INDEX_DB_VERSION = 22;
+// v23 adds an isolated fragment FTS population. v22 is the last shipped
+// generation and is intentionally rebuilt rather than migrated in place.
+export const CANONICAL_INDEX_DB_VERSION = 23;
 export const CANONICAL_ENTRY_SCHEMA_SQL = `
   CREATE TABLE IF NOT EXISTS entries (
     id            INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -31,6 +33,31 @@ export const CANONICAL_ENTRY_SCHEMA_SQL = `
   CREATE INDEX IF NOT EXISTS idx_entries_type ON entries(type);
   CREATE INDEX IF NOT EXISTS idx_entries_file_path ON entries(file_path);
   CREATE INDEX IF NOT EXISTS idx_entries_derived_from ON entries(derived_from);
+
+  -- Keep parent metadata and body fragments in separate FTS populations.
+  -- Combining them changes parent-document IDF and conjunction semantics.
+  CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(
+    entry_id UNINDEXED,
+    name,
+    description,
+    tags,
+    hints,
+    content,
+    tokenize='porter unicode61'
+  );
+
+  CREATE TABLE IF NOT EXISTS entry_fragments (
+    entry_id INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE,
+    safe_markdown TEXT NOT NULL
+  );
+
+  CREATE VIRTUAL TABLE IF NOT EXISTS entry_fragments_fts USING fts5(
+    entry_id UNINDEXED,
+    fragment_id UNINDEXED,
+    fragment_ordinal UNINDEXED,
+    content,
+    tokenize='porter unicode61'
+  );
 `;
 const CANONICAL_ENTRY_SCHEMA_FINGERPRINT = {
     tableSql: "CREATE TABLE entries ( id INTEGER PRIMARY KEY AUTOINCREMENT, item_ref TEXT NOT NULL UNIQUE, bundle_id TEXT NOT NULL, component_id TEXT NOT NULL, concept_id TEXT NOT NULL, adapter_id TEXT NOT NULL, type TEXT NOT NULL, file_path TEXT NOT NULL, content_hash TEXT, document_json TEXT NOT NULL, search_text TEXT NOT NULL, derived_from TEXT )",
@@ -182,6 +209,11 @@ const CANONICAL_ENTRY_SCHEMA_FINGERPRINT = {
             ],
         },
     ],
+    searchSurfaces: {
+        entriesFtsSql: "CREATE VIRTUAL TABLE entries_fts USING fts5( entry_id UNINDEXED, name, description, tags, hints, content, tokenize='porter unicode61' )",
+        fragmentSourceSql: "CREATE TABLE entry_fragments ( entry_id INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE, safe_markdown TEXT NOT NULL )",
+        fragmentsFtsSql: "CREATE VIRTUAL TABLE entry_fragments_fts USING fts5( entry_id UNINDEXED, fragment_id UNINDEXED, fragment_ordinal UNINDEXED, content, tokenize='porter unicode61' )",
+    },
 };
 function sqlString(value) {
     return `'${value.replaceAll("'", "''")}'`;
@@ -191,8 +223,11 @@ function normalizeSchemaSql(value) {
         return null;
     return value.replace(/\s+/g, " ").trim();
 }
+function readNamedTableSql(db, name) {
+    const row = db.prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ${sqlString(name)}`).get();
+    return normalizeSchemaSql(row?.sql);
+}
 export function readEntrySchemaFingerprint(db) {
-    const tableRow = db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'entries'").get();
     const sqliteSequenceTable = db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'sqlite_sequence'").get() !=
         null;
     const maxId = Number(db.prepare("SELECT COALESCE(MAX(id), 0) AS maxId FROM entries").get().maxId);
@@ -229,11 +264,16 @@ export function readEntrySchemaFingerprint(db) {
     }))
         .sort((left, right) => left.name.localeCompare(right.name));
     return {
-        tableSql: normalizeSchemaSql(tableRow?.sql),
+        tableSql: readNamedTableSql(db, "entries"),
         sqliteSequenceTable,
         sqliteSequenceValid,
         columns,
         indexes,
+        searchSurfaces: {
+            entriesFtsSql: readNamedTableSql(db, "entries_fts"),
+            fragmentSourceSql: readNamedTableSql(db, "entry_fragments"),
+            fragmentsFtsSql: readNamedTableSql(db, "entry_fragments_fts"),
+        },
     };
 }
 export function hasCanonicalEntrySchema(db) {

package/dist/storage/repositories/index-fts-repository.js

@@ -7,11 +7,14 @@
  * Owns the `entries_fts` full-text query path, per-entry projections, and the
  * explicit full recovery rebuild.
  */
+import { fragmentForSelector, splitMarkdownFragments } from "../../core/asset/markdown-fragments.js";
+import { stableFtsScore } from "../../core/lexical-score.js";
 import { warn } from "../../core/warn.js";
 import { buildLexicalQueryPlan } from "../../indexer/search/fts-query.js";
 import { buildSearchFields } from "../../indexer/search/search-fields.js";
 import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
 const INSERT_FTS_SQL = "INSERT INTO entries_fts (entry_id, name, description, tags, hints, content) VALUES (?, ?, ?, ?, ?, ?)";
+const INSERT_FRAGMENT_SQL = "INSERT INTO entry_fragments_fts (entry_id, fragment_id, fragment_ordinal, content) VALUES (?, ?, ?, ?)";
 const ftsMutationStatementsByDb = new WeakMap();
 function getFtsMutationStatements(db) {
     const existing = ftsMutationStatementsByDb.get(db);
@@ -20,16 +23,34 @@ function getFtsMutationStatements(db) {
     const statements = {
         deleteOne: db.prepare("DELETE FROM entries_fts WHERE entry_id = ?"),
         insert: db.prepare(INSERT_FTS_SQL),
+        deleteFragments: db.prepare("DELETE FROM entry_fragments_fts WHERE entry_id = ?"),
+        upsertFragmentSource: db.prepare("INSERT INTO entry_fragments (entry_id, safe_markdown) VALUES (?, ?) ON CONFLICT(entry_id) DO UPDATE SET safe_markdown = excluded.safe_markdown"),
+        deleteFragmentSource: db.prepare("DELETE FROM entry_fragments WHERE entry_id = ?"),
+        insertFragment: db.prepare(INSERT_FRAGMENT_SQL),
     };
     ftsMutationStatementsByDb.set(db, statements);
     return statements;
 }
 /** Replace one entry's derived FTS projection inside the caller's transaction. */
-export function replaceFtsEntry(db, entryId, entry) {
+export function replaceFtsEntry(db, entryId, entry, fragmentContent) {
     const fields = buildSearchFields(entry);
     const statements = getFtsMutationStatements(db);
     statements.deleteOne.run(entryId);
     statements.insert.run(entryId, fields.name, fields.description, fields.tags, fields.hints, fields.content);
+    if (fragmentContent === undefined) {
+        // Metadata-only re-upserts and re-keys deserialize the public document
+        // without the internal substrate. Leave the persisted source untouched.
+        // A scan that did read Markdown always supplies a value below.
+        return;
+    }
+    statements.deleteFragments.run(entryId);
+    statements.deleteFragmentSource.run(entryId);
+    if (!fragmentContent)
+        return;
+    statements.upsertFragmentSource.run(entryId, fragmentContent);
+    for (const fragment of splitMarkdownFragments(fragmentContent)) {
+        statements.insertFragment.run(entryId, fragment.fragmentId, fragment.ordinal, fragment.text.toLowerCase());
+    }
 }
 /** Delete derived FTS projections for canonical entries that are being removed. */
 export function deleteFtsEntries(db, entryIds) {
@@ -37,6 +58,8 @@ export function deleteFtsEntries(db, entryIds) {
         const chunk = entryIds.slice(i, i + SQLITE_CHUNK_SIZE);
         const placeholders = chunk.map(() => "?").join(",");
         db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
+        db.prepare(`DELETE FROM entry_fragments_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
+        db.prepare(`DELETE FROM entry_fragments WHERE entry_id IN (${placeholders})`).run(...chunk);
     }
 }
 export function searchFts(db, query, limit, entryType, excludeTypes) {
@@ -58,41 +81,92 @@ export function searchFts(db, query, limit, entryType, excludeTypes) {
     // input whose filler terms prevented a strict hit.
     return plan.relaxed ? runFtsQuery(db, plan.relaxed, "relaxed", limit, entryType, excludeTypes) : [];
 }
+/**
+ * Resolve an opaque fragment selector from the indexed safe projection, not
+ * from current disk. A search result remains self-consistent across a later
+ * file edit; the next index refresh atomically publishes the new revision.
+ */
+export function getIndexedMarkdownFragment(db, itemRef, fragmentId) {
+    const row = db
+        .prepare("SELECT s.safe_markdown FROM entry_fragments_fts f JOIN entry_fragments s ON s.entry_id = f.entry_id JOIN entries e ON e.id = f.entry_id WHERE e.item_ref = ? AND f.fragment_id = ?")
+        .get(itemRef, fragmentId);
+    const fragment = row ? fragmentForSelector(row.safe_markdown, fragmentId) : undefined;
+    return fragment ? { content: fragment.text } : undefined;
+}
 function runFtsQuery(db, ftsQuery, lexicalMatch, limit, entryType, excludeTypes) {
+    // Preserve the repository's ordinary limit contract for direct callers.
+    // The boundary-expansion rule applies only to a positive candidate pool.
+    if (limit <= 0)
+        return [];
     // #627 — exclude-type clause. Only applies on the untyped ('any') path; an
     // explicit include filter (entryType) already narrows to a single type, so
     // exclusion is redundant there. An empty list skips the clause entirely
     // (never emit `NOT IN ()`, which is a SQL error / always-false).
     const excludes = excludeTypes && excludeTypes.length > 0 ? excludeTypes : [];
+    const candidateBoundaryOffset = Math.max(0, limit - 1);
     // The typed and untyped paths differ only by one `type` WHERE clause
-    // equality vs. an optional NOT IN exclusion) and their param order — the
-    // SELECT/JOIN/ORDER/LIMIT is shared, so build it once. Join on integer
-    // entry_id directly (no CAST; we store integer). bm25() per-column weights:
+    // equality vs. an optional NOT IN exclusion) and their parameter order.
+    // Join on integer entry_id directly (no CAST; we store integer). bm25()
+    // per-column weights:
     // entry_id(0), name(10), description(5), tags(3), hints(2), content(1).
     let filterClause;
     let params;
     if (entryType && entryType !== "any") {
         filterClause = "AND e.type = ?";
-        params = [ftsQuery, entryType, limit];
+        params = [ftsQuery, entryType, candidateBoundaryOffset];
     }
     else {
         filterClause = excludes.length > 0 ? `AND e.type NOT IN (${excludes.map(() => "?").join(", ")})` : "";
-        // Param order: MATCH, then the NOT IN values, then LIMIT.
-        params = [ftsQuery, ...excludes, limit];
+        // Param order: MATCH, then the NOT IN values, then the zero-based
+        // candidate-boundary offset.
+        params = [ftsQuery, ...excludes, candidateBoundaryOffset];
     }
     const sql = `
-    SELECT e.id, e.file_path AS filePath, e.document_json AS documentJson, e.search_text AS searchText,
-           e.item_ref AS itemRef, e.bundle_id AS bundleId, e.concept_id AS conceptId, e.adapter_id AS adapterId,
-           bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
+    -- Do not make a SQL-only relevance decision inside a tied BM25 boundary:
+    -- the TypeScript ranker adds exact-name, type, and other contributors
+    -- afterwards.  Materialize BM25 once, locate the Nth score, and admit
+    -- every row tied with it.  This deliberately makes the result set
+    -- data-bound for a pathological all-tied query; that is the only way to
+    -- avoid silently dropping a legitimate later ranking winner.
+    WITH scored AS MATERIALIZED (
+      -- Keep this materialized set deliberately narrow. document_json can be
+      -- large, and only rows admitted through the BM25 boundary need it.
+      SELECT e.id, bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
     FROM entries_fts f
     JOIN entries e ON e.id = f.entry_id
     WHERE entries_fts MATCH ?
       ${filterClause}
-    ORDER BY bm25Score, e.id ASC
-    LIMIT ?
+    ), boundary AS (
+      SELECT bm25Score
+      FROM scored
+      ORDER BY bm25Score
+      LIMIT 1 OFFSET ?
+    )
+    SELECT e.id, e.file_path AS filePath, e.document_json AS documentJson, e.search_text AS searchText,
+           e.item_ref AS itemRef, e.bundle_id AS bundleId, e.concept_id AS conceptId, e.adapter_id AS adapterId,
+           scored.bm25Score
+    FROM scored
+    JOIN entries e ON e.id = scored.id
+    WHERE NOT EXISTS (SELECT 1 FROM boundary)
+       OR scored.bm25Score <= (SELECT bm25Score FROM boundary)
+    ORDER BY scored.bm25Score, e.id ASC
   `;
     const rows = db.prepare(sql).all(...params);
-    // Guard against corrupt JSON — skip the row rather than crashing
+    const results = materializeRows(rows, lexicalMatch);
+    // Fragments are a separate, intentionally calibrated evidence population:
+    // parent FTS remains the sole implementation of metadata/body conjunction.
+    // A selector is emitted only for one fragment that independently satisfies
+    // this query. Raw BM25 values are never claimed comparable across tables;
+    // each is passed through #933's stable mapping before merge.
+    const fragmentResults = hasFragmentFts(db)
+        ? runFragmentQuery(db, ftsQuery, lexicalMatch, limit, entryType, excludes)
+        : [];
+    return mergeParentAndFragmentResults(results, fragmentResults);
+}
+function hasFragmentFts(db) {
+    return Boolean(db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'entry_fragments_fts'").get());
+}
+function materializeRows(rows, lexicalMatch) {
     const results = [];
     for (const row of rows) {
         let entry;
@@ -118,6 +192,69 @@ function runFtsQuery(db, ftsQuery, lexicalMatch, limit, entryType, excludeTypes)
     }
     return results;
 }
+function runFragmentQuery(db, ftsQuery, lexicalMatch, limit, entryType, excludes) {
+    const filter = entryType && entryType !== "any"
+        ? "AND e.type = ?"
+        : excludes.length
+            ? `AND e.type NOT IN (${excludes.map(() => "?").join(",")})`
+            : "";
+    const filterParams = entryType && entryType !== "any" ? [entryType] : excludes;
+    const candidateBoundaryOffset = Math.max(0, limit - 1);
+    // Select the winning child per parent inside SQLite before finding the
+    // candidate boundary. A document with many matching fragments therefore
+    // occupies one parent slot, while a boundary tie retains every parent for
+    // the TypeScript ranker to decide with its non-BM25 contributors. This has
+    // one FTS query and no OFFSET walk; the returned boundary is intentionally
+    // data-bound for a pathological all-tied query, just like parent FTS.
+    const sql = `
+    WITH matches AS MATERIALIZED (
+      -- Keep repeated child rows as narrow as parent FTS's scored CTE. The
+      -- document projection can be large; hydrate it only after the one-child
+      -- per-parent collapse and BM25 boundary filtering below.
+      SELECT e.id, f.fragment_id AS fragmentId, f.fragment_ordinal AS fragmentOrdinal,
+             bm25(entry_fragments_fts) AS bm25Score
+      FROM entry_fragments_fts f JOIN entries e ON e.id = f.entry_id
+      WHERE entry_fragments_fts MATCH ? ${filter}
+    ), ranked AS MATERIALIZED (
+      SELECT *, ROW_NUMBER() OVER (PARTITION BY id ORDER BY bm25Score ASC, fragmentOrdinal ASC, fragmentId ASC) AS parentRank
+      FROM matches
+    ), parents AS MATERIALIZED (
+      SELECT * FROM ranked WHERE parentRank = 1
+    ), boundary AS (
+      SELECT bm25Score FROM parents ORDER BY bm25Score ASC LIMIT 1 OFFSET ?
+    )
+    SELECT e.id, e.file_path AS filePath, e.document_json AS documentJson, e.search_text AS searchText,
+           e.item_ref AS itemRef, e.bundle_id AS bundleId, e.concept_id AS conceptId, e.adapter_id AS adapterId,
+           parents.fragmentId, parents.bm25Score
+    FROM parents JOIN entries e ON e.id = parents.id
+    WHERE NOT EXISTS (SELECT 1 FROM boundary)
+       OR parents.bm25Score <= (SELECT bm25Score FROM boundary)
+    ORDER BY parents.bm25Score ASC, parents.id ASC`;
+    const rows = db.prepare(sql).all(ftsQuery, ...filterParams, candidateBoundaryOffset);
+    const results = [];
+    for (const row of rows) {
+        const [result] = materializeRows([row], lexicalMatch);
+        if (result) {
+            results.push({
+                ...result,
+                fragmentId: row.fragmentId,
+                lexicalScore: stableFtsScore(result.bm25Score, "fragment"),
+            });
+        }
+    }
+    return results;
+}
+function mergeParentAndFragmentResults(parents, fragments) {
+    const winners = new Map();
+    for (const parent of parents)
+        winners.set(parent.id, { ...parent, lexicalScore: stableFtsScore(parent.bm25Score) });
+    for (const fragment of fragments) {
+        const existing = winners.get(fragment.id);
+        if (!existing || (fragment.lexicalScore ?? 0) > (existing.lexicalScore ?? 0))
+            winners.set(fragment.id, fragment);
+    }
+    return [...winners.values()].sort((left, right) => (right.lexicalScore ?? 0) - (left.lexicalScore ?? 0) || left.id - right.id);
+}
 /**
  * Explicitly rebuild the complete FTS5 projection from canonical entries.
  * Ordinary entry mutations do not call this: `upsertEntry` and the delete
@@ -131,8 +268,12 @@ function runFtsQuery(db, ftsQuery, lexicalMatch, limit, entryType, excludeTypes)
 export function rebuildFts(db) {
     db.transaction(() => {
         db.exec("DELETE FROM entries_fts");
-        const rows = db.prepare("SELECT id, document_json FROM entries").all();
+        db.exec("DELETE FROM entry_fragments_fts");
+        const rows = db
+            .prepare("SELECT e.id, e.document_json, f.safe_markdown FROM entries e LEFT JOIN entry_fragments f ON f.entry_id = e.id")
+            .all();
         const insertStmt = db.prepare(INSERT_FTS_SQL);
+        const fragmentStmt = db.prepare(INSERT_FRAGMENT_SQL);
         let skipped = 0;
         for (const row of rows) {
             let entry;
@@ -146,6 +287,11 @@ export function rebuildFts(db) {
                 continue;
             }
             insertStmt.run(row.id, fields.name, fields.description, fields.tags, fields.hints, fields.content);
+            if (row.safe_markdown) {
+                for (const fragment of splitMarkdownFragments(row.safe_markdown)) {
+                    fragmentStmt.run(row.id, fragment.fragmentId, fragment.ordinal, fragment.text.toLowerCase());
+                }
+            }
         }
         if (skipped > 0) {
             warn(`[db] rebuildFts: skipped ${skipped} entr${skipped === 1 ? "y" : "ies"} with invalid document_json`);

package/dist/storage/repositories/index-schema.js

@@ -179,8 +179,8 @@ function rebuildIncompatibleIndexGeneration(db) {
     const classification = classifyIndexGeneration(db);
     if (classification.status === "newer") {
         throw new ConfigError(`Index database was built by a newer akm (stored generation ${classification.storedVersion ?? "unknown"}; ` +
-            `this binary understands generation ${CANONICAL_INDEX_DB_VERSION}). Refusing to modify it — upgrade akm to ` +
-            "write to this index, or delete index.db to rebuild it from scratch with this binary.", "INDEX_SCHEMA_INCOMPATIBLE");
+            `this binary understands generation ${CANONICAL_INDEX_DB_VERSION}). Refusing to modify it — upgrade akm ` +
+            "to use this index.", "INDEX_SCHEMA_INCOMPATIBLE", "Upgrade akm to a version that understands this index generation.");
     }
     warn(`Index database generation ${classification.storedVersion ?? "unknown"} is older than this akm's generation ` +
         `${CANONICAL_INDEX_DB_VERSION} — rebuilding the derived index (entries, FTS, embeddings, graph tables, ` +
@@ -202,6 +202,8 @@ function rebuildIncompatibleIndexGeneration(db) {
         db.exec("DROP TABLE IF EXISTS graph_extraction_queue");
         db.exec("DROP TABLE IF EXISTS graph_meta");
         db.exec("DROP TABLE IF EXISTS entries_fts_dirty");
+        db.exec("DROP TABLE IF EXISTS entry_fragments_fts");
+        db.exec("DROP TABLE IF EXISTS entry_fragments");
         db.exec("DROP TABLE IF EXISTS entries_fts");
         db.exec("DROP TABLE IF EXISTS embeddings");
         db.exec("DROP TABLE IF EXISTS utility_scores_scoped");
@@ -229,7 +231,6 @@ export function ensureSchema(db, embeddingDim) {
     // second persisted representation and was never used by current execution.
     // index.db is derived state, so remove the obsolete table on every open.
     db.exec("DROP TABLE IF EXISTS workflow_documents");
-    setMeta(db, "version", String(DB_VERSION));
     // BLOB-based embedding storage (always available, no sqlite-vec needed)
     db.exec(`
     CREATE TABLE IF NOT EXISTS embeddings (
@@ -238,21 +239,6 @@ export function ensureSchema(db, embeddingDim) {
       FOREIGN KEY (id) REFERENCES entries(id)
     );
   `);
-    // FTS5 table — multi-column with per-field weighting via bm25()
-    const ftsExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='entries_fts'").get();
-    if (!ftsExists) {
-        db.exec(`
-      CREATE VIRTUAL TABLE entries_fts USING fts5(
-        entry_id UNINDEXED,
-        name,
-        description,
-        tags,
-        hints,
-        content,
-        tokenize='porter unicode61'
-      );
-    `);
-    }
     // usage_events lives in state.db. utility_scores remains a regenerable
     // index.db cache.
     // Utility scores table (aggregated per-entry utility metrics)
@@ -394,6 +380,10 @@ export function ensureSchema(db, embeddingDim) {
     // Registry index cache table — caches remote registry index documents so
     // `akm search` does not hit the network on every invocation.
     db.exec(REGISTRY_INDEX_CACHE_DDL);
+    // Write the generation stamp only after every required DDL surface exists.
+    // A crash before this point leaves an unversioned generation that the next
+    // writable open safely rebuilds instead of admitting a partial v23 index.
+    setMeta(db, "version", String(DB_VERSION));
 }
 /**
  * Returns true when a table exists in the current database.

package/dist/storage/repositories/workflow-runs-repository.js

@@ -5,6 +5,7 @@ import { randomUUID } from "node:crypto";
 import { NotFoundError, UsageError } from "../../core/errors.js";
 import { openStateDatabase, withImmediateTransaction } from "../../core/state-db.js";
 import { borrowScopedStateDb, withStateDbScope } from "../../core/state-db-scope.js";
+import { sleepSync } from "../../runtime.js";
 import { escapeLikePattern } from "../like-pattern.js";
 import { resolveStorageLocations } from "../locations.js";
 import { insertEventOnce, insertEventStrict } from "./events-repository.js";
@@ -21,6 +22,51 @@ function assertAttemptReservationLease(input, run) {
         throw new UsageError(`Workflow run ${input.runId} engine lease expired before durable dispatch reservation.`, "RESOURCE_ALREADY_EXISTS");
     }
 }
+/**
+ * Whether `error` is one of the specific SQLite conditions a run-lease
+ * statement can throw under real cross-process contention on the same row:
+ * `SQLITE_BUSY`/`SQLITE_LOCKED` (both drivers), or the message text a
+ * transient contention blip has been observed producing, "database is
+ * locked", "disk I/O error", or "database disk image is malformed".
+ * Matching on this set alone is never sufficient to call something lease
+ * contention — see {@link WorkflowRunsRepository.acquireEngineLease}, which
+ * additionally requires a fresh read confirming a live lease before
+ * substituting the lease-held message for the original error.
+ */
+function isLeaseContentionSqliteError(error) {
+    const code = error?.code;
+    if (code === "SQLITE_BUSY" || code === "SQLITE_LOCKED")
+        return true;
+    const message = error instanceof Error ? error.message : String(error);
+    return (message.includes("database is locked") ||
+        message.includes("disk I/O error") ||
+        message.includes("database disk image is malformed"));
+}
+const LEASE_RETRY_ATTEMPTS = 4;
+const LEASE_RETRY_BASE_DELAY_MS = 15;
+/**
+ * Retry a single lease statement across a short, bounded set of attempts when
+ * it throws one of {@link isLeaseContentionSqliteError}'s conditions —
+ * absorbing a blip that a fresh attempt on the same connection clears on its
+ * own. Any other error, or the same error surviving every attempt, propagates
+ * unchanged; this never converts a persistent failure into a false success.
+ */
+function runLeaseStatementWithRetry(fn) {
+    let lastError;
+    for (let attempt = 0; attempt < LEASE_RETRY_ATTEMPTS; attempt += 1) {
+        try {
+            return fn();
+        }
+        catch (error) {
+            if (!isLeaseContentionSqliteError(error))
+                throw error;
+            lastError = error;
+            if (attempt < LEASE_RETRY_ATTEMPTS - 1)
+                sleepSync(LEASE_RETRY_BASE_DELAY_MS * 2 ** attempt);
+        }
+    }
+    throw lastError;
+}
 /**
  * Repository owning every raw SQL statement against `workflow_runs` and
  * `workflow_run_steps`. It is DB-location-agnostic: the lifecycle helper
@@ -365,25 +411,57 @@ export class WorkflowRunsRepository {
      * A live lease held by anyone (including a stale copy of the same holder)
      * is NOT reclaimable through this method; the single UPDATE is the whole
      * claim, so two racing invocations cannot both win.
+     *
+     * The UPDATE can throw instead of cleanly returning `changes: 0` under real
+     * cross-process contention on this row: a `SQLITE_BUSY`/`SQLITE_LOCKED`
+     * from two engines racing the same statement, occasionally surfacing as
+     * "database is locked" or even "database disk image is malformed" text that
+     * reads as corruption but is not. `runLeaseStatementWithRetry` absorbs a
+     * blip that a fresh attempt clears on its own. If it is still failing after
+     * every retry, the row is read fresh (a plain SELECT, far less likely to
+     * trip whatever the write hit) to get independent evidence of what is
+     * actually going on: a live lease there means this really was contention,
+     * so the caller gets the same lease-held message `akm workflow run` already
+     * shows for the clean (non-throwing) case, now with `RUN_LEASE_HELD`. No
+     * live lease — or the verifying read itself fails — means the error was
+     * never actually about the lease, so it is rethrown exactly as raised.
+     * Nothing here invents a diagnosis from error text alone or suppresses a
+     * genuine SQLite failure.
      */
     acquireEngineLease(runId, holder, until, now) {
-        const result = this.db
-            .prepare(`UPDATE workflow_runs
-           SET engine_lease_holder = ?, engine_lease_until = ?
-           WHERE id = ? AND status = 'active'
-             AND (engine_lease_holder IS NULL OR engine_lease_until IS NULL OR engine_lease_until < ?)`)
-            .run(holder, until, runId, now);
-        return Number(result.changes) > 0;
+        try {
+            const result = runLeaseStatementWithRetry(() => this.db
+                .prepare(`UPDATE workflow_runs
+               SET engine_lease_holder = ?, engine_lease_until = ?
+               WHERE id = ? AND status = 'active'
+                 AND (engine_lease_holder IS NULL OR engine_lease_until IS NULL OR engine_lease_until < ?)`)
+                .run(holder, until, runId, now));
+            return Number(result.changes) > 0;
+        }
+        catch (error) {
+            if (!isLeaseContentionSqliteError(error))
+                throw error;
+            const row = this.tryReadLeaseColumns(runId);
+            if (row?.engine_lease_holder && row.engine_lease_until && row.engine_lease_until >= now) {
+                throw new UsageError(`Workflow run ${runId} is already being driven by engine ${row.engine_lease_holder} ` +
+                    `(run lease expires ${row.engine_lease_until}). A second \`akm workflow run\` would race it — ` +
+                    `wait for that invocation to finish or for the lease to expire.`, "RUN_LEASE_HELD");
+            }
+            throw error;
+        }
     }
     /**
      * Extend the lease expiry — only while `holder` still owns it. Returns
      * false when the lease was lost (expired and claimed by another engine),
-     * so the caller can stop driving instead of racing the new owner.
+     * so the caller can stop driving instead of racing the new owner. Wrapped
+     * in the same transient-error retry as {@link acquireEngineLease}; a
+     * renewal that still fails after retries is rethrown as-is (no confirmed
+     * "lost lease" diagnosis to substitute, unlike the acquire case above).
      */
     renewEngineLease(runId, holder, until) {
-        const result = this.db
+        const result = runLeaseStatementWithRetry(() => this.db
             .prepare("UPDATE workflow_runs SET engine_lease_until = ? WHERE id = ? AND engine_lease_holder = ? AND status = 'active'")
-            .run(until, runId, holder);
+            .run(until, runId, holder));
         return Number(result.changes) > 0;
     }
     /**
@@ -396,6 +474,36 @@ export class WorkflowRunsRepository {
             .prepare("UPDATE workflow_runs SET engine_lease_holder = NULL, engine_lease_until = NULL WHERE id = ? AND engine_lease_holder = ? AND status <> 'failed'")
             .run(runId, holder);
     }
+    /**
+     * Self-heal an engine lease its holder crashed without releasing: once
+     * `engine_lease_until` has passed, clear it so a read (`workflow status`,
+     * `workflow list`) stops reporting a run as engine-driven when the engine is
+     * long gone — mirroring the maintenance barrier's self-reclaim of a wedged
+     * sentinel (`tryAcquireMaintenanceBarrier`) rather than a bespoke mechanism.
+     * The WHERE clause repeats the exact (holder, until) snapshot the caller
+     * read, so a lease renewed or re-acquired in between never gets clobbered —
+     * same compare-and-swap shape as the claim above. Never touches a lease
+     * that is still live.
+     */
+    reclaimExpiredEngineLease(runId, holder, until, now) {
+        if (until >= now)
+            return false;
+        const result = this.db
+            .prepare(`UPDATE workflow_runs
+           SET engine_lease_holder = NULL, engine_lease_until = NULL
+           WHERE id = ? AND engine_lease_holder = ? AND engine_lease_until = ? AND engine_lease_until < ?`)
+            .run(runId, holder, until, now);
+        return Number(result.changes) > 0;
+    }
+    /** Best-effort lease-column read used only to confirm genuine contention after {@link acquireEngineLease} exhausts its retries. `undefined` on any failure — never a diagnosis, just "couldn't confirm". */
+    tryReadLeaseColumns(runId) {
+        try {
+            return (this.db.prepare("SELECT engine_lease_holder, engine_lease_until FROM workflow_runs WHERE id = ?").get(runId) ?? undefined);
+        }
+        catch {
+            return undefined;
+        }
+    }
     // ── durable v4 append-only dispatch attempts (migration 022) ─────────────
     getUnitAttempts(runId, unitId) {
         return this.db