akm-cli 0.9.13 → 0.9.14-beta.1

package/dist/storage/repositories/index-connection.js

@@ -126,23 +126,38 @@ export function openExistingDatabase(dbPath) {
     if (classifyPathAccess(resolvedPath).access === "absent") {
         throw new Error(`Index database not found at ${resolvedPath}. Run 'akm index' to build it.`);
     }
-    return openManagedDatabase({
+    const db = openManagedDatabase({
         path: resolvedPath,
         init: (db) => {
             loadVecExtension(db);
-            warnIfNonCanonicalIndexGeneration(db, resolvedPath);
         },
         create: false,
     });
+    try {
+        assertCanonicalIndexGeneration(db, resolvedPath);
+        return db;
+    }
+    catch (error) {
+        db.close();
+        throw error;
+    }
 }
-function warnIfNonCanonicalIndexGeneration(db, resolvedPath) {
+/**
+ * Read callers must never receive a known-incompatible derived index.  The
+ * writable opener owns rebuilding an older generation; a reader can only
+ * report the one action that is safe for the direction of the mismatch.
+ */
+function assertCanonicalIndexGeneration(db, resolvedPath) {
     if (isCanonicalIndexGeneration(db))
         return;
     const classification = classifyIndexGeneration(db);
-    warnOnce(`index-read-noncanonical:${resolvedPath}`, `Index database at ${resolvedPath} does not match this akm's derived schema (stored generation ` +
-        `${classification.storedVersion ?? "unknown"}; this binary understands ${CANONICAL_INDEX_DB_VERSION}). ` +
-        "Reading it as-is; a query that needs a table or column this generation lacks will fail on its own. " +
-        "Run 'akm index' to rebuild it for this binary.");
+    const stored = classification.storedVersion ?? "unknown";
+    if (classification.status === "newer") {
+        throw new ConfigError(`Index database at ${resolvedPath} was built by a newer akm (stored generation ${stored}; ` +
+            `this binary understands ${CANONICAL_INDEX_DB_VERSION}). Upgrade akm to use this index.`, "INDEX_SCHEMA_INCOMPATIBLE", "Upgrade akm to a version that understands this index generation.");
+    }
+    throw new ConfigError(`Index database at ${resolvedPath} is not usable with this akm's derived schema (stored generation ${stored}; ` +
+        `this binary understands ${CANONICAL_INDEX_DB_VERSION}). Run 'akm index' to rebuild it.`, "INDEX_SCHEMA_INCOMPATIBLE", "Run `akm index` to rebuild the derived index from the currently materialized sources.");
 }
 /**
  * Refuse to treat an UNREADABLE index as a missing one (#791).
@@ -204,7 +219,7 @@ export function openReadonlyExistingDatabase(dbPath, options) {
     // connection, so apply just that one.
     try {
         db.exec(`PRAGMA busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`);
-        warnIfNonCanonicalIndexGeneration(db, resolvedPath);
+        assertCanonicalIndexGeneration(db, resolvedPath);
         return db;
     }
     catch (error) {

package/dist/storage/repositories/index-entries-repository.js

@@ -16,6 +16,7 @@ import { bestEffort } from "../../core/best-effort.js";
 import { isPathAbsent } from "../../core/path-access.js";
 import { getStateDbPath, withStateDb } from "../../core/state-db.js";
 import { warn } from "../../core/warn.js";
+import { getMarkdownFragmentContent, hasMarkdownFragmentContent, } from "../../indexer/passes/metadata.js";
 import { buildSearchText } from "../../indexer/search/search-fields.js";
 import { ENTRY_COLUMNS, rowToIndexedEntry } from "./index-entry-mapper.js";
 import { deleteFtsEntries, replaceFtsEntry } from "./index-fts-repository.js";
@@ -47,7 +48,7 @@ export function upsertEntry(db, filePath, entry, searchText, provenance, content
             throw new Error("upsertEntry: item_ref not found after upsert");
         if (previous?.id === result.id && previous.search_text !== searchText)
             deleteEntryVectors(db, result.id);
-        replaceFtsEntry(db, result.id, entry);
+        replaceFtsEntry(db, result.id, entry, hasMarkdownFragmentContent(entry) ? (getMarkdownFragmentContent(entry) ?? null) : undefined);
         return result.id;
     };
     // Always enter the driver's transaction wrapper. Both supported SQLite
@@ -249,7 +250,7 @@ export function rekeyEntryInPlace(db, opts) {
         if (row.search_text !== searchText)
             deleteEntryVectors(db, row.id);
         if (document)
-            replaceFtsEntry(db, row.id, document);
+            replaceFtsEntry(db, row.id, document, hasMarkdownFragmentContent(document) ? (getMarkdownFragmentContent(document) ?? null) : undefined);
         else
             deleteFtsEntries(db, [row.id]);
     })();

package/dist/storage/repositories/index-entry-schema.js

@@ -10,7 +10,9 @@
  * serving preflights, and read-only evaluator tooling cannot drift into
  * separate definitions of "current".
  */
-export const CANONICAL_INDEX_DB_VERSION = 22;
+// v23 adds an isolated fragment FTS population. v22 is the last shipped
+// generation and is intentionally rebuilt rather than migrated in place.
+export const CANONICAL_INDEX_DB_VERSION = 23;
 export const CANONICAL_ENTRY_SCHEMA_SQL = `
   CREATE TABLE IF NOT EXISTS entries (
     id            INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -31,6 +33,31 @@ export const CANONICAL_ENTRY_SCHEMA_SQL = `
   CREATE INDEX IF NOT EXISTS idx_entries_type ON entries(type);
   CREATE INDEX IF NOT EXISTS idx_entries_file_path ON entries(file_path);
   CREATE INDEX IF NOT EXISTS idx_entries_derived_from ON entries(derived_from);
+
+  -- Keep parent metadata and body fragments in separate FTS populations.
+  -- Combining them changes parent-document IDF and conjunction semantics.
+  CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(
+    entry_id UNINDEXED,
+    name,
+    description,
+    tags,
+    hints,
+    content,
+    tokenize='porter unicode61'
+  );
+
+  CREATE TABLE IF NOT EXISTS entry_fragments (
+    entry_id INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE,
+    safe_markdown TEXT NOT NULL
+  );
+
+  CREATE VIRTUAL TABLE IF NOT EXISTS entry_fragments_fts USING fts5(
+    entry_id UNINDEXED,
+    fragment_id UNINDEXED,
+    fragment_ordinal UNINDEXED,
+    content,
+    tokenize='porter unicode61'
+  );
 `;
 const CANONICAL_ENTRY_SCHEMA_FINGERPRINT = {
     tableSql: "CREATE TABLE entries ( id INTEGER PRIMARY KEY AUTOINCREMENT, item_ref TEXT NOT NULL UNIQUE, bundle_id TEXT NOT NULL, component_id TEXT NOT NULL, concept_id TEXT NOT NULL, adapter_id TEXT NOT NULL, type TEXT NOT NULL, file_path TEXT NOT NULL, content_hash TEXT, document_json TEXT NOT NULL, search_text TEXT NOT NULL, derived_from TEXT )",
@@ -182,6 +209,11 @@ const CANONICAL_ENTRY_SCHEMA_FINGERPRINT = {
             ],
         },
     ],
+    searchSurfaces: {
+        entriesFtsSql: "CREATE VIRTUAL TABLE entries_fts USING fts5( entry_id UNINDEXED, name, description, tags, hints, content, tokenize='porter unicode61' )",
+        fragmentSourceSql: "CREATE TABLE entry_fragments ( entry_id INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE, safe_markdown TEXT NOT NULL )",
+        fragmentsFtsSql: "CREATE VIRTUAL TABLE entry_fragments_fts USING fts5( entry_id UNINDEXED, fragment_id UNINDEXED, fragment_ordinal UNINDEXED, content, tokenize='porter unicode61' )",
+    },
 };
 function sqlString(value) {
     return `'${value.replaceAll("'", "''")}'`;
@@ -191,8 +223,11 @@ function normalizeSchemaSql(value) {
         return null;
     return value.replace(/\s+/g, " ").trim();
 }
+function readNamedTableSql(db, name) {
+    const row = db.prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ${sqlString(name)}`).get();
+    return normalizeSchemaSql(row?.sql);
+}
 export function readEntrySchemaFingerprint(db) {
-    const tableRow = db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'entries'").get();
     const sqliteSequenceTable = db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'sqlite_sequence'").get() !=
         null;
     const maxId = Number(db.prepare("SELECT COALESCE(MAX(id), 0) AS maxId FROM entries").get().maxId);
@@ -229,11 +264,16 @@ export function readEntrySchemaFingerprint(db) {
     }))
         .sort((left, right) => left.name.localeCompare(right.name));
     return {
-        tableSql: normalizeSchemaSql(tableRow?.sql),
+        tableSql: readNamedTableSql(db, "entries"),
         sqliteSequenceTable,
         sqliteSequenceValid,
         columns,
         indexes,
+        searchSurfaces: {
+            entriesFtsSql: readNamedTableSql(db, "entries_fts"),
+            fragmentSourceSql: readNamedTableSql(db, "entry_fragments"),
+            fragmentsFtsSql: readNamedTableSql(db, "entry_fragments_fts"),
+        },
     };
 }
 export function hasCanonicalEntrySchema(db) {

package/dist/storage/repositories/index-fts-repository.js

@@ -7,11 +7,14 @@
  * Owns the `entries_fts` full-text query path, per-entry projections, and the
  * explicit full recovery rebuild.
  */
+import { fragmentForSelector, splitMarkdownFragments } from "../../core/asset/markdown-fragments.js";
+import { stableFtsScore } from "../../core/lexical-score.js";
 import { warn } from "../../core/warn.js";
 import { buildLexicalQueryPlan } from "../../indexer/search/fts-query.js";
 import { buildSearchFields } from "../../indexer/search/search-fields.js";
 import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
 const INSERT_FTS_SQL = "INSERT INTO entries_fts (entry_id, name, description, tags, hints, content) VALUES (?, ?, ?, ?, ?, ?)";
+const INSERT_FRAGMENT_SQL = "INSERT INTO entry_fragments_fts (entry_id, fragment_id, fragment_ordinal, content) VALUES (?, ?, ?, ?)";
 const ftsMutationStatementsByDb = new WeakMap();
 function getFtsMutationStatements(db) {
     const existing = ftsMutationStatementsByDb.get(db);
@@ -20,16 +23,34 @@ function getFtsMutationStatements(db) {
     const statements = {
         deleteOne: db.prepare("DELETE FROM entries_fts WHERE entry_id = ?"),
         insert: db.prepare(INSERT_FTS_SQL),
+        deleteFragments: db.prepare("DELETE FROM entry_fragments_fts WHERE entry_id = ?"),
+        upsertFragmentSource: db.prepare("INSERT INTO entry_fragments (entry_id, safe_markdown) VALUES (?, ?) ON CONFLICT(entry_id) DO UPDATE SET safe_markdown = excluded.safe_markdown"),
+        deleteFragmentSource: db.prepare("DELETE FROM entry_fragments WHERE entry_id = ?"),
+        insertFragment: db.prepare(INSERT_FRAGMENT_SQL),
     };
     ftsMutationStatementsByDb.set(db, statements);
     return statements;
 }
 /** Replace one entry's derived FTS projection inside the caller's transaction. */
-export function replaceFtsEntry(db, entryId, entry) {
+export function replaceFtsEntry(db, entryId, entry, fragmentContent) {
     const fields = buildSearchFields(entry);
     const statements = getFtsMutationStatements(db);
     statements.deleteOne.run(entryId);
     statements.insert.run(entryId, fields.name, fields.description, fields.tags, fields.hints, fields.content);
+    if (fragmentContent === undefined) {
+        // Metadata-only re-upserts and re-keys deserialize the public document
+        // without the internal substrate. Leave the persisted source untouched.
+        // A scan that did read Markdown always supplies a value below.
+        return;
+    }
+    statements.deleteFragments.run(entryId);
+    statements.deleteFragmentSource.run(entryId);
+    if (!fragmentContent)
+        return;
+    statements.upsertFragmentSource.run(entryId, fragmentContent);
+    for (const fragment of splitMarkdownFragments(fragmentContent)) {
+        statements.insertFragment.run(entryId, fragment.fragmentId, fragment.ordinal, fragment.text.toLowerCase());
+    }
 }
 /** Delete derived FTS projections for canonical entries that are being removed. */
 export function deleteFtsEntries(db, entryIds) {
@@ -37,6 +58,8 @@ export function deleteFtsEntries(db, entryIds) {
         const chunk = entryIds.slice(i, i + SQLITE_CHUNK_SIZE);
         const placeholders = chunk.map(() => "?").join(",");
         db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
+        db.prepare(`DELETE FROM entry_fragments_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
+        db.prepare(`DELETE FROM entry_fragments WHERE entry_id IN (${placeholders})`).run(...chunk);
     }
 }
 export function searchFts(db, query, limit, entryType, excludeTypes) {
@@ -58,41 +81,92 @@ export function searchFts(db, query, limit, entryType, excludeTypes) {
     // input whose filler terms prevented a strict hit.
     return plan.relaxed ? runFtsQuery(db, plan.relaxed, "relaxed", limit, entryType, excludeTypes) : [];
 }
+/**
+ * Resolve an opaque fragment selector from the indexed safe projection, not
+ * from current disk. A search result remains self-consistent across a later
+ * file edit; the next index refresh atomically publishes the new revision.
+ */
+export function getIndexedMarkdownFragment(db, itemRef, fragmentId) {
+    const row = db
+        .prepare("SELECT s.safe_markdown FROM entry_fragments_fts f JOIN entry_fragments s ON s.entry_id = f.entry_id JOIN entries e ON e.id = f.entry_id WHERE e.item_ref = ? AND f.fragment_id = ?")
+        .get(itemRef, fragmentId);
+    const fragment = row ? fragmentForSelector(row.safe_markdown, fragmentId) : undefined;
+    return fragment ? { content: fragment.text } : undefined;
+}
 function runFtsQuery(db, ftsQuery, lexicalMatch, limit, entryType, excludeTypes) {
+    // Preserve the repository's ordinary limit contract for direct callers.
+    // The boundary-expansion rule applies only to a positive candidate pool.
+    if (limit <= 0)
+        return [];
     // #627 — exclude-type clause. Only applies on the untyped ('any') path; an
     // explicit include filter (entryType) already narrows to a single type, so
     // exclusion is redundant there. An empty list skips the clause entirely
     // (never emit `NOT IN ()`, which is a SQL error / always-false).
     const excludes = excludeTypes && excludeTypes.length > 0 ? excludeTypes : [];
+    const candidateBoundaryOffset = Math.max(0, limit - 1);
     // The typed and untyped paths differ only by one `type` WHERE clause
-    // equality vs. an optional NOT IN exclusion) and their param order — the
-    // SELECT/JOIN/ORDER/LIMIT is shared, so build it once. Join on integer
-    // entry_id directly (no CAST; we store integer). bm25() per-column weights:
+    // equality vs. an optional NOT IN exclusion) and their parameter order.
+    // Join on integer entry_id directly (no CAST; we store integer). bm25()
+    // per-column weights:
     // entry_id(0), name(10), description(5), tags(3), hints(2), content(1).
     let filterClause;
     let params;
     if (entryType && entryType !== "any") {
         filterClause = "AND e.type = ?";
-        params = [ftsQuery, entryType, limit];
+        params = [ftsQuery, entryType, candidateBoundaryOffset];
     }
     else {
         filterClause = excludes.length > 0 ? `AND e.type NOT IN (${excludes.map(() => "?").join(", ")})` : "";
-        // Param order: MATCH, then the NOT IN values, then LIMIT.
-        params = [ftsQuery, ...excludes, limit];
+        // Param order: MATCH, then the NOT IN values, then the zero-based
+        // candidate-boundary offset.
+        params = [ftsQuery, ...excludes, candidateBoundaryOffset];
     }
     const sql = `
-    SELECT e.id, e.file_path AS filePath, e.document_json AS documentJson, e.search_text AS searchText,
-           e.item_ref AS itemRef, e.bundle_id AS bundleId, e.concept_id AS conceptId, e.adapter_id AS adapterId,
-           bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
+    -- Do not make a SQL-only relevance decision inside a tied BM25 boundary:
+    -- the TypeScript ranker adds exact-name, type, and other contributors
+    -- afterwards.  Materialize BM25 once, locate the Nth score, and admit
+    -- every row tied with it.  This deliberately makes the result set
+    -- data-bound for a pathological all-tied query; that is the only way to
+    -- avoid silently dropping a legitimate later ranking winner.
+    WITH scored AS MATERIALIZED (
+      -- Keep this materialized set deliberately narrow. document_json can be
+      -- large, and only rows admitted through the BM25 boundary need it.
+      SELECT e.id, bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
     FROM entries_fts f
     JOIN entries e ON e.id = f.entry_id
     WHERE entries_fts MATCH ?
       ${filterClause}
-    ORDER BY bm25Score, e.id ASC
-    LIMIT ?
+    ), boundary AS (
+      SELECT bm25Score
+      FROM scored
+      ORDER BY bm25Score
+      LIMIT 1 OFFSET ?
+    )
+    SELECT e.id, e.file_path AS filePath, e.document_json AS documentJson, e.search_text AS searchText,
+           e.item_ref AS itemRef, e.bundle_id AS bundleId, e.concept_id AS conceptId, e.adapter_id AS adapterId,
+           scored.bm25Score
+    FROM scored
+    JOIN entries e ON e.id = scored.id
+    WHERE NOT EXISTS (SELECT 1 FROM boundary)
+       OR scored.bm25Score <= (SELECT bm25Score FROM boundary)
+    ORDER BY scored.bm25Score, e.id ASC
   `;
     const rows = db.prepare(sql).all(...params);
-    // Guard against corrupt JSON — skip the row rather than crashing
+    const results = materializeRows(rows, lexicalMatch);
+    // Fragments are a separate, intentionally calibrated evidence population:
+    // parent FTS remains the sole implementation of metadata/body conjunction.
+    // A selector is emitted only for one fragment that independently satisfies
+    // this query. Raw BM25 values are never claimed comparable across tables;
+    // each is passed through #933's stable mapping before merge.
+    const fragmentResults = hasFragmentFts(db)
+        ? runFragmentQuery(db, ftsQuery, lexicalMatch, limit, entryType, excludes)
+        : [];
+    return mergeParentAndFragmentResults(results, fragmentResults);
+}
+function hasFragmentFts(db) {
+    return Boolean(db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'entry_fragments_fts'").get());
+}
+function materializeRows(rows, lexicalMatch) {
     const results = [];
     for (const row of rows) {
         let entry;
@@ -118,6 +192,69 @@ function runFtsQuery(db, ftsQuery, lexicalMatch, limit, entryType, excludeTypes)
     }
     return results;
 }
+function runFragmentQuery(db, ftsQuery, lexicalMatch, limit, entryType, excludes) {
+    const filter = entryType && entryType !== "any"
+        ? "AND e.type = ?"
+        : excludes.length
+            ? `AND e.type NOT IN (${excludes.map(() => "?").join(",")})`
+            : "";
+    const filterParams = entryType && entryType !== "any" ? [entryType] : excludes;
+    const candidateBoundaryOffset = Math.max(0, limit - 1);
+    // Select the winning child per parent inside SQLite before finding the
+    // candidate boundary. A document with many matching fragments therefore
+    // occupies one parent slot, while a boundary tie retains every parent for
+    // the TypeScript ranker to decide with its non-BM25 contributors. This has
+    // one FTS query and no OFFSET walk; the returned boundary is intentionally
+    // data-bound for a pathological all-tied query, just like parent FTS.
+    const sql = `
+    WITH matches AS MATERIALIZED (
+      -- Keep repeated child rows as narrow as parent FTS's scored CTE. The
+      -- document projection can be large; hydrate it only after the one-child
+      -- per-parent collapse and BM25 boundary filtering below.
+      SELECT e.id, f.fragment_id AS fragmentId, f.fragment_ordinal AS fragmentOrdinal,
+             bm25(entry_fragments_fts) AS bm25Score
+      FROM entry_fragments_fts f JOIN entries e ON e.id = f.entry_id
+      WHERE entry_fragments_fts MATCH ? ${filter}
+    ), ranked AS MATERIALIZED (
+      SELECT *, ROW_NUMBER() OVER (PARTITION BY id ORDER BY bm25Score ASC, fragmentOrdinal ASC, fragmentId ASC) AS parentRank
+      FROM matches
+    ), parents AS MATERIALIZED (
+      SELECT * FROM ranked WHERE parentRank = 1
+    ), boundary AS (
+      SELECT bm25Score FROM parents ORDER BY bm25Score ASC LIMIT 1 OFFSET ?
+    )
+    SELECT e.id, e.file_path AS filePath, e.document_json AS documentJson, e.search_text AS searchText,
+           e.item_ref AS itemRef, e.bundle_id AS bundleId, e.concept_id AS conceptId, e.adapter_id AS adapterId,
+           parents.fragmentId, parents.bm25Score
+    FROM parents JOIN entries e ON e.id = parents.id
+    WHERE NOT EXISTS (SELECT 1 FROM boundary)
+       OR parents.bm25Score <= (SELECT bm25Score FROM boundary)
+    ORDER BY parents.bm25Score ASC, parents.id ASC`;
+    const rows = db.prepare(sql).all(ftsQuery, ...filterParams, candidateBoundaryOffset);
+    const results = [];
+    for (const row of rows) {
+        const [result] = materializeRows([row], lexicalMatch);
+        if (result) {
+            results.push({
+                ...result,
+                fragmentId: row.fragmentId,
+                lexicalScore: stableFtsScore(result.bm25Score, "fragment"),
+            });
+        }
+    }
+    return results;
+}
+function mergeParentAndFragmentResults(parents, fragments) {
+    const winners = new Map();
+    for (const parent of parents)
+        winners.set(parent.id, { ...parent, lexicalScore: stableFtsScore(parent.bm25Score) });
+    for (const fragment of fragments) {
+        const existing = winners.get(fragment.id);
+        if (!existing || (fragment.lexicalScore ?? 0) > (existing.lexicalScore ?? 0))
+            winners.set(fragment.id, fragment);
+    }
+    return [...winners.values()].sort((left, right) => (right.lexicalScore ?? 0) - (left.lexicalScore ?? 0) || left.id - right.id);
+}
 /**
  * Explicitly rebuild the complete FTS5 projection from canonical entries.
  * Ordinary entry mutations do not call this: `upsertEntry` and the delete
@@ -131,8 +268,12 @@ function runFtsQuery(db, ftsQuery, lexicalMatch, limit, entryType, excludeTypes)
 export function rebuildFts(db) {
     db.transaction(() => {
         db.exec("DELETE FROM entries_fts");
-        const rows = db.prepare("SELECT id, document_json FROM entries").all();
+        db.exec("DELETE FROM entry_fragments_fts");
+        const rows = db
+            .prepare("SELECT e.id, e.document_json, f.safe_markdown FROM entries e LEFT JOIN entry_fragments f ON f.entry_id = e.id")
+            .all();
         const insertStmt = db.prepare(INSERT_FTS_SQL);
+        const fragmentStmt = db.prepare(INSERT_FRAGMENT_SQL);
         let skipped = 0;
         for (const row of rows) {
             let entry;
@@ -146,6 +287,11 @@ export function rebuildFts(db) {
                 continue;
             }
             insertStmt.run(row.id, fields.name, fields.description, fields.tags, fields.hints, fields.content);
+            if (row.safe_markdown) {
+                for (const fragment of splitMarkdownFragments(row.safe_markdown)) {
+                    fragmentStmt.run(row.id, fragment.fragmentId, fragment.ordinal, fragment.text.toLowerCase());
+                }
+            }
         }
         if (skipped > 0) {
             warn(`[db] rebuildFts: skipped ${skipped} entr${skipped === 1 ? "y" : "ies"} with invalid document_json`);

package/dist/storage/repositories/index-schema.js

@@ -179,8 +179,8 @@ function rebuildIncompatibleIndexGeneration(db) {
     const classification = classifyIndexGeneration(db);
     if (classification.status === "newer") {
         throw new ConfigError(`Index database was built by a newer akm (stored generation ${classification.storedVersion ?? "unknown"}; ` +
-            `this binary understands generation ${CANONICAL_INDEX_DB_VERSION}). Refusing to modify it — upgrade akm to ` +
-            "write to this index, or delete index.db to rebuild it from scratch with this binary.", "INDEX_SCHEMA_INCOMPATIBLE");
+            `this binary understands generation ${CANONICAL_INDEX_DB_VERSION}). Refusing to modify it — upgrade akm ` +
+            "to use this index.", "INDEX_SCHEMA_INCOMPATIBLE", "Upgrade akm to a version that understands this index generation.");
     }
     warn(`Index database generation ${classification.storedVersion ?? "unknown"} is older than this akm's generation ` +
         `${CANONICAL_INDEX_DB_VERSION} — rebuilding the derived index (entries, FTS, embeddings, graph tables, ` +
@@ -202,6 +202,8 @@ function rebuildIncompatibleIndexGeneration(db) {
         db.exec("DROP TABLE IF EXISTS graph_extraction_queue");
         db.exec("DROP TABLE IF EXISTS graph_meta");
         db.exec("DROP TABLE IF EXISTS entries_fts_dirty");
+        db.exec("DROP TABLE IF EXISTS entry_fragments_fts");
+        db.exec("DROP TABLE IF EXISTS entry_fragments");
         db.exec("DROP TABLE IF EXISTS entries_fts");
         db.exec("DROP TABLE IF EXISTS embeddings");
         db.exec("DROP TABLE IF EXISTS utility_scores_scoped");
@@ -229,7 +231,6 @@ export function ensureSchema(db, embeddingDim) {
     // second persisted representation and was never used by current execution.
     // index.db is derived state, so remove the obsolete table on every open.
     db.exec("DROP TABLE IF EXISTS workflow_documents");
-    setMeta(db, "version", String(DB_VERSION));
     // BLOB-based embedding storage (always available, no sqlite-vec needed)
     db.exec(`
     CREATE TABLE IF NOT EXISTS embeddings (
@@ -238,21 +239,6 @@ export function ensureSchema(db, embeddingDim) {
       FOREIGN KEY (id) REFERENCES entries(id)
     );
   `);
-    // FTS5 table — multi-column with per-field weighting via bm25()
-    const ftsExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='entries_fts'").get();
-    if (!ftsExists) {
-        db.exec(`
-      CREATE VIRTUAL TABLE entries_fts USING fts5(
-        entry_id UNINDEXED,
-        name,
-        description,
-        tags,
-        hints,
-        content,
-        tokenize='porter unicode61'
-      );
-    `);
-    }
     // usage_events lives in state.db. utility_scores remains a regenerable
     // index.db cache.
     // Utility scores table (aggregated per-entry utility metrics)
@@ -394,6 +380,10 @@ export function ensureSchema(db, embeddingDim) {
     // Registry index cache table — caches remote registry index documents so
     // `akm search` does not hit the network on every invocation.
     db.exec(REGISTRY_INDEX_CACHE_DDL);
+    // Write the generation stamp only after every required DDL surface exists.
+    // A crash before this point leaves an unversioned generation that the next
+    // writable open safely rebuilds instead of admitting a partial v23 index.
+    setMeta(db, "version", String(DB_VERSION));
 }
 /**
  * Returns true when a table exists in the current database.

package/docs/migration/release-notes/0.9.14.md

@@ -0,0 +1,26 @@
+Migration notes for akm v0.9.14
+
+The derived `index.db` generation changes from v22 to v23 to support lexical
+Markdown fragments. On the first normal read after upgrade, akm detects that
+the v22 cache cannot serve queries and performs one inline v23 rebuild from
+currently materialized sources before serving the request. An explicit
+`akm index` performs the same writable rebuild. No hand-written database
+migration or manual deletion is needed, but the first rebuild may take longer
+than a normal read. Do not run an older akm against an index this release has
+already rebuilt: upgrade that binary instead. If v23 schema creation is
+interrupted, the partial cache is not admitted as current; the next writable
+open rebuilds it, while an existing/read-only opener reports that `akm index`
+is required.
+
+Search may now return an addressable `#akm-fragment-…` suffix for the matching
+part of a long Markdown asset. Pass that returned ref to `akm show` to display
+the exact indexed fragment.
+
+If you use collapse-detector canaries, their baseline predates the new index
+generation. After the `akm index` rebuild/reindex completes, explicitly mint a
+new set from the repository checkout:
+
+    bun scripts/refresh-canary-set.ts --refresh
+
+The refresh is deliberate and is never automatic, so historical canary cycles
+remain interpretable against their original baseline.

package/docs/migration/release-notes/README.md

@@ -7,6 +7,8 @@ live one level up in `docs/migration/`.
 
 ## Available notes
 
+- [0.9.14](0.9.14.md) — index v22-to-v23 derived-cache rebuild, lexical
+  fragments, and collapse-detector canary re-minting
 - [0.9.2](0.9.2.md) — task source v4 migration, workflow source IR v1 and
   durable-v4-family `irVersion: 5`, command diagnostics, and strategy judgment
   migration

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "akm-cli",
-  "version": "0.9.13",
+  "version": "0.9.14-beta.1",
   "type": "module",
   "description": "akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cursor.",
   "keywords": [