akm-cli 0.9.26 → 0.9.27-alpha.1

package/dist/storage/repositories/index-fts-repository.js

@@ -19,9 +19,27 @@ import { splitMarkdownFragments } from "../../core/asset/markdown-fragments.js";
 import { warn } from "../../core/warn.js";
 import { ftsOrMatch, ftsQueryTokens } from "../../indexer/search/fts-query.js";
 import { buildSearchFields } from "../../indexer/search/search-fields.js";
+import { isContentlessFtsDdl, readTableSql } from "./index-entry-schema.js";
+import { deleteMeta, getMeta } from "./index-meta-repository.js";
 import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
 // `entries_fts.rowid = entries.id`, so a per-entry delete is a rowid lookup.
 const INSERT_FTS_SQL = "INSERT INTO entries_fts (rowid, entry_id, name, description, tags, hints, content) VALUES (?, ?, ?, ?, ?, ?, ?)";
+/**
+ * `index_meta` key stamped when a delete took a row out of `entries_fts`.
+ *
+ * FTS5 cannot take a deleted row out of a contentless table's BM25 totals (its
+ * row count, and the token counts the average document length comes from), so
+ * every removal or replacement leaves them one row too high and an updated
+ * index scores differently from a fresh one (#1048). SQLite has no command that
+ * recomputes them (`delete` and `rebuild` are refused on a contentless table),
+ * so the next `akm index` rebuilds the table from `entries`
+ * ({@link rebuildFtsIfTotalsStale}).
+ */
+const FTS_TOTALS_STALE_META = "ftsTotalsStale";
+/** The content-bearing table older SQLite gets subtracts a deleted row itself. */
+function isContentlessFts(db) {
+    return isContentlessFtsDdl(readTableSql(db, "entries_fts"));
+}
 const ftsMutationStatementsByDb = new WeakMap();
 function getFtsMutationStatements(db) {
     const existing = ftsMutationStatementsByDb.get(db);
@@ -29,6 +47,7 @@ function getFtsMutationStatements(db) {
         return existing;
     const statements = {
         deleteOne: db.prepare("DELETE FROM entries_fts WHERE rowid = ?"),
+        markTotalsStale: db.prepare(`INSERT OR IGNORE INTO index_meta (key, value) VALUES ('${FTS_TOTALS_STALE_META}', '1')`),
         insert: db.prepare(INSERT_FTS_SQL),
         upsertFragmentSource: db.prepare("INSERT INTO entry_fragments (entry_id, safe_markdown) VALUES (?, ?) ON CONFLICT(entry_id) DO UPDATE SET safe_markdown = excluded.safe_markdown"),
         deleteFragmentSource: db.prepare("DELETE FROM entry_fragments WHERE entry_id = ?"),
@@ -36,6 +55,11 @@ function getFtsMutationStatements(db) {
     ftsMutationStatementsByDb.set(db, statements);
     return statements;
 }
+/** A delete that removed a row leaves the table's BM25 totals too high until it is rebuilt. */
+function noteRemoval(statements, removed) {
+    if (removed.changes > 0)
+        statements.markTotalsStale.run();
+}
 /**
  * Replace one entry's derived FTS row, and its fragment source when the scan
  * read Markdown, inside the caller's transaction.
@@ -43,7 +67,7 @@ function getFtsMutationStatements(db) {
 export function replaceFtsEntry(db, entryId, entry, fragmentContent) {
     const fields = buildSearchFields(entry);
     const statements = getFtsMutationStatements(db);
-    statements.deleteOne.run(entryId);
+    noteRemoval(statements, statements.deleteOne.run(entryId));
     statements.insert.run(entryId, entryId, fields.name, fields.description, fields.tags, fields.hints, fields.content);
     if (fragmentContent === undefined) {
         // Metadata-only re-upserts and re-keys deserialize the public document
@@ -61,7 +85,7 @@ export function deleteFtsEntries(db, entryIds) {
     for (let i = 0; i < entryIds.length; i += SQLITE_CHUNK_SIZE) {
         const chunk = entryIds.slice(i, i + SQLITE_CHUNK_SIZE);
         const placeholders = chunk.map(() => "?").join(",");
-        db.prepare(`DELETE FROM entries_fts WHERE rowid IN (${placeholders})`).run(...chunk);
+        noteRemoval(getFtsMutationStatements(db), db.prepare(`DELETE FROM entries_fts WHERE rowid IN (${placeholders})`).run(...chunk));
         db.prepare(`DELETE FROM entry_fragments WHERE entry_id IN (${placeholders})`).run(...chunk);
     }
 }
@@ -143,7 +167,9 @@ function materializeIndexedMarkdownFragment(fragment, fragments, parentChars) {
  */
 export function rebuildFts(db) {
     db.transaction(() => {
-        db.exec("DELETE FROM entries_fts");
+        // `delete-all` also resets the BM25 totals, which a plain DELETE leaves as they were (#1048).
+        db.exec(isContentlessFts(db) ? "INSERT INTO entries_fts(entries_fts) VALUES('delete-all')" : "DELETE FROM entries_fts");
+        deleteMeta(db, FTS_TOTALS_STALE_META);
         // Keyset pages, so a large index is never held in memory at once.
         const page = db.prepare("SELECT id, document_json FROM entries WHERE id > ? ORDER BY id LIMIT 500");
         const insertStmt = db.prepare(INSERT_FTS_SQL);
@@ -171,3 +197,18 @@ export function rebuildFts(db) {
         }
     })();
 }
+/**
+ * Rebuild `entries_fts` from `entries` when rows have left it since its BM25
+ * totals were last taken (#1048): about a second at 25,000 entries. Returns
+ * whether it did.
+ */
+export function rebuildFtsIfTotalsStale(db) {
+    if (getMeta(db, FTS_TOTALS_STALE_META) === undefined)
+        return false;
+    if (!isContentlessFts(db)) {
+        deleteMeta(db, FTS_TOTALS_STALE_META);
+        return false;
+    }
+    rebuildFts(db);
+    return true;
+}

package/dist/storage/repositories/index-schema.js

@@ -25,7 +25,7 @@ import path from "node:path";
 import { ConfigError } from "../../core/errors.js";
 import { warn } from "../../core/warn.js";
 import { sha256Hex } from "../../runtime.js";
-import { CANONICAL_ENTRY_SCHEMA_SQL, CANONICAL_INDEX_DB_VERSION, entriesFtsDdl, isContentlessFtsDdl, missingEntryColumns, readTableSql, supportsContentlessDelete, tableExists, } from "./index-entry-schema.js";
+import { CANONICAL_ENTRY_SCHEMA_SQL, CANONICAL_INDEX_DB_VERSION, entriesFtsDdl, isContentlessFtsDdl, missingEntryColumns, readTableSql, retiredEntryColumns, supportsContentlessDelete, tableExists, } from "./index-entry-schema.js";
 import { rebuildFts } from "./index-fts-repository.js";
 import { rebuildAllEntryLinks } from "./index-links-repository.js";
 import { getMeta, setMeta } from "./index-meta-repository.js";
@@ -67,11 +67,13 @@ const REGISTRY_INDEX_CACHE_DDL = `
     ON registry_index_cache(fetched_at);
 `;
 /**
- * An `entries` table missing a required column cannot be read or written by
- * this release (the last such change was v20→v21, which removed the
- * transitional `entry_key`/`dir_path`/... columns and made `item_ref` the
- * key). Recreate only the tables keyed by `entries.id` — their ids are about
- * to be re-minted, so the rows would dangle anyway. The LLM enrichment cache
+ * An `entries` table missing a required column, or still carrying a retired
+ * one, cannot be read or written by this release (the last such change was
+ * v20→v21, which removed the transitional `entry_key`/`dir_path`/... columns
+ * and made `item_ref` the key). Layouts 18–20 carried the current columns
+ * beside the retired ones, so only the retired ones give them away.
+ * Recreate only the tables keyed by `entries.id` — their ids are about to be
+ * re-minted, so the rows would dangle anyway. The LLM enrichment cache
  * (keyed by ref) is kept. The LLM entity-graph tables are unconditionally
  * dropped elsewhere in this file regardless of this recreation (retired
  * 0.9.17-alpha.9), not kept. The next index run re-walks every source.
@@ -80,9 +82,13 @@ function ensureEntriesLayout(db) {
     if (!tableExists(db, "entries"))
         return;
     const missing = missingEntryColumns(db);
-    if (missing.length === 0)
+    const retired = retiredEntryColumns(db);
+    if (missing.length === 0 && retired.length === 0)
         return;
-    warn(`Index database entries table predates the ${missing.join(", ")} column${missing.length === 1 ? "" : "s"} — ` +
+    const why = missing.length > 0
+        ? `predates the ${missing.join(", ")} column${missing.length === 1 ? "" : "s"}`
+        : `still has the retired ${retired.join(", ")} column${retired.length === 1 ? "" : "s"}`;
+    warn(`Index database entries table ${why} — ` +
         "recreating the entries-keyed tables (entries, full-text, embeddings, utility scores); the " +
         "LLM enrichment cache is kept. The next index run re-walks every source.");
     db.transaction(() => {

package/docs/reference/cli.md

@@ -2780,6 +2780,7 @@ harvest" branch on `skipReasons`, `warnings`, or `sessionsProcessed` /
 | `engineKind` | `"llm"`, `"sdk"`, or `"agent"` — the kind of runner `engine` resolved to. Same absence condition as `engine`. |
 | `skipReasons` | Per-`skipReason` count across `sessions[]` (e.g. `{ "llm_unavailable": 25 }`). Present only when `sessionsSkipped > 0`. |
 | `warnings` | Includes one aggregate line per infrastructure skip reason that fired (`llm_unavailable`, `read_failed`, `exception`, `locked_concurrent`) — e.g. `25 of 25 sessions skipped: llm_unavailable (engine "default")` — so an engine outage is visible without inspecting `sessions[]`. Session-content skips (`already_extracted`, `too_short`, `triaged_out`) are counted in `skipReasons` but never produce a warning line. |
+| `sessions[].warnings` | One line per candidate the model wrote that the output contract refuses, as `<type>:<name> dropped: <reason>` — a lesson without a `when_to_use` of 15 characters, a description under 20, a name that is not a kebab-case slug — and one per candidate held back by the improve ledger. A session whose every candidate was dropped has `candidateCount: 0` and no `rationaleIfEmpty`, so this is where the loss shows. |
 
 #### proposal new
 

package/docs/reference/configuration.md

@@ -110,8 +110,11 @@ distill quality-gate judges do too unless the judge's engine sets
 `reasoning_effort` when set. Backend support: llama.cpp direct honors both forms
 (`reasoning_effort` from build ≥ b10644); vLLM honors
 `chat_template_kwargs`; Bifrost drops `chat_template_kwargs` and passes
-`reasoning_effort` through, so also set `reasoningEffort: "none"` behind it; a
-strict hosted API may 400 on unrecognized keys. Both fields are AKM-owned, not
+`reasoning_effort` through, so also set `reasoningEffort: "none"` behind it. A
+strict hosted API that rejects the two fields (OpenAI answers 400 `Unknown
+parameter: 'chat_template_kwargs'`) gets one retry without both, and AKM stops
+sending them to that endpoint and model for the rest of the process, as it
+does for `response_format` below. Both fields are AKM-owned, not
 settable via `extraParams`. A response with reasoning tokens despite
 `enableThinking: false` triggers a runtime warning and the `akm health`
 `thinking-control` advisory.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "akm-cli",
-  "version": "0.9.26",
+  "version": "0.9.27-alpha.1",
   "type": "module",
   "description": "akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cursor.",
   "keywords": [