akm-cli 0.9.16 → 0.9.17-alpha.10

package/dist/storage/database.js

@@ -23,7 +23,8 @@
  * This file is intentionally NOT an adapter/DI/ports-and-adapters layer. It is
  * a plain module: a structural type plus a factory function. The handle it
  * returns is the real underlying driver instance (so e.g. `sqlite-vec`'s
- * `load(db)` receives the genuine driver handle and works unchanged).
+ * `load(db)`, used to drop the retired `entries_vec` table, receives the
+ * genuine driver handle and works unchanged).
  *
  * @module storage/database
  */
@@ -35,39 +36,13 @@ const isBun = !!process.versions?.bun;
 // Bun built-in, unresolvable on Node) nor `better-sqlite3` (an optional native
 // dep, possibly absent under Bun) is statically imported.
 const nodeRequire = createRequire(import.meta.url);
-// bun:sqlite — Bun built-in, no native build. Cannot run on Node.
-const bunSqliteProvider = {
-    name: "bun:sqlite",
-    supported: () => isBun,
-    open: openBunDatabase,
-};
-// better-sqlite3 — native Node driver. Cannot run on Bun (oven-sh/bun#4290).
-const nodeSqliteProvider = {
-    name: "better-sqlite3",
-    supported: () => !isBun,
-    open: openNodeDatabase,
-};
 /**
- * Ordered provider registry. The factory selects the first supported provider.
- * A future Postgres provider is appended here — and only here. Both SQLite
- * engines are kept as distinct providers (not collapsed) because no single
- * SQLite driver runs on both Bun and Node today.
- */
-const PROVIDERS = [bunSqliteProvider, nodeSqliteProvider];
-/** Select the provider for the current runtime. */
-function selectProvider() {
-    const provider = PROVIDERS.find((p) => p.supported());
-    if (!provider) {
-        throw new Error(`No storage provider supports the current runtime (${isBun ? "Bun" : "Node"}).`);
-    }
-    return provider;
-}
-/**
- * Open a SQLite database handle at `path` via the active {@link StorageProvider}.
- * Returns a handle conforming to the structural {@link Database} type.
+ * Open a SQLite database handle at `path` on the current runtime's driver
+ * (`bun:sqlite` on Bun, `better-sqlite3` on Node). Returns a handle conforming
+ * to the structural {@link Database} type.
  */
 export function openDatabase(path, opts) {
-    return selectProvider().open(path, opts);
+    return isBun ? openBunDatabase(path, opts) : openNodeDatabase(path, opts);
 }
 /**
  * {@link openDatabase} with a finalize-on-close guard (issue #720).
@@ -90,7 +65,7 @@ export function openDatabase(path, opts) {
  * force-finalizing under them changes behavior they were built on.
  */
 export function openDatabaseFinalizing(path, opts) {
-    const db = selectProvider().open(path, opts);
+    const db = openDatabase(path, opts);
     const tracked = new Set();
     const origPrepare = db.prepare.bind(db);
     const origClose = db.close.bind(db);
@@ -251,9 +226,8 @@ function openNodeDatabase(path, opts) {
         // bun:sqlite also provides db.run(). Normalize the latter at the provider
         // boundary so callers and maintenance wrappers can rely on one contract.
         run: (sql, ...params) => db.prepare(sql).run(...params),
-        // sqlite-vec's load(db) calls db.loadExtension(). Without forwarding it the
-        // extension could never load on Node, so the vector fast path was dead
-        // across the entire npm distribution even when sqlite-vec was installed.
+        // sqlite-vec's load(db) calls db.loadExtension(); without forwarding it the
+        // extension could never load on Node.
         loadExtension: db.loadExtension.bind(db),
         transaction: db.transaction.bind(db),
         get inTransaction() {

package/dist/storage/repositories/improve-ledger-repository.js

@@ -0,0 +1,209 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+export const IMPROVE_LEDGER_OUTCOMES = [
+    "proposed",
+    "accepted",
+    "rejected",
+    "quality_rejected",
+    "review_needed",
+    "expired",
+    "unchanged",
+    "failed",
+    "judged_no_action",
+];
+const MS_PER_DAY = 86_400_000;
+const DETAIL_MAX_CHARS = 500;
+/**
+ * Post-rejection windows by source: reflect 14 d, distill 30 d, every other
+ * source 7 d — the constants the rejection backoff always used.
+ */
+export const LEDGER_REJECTION_WINDOW_DAYS = Object.freeze({
+    reflect: 14,
+    distill: 30,
+});
+export const LEDGER_DEFAULT_REJECTION_WINDOW_DAYS = 7;
+/** A proposal nobody reviewed inside the retention window: short grace, not a rejection backoff. */
+export const LEDGER_EXPIRED_GRACE_DAYS = 1;
+/** Revisit cadence for a ref the stage looked at and had nothing to do (or is still pending). */
+export const LEDGER_REVISIT_CADENCE_DAYS = 7;
+/**
+ * The consolidate pair pass's own ledger source (alpha.9): kept apart from
+ * the promote pass's `consolidate` rows so the two candidate-selection
+ * cadences never collide on the same `(stash, ref, source)` key. Its
+ * eligibility is entirely content-driven (`content_hash` above, compared by
+ * `selectInitiators` in `src/commands/improve/consolidate/pair-pass.ts`) —
+ * {@link windowDays} below gives it no `next_eligible_at` timer at all, so a
+ * row never "expires" on its own; only a content change makes the ref
+ * eligible again.
+ */
+export const PAIR_PASS_LEDGER_SOURCE = "consolidate-pair";
+/**
+ * Outcomes whose window a fresh signal on the asset (new feedback, a content
+ * change) cannot lift. Every other window is a revisit cadence that a signal
+ * newer than `last_attempt_at` lifts.
+ */
+export const LEDGER_HARD_OUTCOMES = new Set([
+    "rejected",
+    "quality_rejected",
+    "expired",
+]);
+function windowDays(source, outcome) {
+    // The pair pass's own eligibility never reads next_eligible_at (it compares
+    // content_hash instead — selectInitiators in pair-pass.ts) — recording a
+    // window here would be a number nothing enforces, so every row it writes
+    // stays "eligible now" regardless of outcome.
+    if (source === PAIR_PASS_LEDGER_SOURCE)
+        return null;
+    switch (outcome) {
+        case "rejected":
+        case "quality_rejected":
+            return LEDGER_REJECTION_WINDOW_DAYS[source] ?? LEDGER_DEFAULT_REJECTION_WINDOW_DAYS;
+        case "expired":
+            return LEDGER_EXPIRED_GRACE_DAYS;
+        case "unchanged":
+        case "judged_no_action":
+        case "proposed":
+        case "review_needed":
+            return LEDGER_REVISIT_CADENCE_DAYS;
+        case "accepted":
+        case "failed":
+            return null;
+    }
+}
+/**
+ * The single cadence function: when a `(source, outcome)` recorded at
+ * `fromIso` becomes eligible again, or `null` for "immediately".
+ */
+export function nextEligibleAt(source, outcome, fromIso) {
+    const from = Date.parse(fromIso);
+    if (!Number.isFinite(from))
+        return null;
+    const days = windowDays(source, outcome);
+    return days === null ? null : new Date(from + days * MS_PER_DAY).toISOString();
+}
+/**
+ * Whether the ledger blocks another attempt on this row at `nowIso`.
+ * `signalSinceIso` is the newest signal on the asset (feedback, content
+ * change); a signal newer than the last attempt lifts a soft window but never
+ * a hard one ({@link LEDGER_HARD_OUTCOMES}).
+ */
+export function isLedgerBlocked(row, nowIso, signalSinceIso) {
+    if (!row?.nextEligibleAt)
+        return false;
+    if (row.nextEligibleAt <= nowIso)
+        return false;
+    if (LEDGER_HARD_OUTCOMES.has(row.outcome))
+        return true;
+    return !(signalSinceIso !== undefined && signalSinceIso > row.lastAttemptAt);
+}
+function toRow(row) {
+    return {
+        stashDir: row.stash_dir,
+        ref: row.ref,
+        source: row.source,
+        lastAttemptAt: row.last_attempt_at,
+        // Tolerate an outcome a newer release may add: it still carries a window.
+        outcome: row.outcome,
+        nextEligibleAt: row.next_eligible_at,
+        proposalId: row.proposal_id,
+        detail: row.detail,
+        contentHash: row.content_hash,
+    };
+}
+function trimDetail(detail) {
+    if (detail === undefined)
+        return null;
+    const trimmed = detail.trim();
+    if (trimmed.length === 0)
+        return null;
+    return trimmed.length > DETAIL_MAX_CHARS ? `${trimmed.slice(0, DETAIL_MAX_CHARS - 1)}…` : trimmed;
+}
+/** Record an attempt on `(stashDir, ref, source)`: upsert the row and its cadence. */
+export function recordImproveLedger(db, input) {
+    const row = {
+        stashDir: input.stashDir,
+        ref: input.ref,
+        source: input.source,
+        lastAttemptAt: input.at,
+        outcome: input.outcome,
+        nextEligibleAt: nextEligibleAt(input.source, input.outcome, input.at),
+        proposalId: input.proposalId ?? null,
+        detail: trimDetail(input.detail),
+        contentHash: input.contentHash ?? null,
+    };
+    db.prepare(`INSERT INTO improve_ledger
+       (stash_dir, ref, source, last_attempt_at, outcome, next_eligible_at, proposal_id, detail, content_hash)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT(stash_dir, ref, source) DO UPDATE SET
+       last_attempt_at  = excluded.last_attempt_at,
+       outcome          = excluded.outcome,
+       next_eligible_at = excluded.next_eligible_at,
+       proposal_id      = excluded.proposal_id,
+       detail           = excluded.detail,
+       content_hash     = excluded.content_hash`).run(row.stashDir, row.ref, row.source, row.lastAttemptAt, row.outcome, row.nextEligibleAt, row.proposalId, row.detail, row.contentHash);
+    return row;
+}
+/**
+ * Record a decision (accept / reject / expire / revert) on the proposal a row
+ * was minted for. The row is found by `proposal_id` — the proposal's ref may
+ * differ from the ledger key (a distill proposal for `lessons/x` is keyed by
+ * its input `memories/x`) — and `last_attempt_at` is kept: the window starts
+ * at the decision, the attempt happened when it happened. A proposal no row
+ * knows (minted before the ledger existed) gets a row keyed by its own ref.
+ */
+export function recordImproveLedgerDecision(db, input) {
+    const changes = db
+        .prepare(`UPDATE improve_ledger
+       SET outcome = ?, next_eligible_at = ?, detail = ?
+       WHERE stash_dir = ? AND proposal_id = ?`)
+        .run(input.outcome, nextEligibleAt(input.source, input.outcome, input.at), trimDetail(input.detail), input.stashDir, input.proposalId).changes;
+    if (Number(changes) > 0)
+        return;
+    recordImproveLedger(db, {
+        stashDir: input.stashDir,
+        ref: input.ref,
+        source: input.source,
+        outcome: input.outcome,
+        at: input.at,
+        proposalId: input.proposalId,
+        ...(input.detail !== undefined ? { detail: input.detail } : {}),
+    });
+}
+/**
+ * Should-fix 6 (second review round): a read-only or dry-run open never
+ * migrates, so it can land on a state.db from before migration 029 added
+ * `content_hash` — reading it there threw "no such column", which
+ * `loadRetrievalScope`'s own catch then reported as "usage history
+ * unreadable", making the WHOLE scope `undefined` (every asset eligible) on
+ * every read-only/dry-run call against an as-yet-unmigrated database. A
+ * per-connection cache, since a real `Database` handle's schema does not
+ * change mid-lifetime and this is checked on every ledger read.
+ */
+const hasContentHashColumnCache = new WeakMap();
+function hasContentHashColumn(db) {
+    const cached = hasContentHashColumnCache.get(db);
+    if (cached !== undefined)
+        return cached;
+    const has = db.prepare("PRAGMA table_info(improve_ledger)").all().some((c) => c.name === "content_hash");
+    hasContentHashColumnCache.set(db, has);
+    return has;
+}
+export function getImproveLedgerRow(db, stashDir, ref, source) {
+    const withHash = hasContentHashColumn(db);
+    const row = db
+        .prepare(`SELECT stash_dir, ref, source, last_attempt_at, outcome, next_eligible_at, proposal_id, detail${withHash ? ", content_hash" : ""}
+       FROM improve_ledger WHERE stash_dir = ? AND ref = ? AND source = ?`)
+        .get(stashDir, ref, source);
+    return row ? toRow(withHash ? row : { ...row, content_hash: null }) : undefined;
+}
+/** Every row for one stash, optionally narrowed to `sources`. */
+export function listImproveLedgerRows(db, stashDir, sources) {
+    const withHash = hasContentHashColumn(db);
+    const sourceFilter = sources && sources.length > 0 ? ` AND source IN (${sources.map(() => "?").join(", ")})` : "";
+    const rows = db
+        .prepare(`SELECT stash_dir, ref, source, last_attempt_at, outcome, next_eligible_at, proposal_id, detail${withHash ? ", content_hash" : ""}
+       FROM improve_ledger WHERE stash_dir = ?${sourceFilter} ORDER BY ref ASC, source ASC`)
+        .all(stashDir, ...(sources && sources.length > 0 ? sources : []));
+    return rows.map((row) => toRow(withHash ? row : { ...row, content_hash: null }));
+}

package/dist/storage/repositories/index-connection.js

@@ -4,14 +4,13 @@
 /**
  * `index.db` connection lifecycle for the storage layer.
  *
- * Opens/closes the index database, arming the sqlite-vec extension and (for the
- * managed open path) running `ensureSchema`. This module lives BELOW the
- * indexer, so the storage loan helpers (`index-db.ts`, `registry-cache.ts`)
+ * Opens/closes the index database, running `ensureSchema` on the managed
+ * (writable) open path. This module lives BELOW the
+ * indexer, so the storage loan helpers (`index-db.ts`, `registry-index-cache-repository.ts`)
  * import their opener from a sibling here instead of reaching up into the
  * indexer — inverting the old storage→indexer arrow.
  */
 import fs from "node:fs";
-import { createRequire } from "node:module";
 import { ConfigError } from "../../core/errors.js";
 import { classifyPathAccess, describeInaccessiblePath } from "../../core/path-access.js";
 import { getDbPath } from "../../core/paths.js";
@@ -20,9 +19,8 @@ import { openDatabase } from "../database.js";
 import { openManagedDatabase } from "../managed-db.js";
 import { SQLITE_BUSY_TIMEOUT_MS } from "../sqlite-pragmas.js";
 import { openSqliteReadSnapshot, SqliteReadSnapshotUnavailableError } from "../sqlite-read-snapshot.js";
-import { CANONICAL_INDEX_DB_VERSION, classifyIndexGeneration, isCanonicalIndexGeneration } from "./index-entry-schema.js";
-import { ensureSchema } from "./index-schema.js";
-import { loadVecExtension, warnIfVecMissing } from "./index-vec-repository.js";
+import { CANONICAL_INDEX_DB_VERSION } from "./index-entry-schema.js";
+import { ensureSchema, newerIndexLayoutError } from "./index-schema.js";
 /**
  * Whether `error` is SQLite reporting on-disk corruption (`SQLITE_CORRUPT`,
  * "database disk image is malformed") rather than a permission, lock, or
@@ -41,21 +39,11 @@ export function openIndexDatabase(dbPath, options) {
     const spec = {
         path: resolvedPath,
         init: (db) => {
-            // Try to load sqlite-vec extension
-            loadVecExtension(db);
             // Source update uses this narrow lifecycle seam to ATTACH state.db and
             // open its coordinator-owned outer transaction before ensureSchema or
             // any indexer write can mutate the live generation.
             options?.beforeSchema?.(db);
-            // Dim resolution: explicit option wins; otherwise consult the on-disk
-            // config so unparameterised opens (registry providers, graph helpers,
-            // ad-hoc CLI subcommands) honour the operator-declared dimension. Only if
-            // both are absent do we fall through to the no-clobber path, which keeps
-            // ensureSchema from touching `index_meta.embeddingDim` at all.
-            const resolvedDim = options?.embeddingDim ?? resolveConfiguredEmbeddingDim();
-            ensureSchema(db, resolvedDim);
-            // Warn once at init if using JS fallback with many entries
-            warnIfVecMissing(db, { once: true });
+            ensureSchema(db);
         },
     };
     try {
@@ -66,10 +54,9 @@ export function openIndexDatabase(dbPath, options) {
         // (see src/core/state-db.ts's "Why a separate database from index.db"
         // note) — so real on-disk corruption is recovered by deleting the file
         // and rebuilding, not by surfacing a raw SQLITE_CORRUPT to the caller or
-        // quietly falling through to an unreadable index. This mirrors the
-        // existing stale-version-marker rebuild below, one layer further down
-        // (that path opens fine and rewrites tables in place; corruption prevents
-        // even opening, so the file itself has to go first).
+        // quietly falling through to an unreadable index (#865). This is the ONE
+        // from-scratch rebuild: an older layout is migrated in place by
+        // ensureSchema, never dropped.
         if (!isCorruptionError(error))
             throw error;
         warn(`Index database is corrupt at ${resolvedPath} — rebuilding.`);
@@ -85,34 +72,9 @@ export function openIndexDatabase(dbPath, options) {
         return openManagedDatabase(spec);
     }
 }
-/**
- * Read the operator-configured embedding dimension from the on-disk config.
- * Returns `undefined` when no config file is present, when the config has
- * no `embedding.dimension` set, or when reading the config throws (e.g.
- * inside isolated test fixtures with no XDG home). Failure is silent on
- * purpose — every openDatabase() call would otherwise have to handle a
- * config-not-found error path, and the fallback (no-clobber semantics) is
- * already correct.
- */
-function resolveConfiguredEmbeddingDim() {
-    try {
-        const esmRequire = createRequire(import.meta.url);
-        const { loadConfig } = esmRequire("../../core/config/config");
-        const dim = loadConfig().embedding?.dimension;
-        if (typeof dim === "number" && Number.isInteger(dim) && dim > 0 && dim <= 4096) {
-            return dim;
-        }
-        return undefined;
-    }
-    catch {
-        return undefined;
-    }
-}
 export function openExistingDatabase(dbPath) {
-    // Existing-DB callers do not mutate schema or embedding metadata on open.
-    // They do validate the exact current derived generation before returning a
-    // handle, so no current reader can accidentally serve a populated legacy
-    // table and fail later on its first canonical-column query.
+    // Existing-DB callers do not mutate schema or embedding metadata on open;
+    // they serve an older layout as-is and refuse a newer one (see checkIndexLayout).
     //
     // "Existing" is load-bearing: a missing file throws instead of being
     // created. Create-on-open used to leave a schema-less index.db behind (a
@@ -126,15 +88,9 @@ export function openExistingDatabase(dbPath) {
     if (classifyPathAccess(resolvedPath).access === "absent") {
         throw new Error(`Index database not found at ${resolvedPath}. Run 'akm index' to build it.`);
     }
-    const db = openManagedDatabase({
-        path: resolvedPath,
-        init: (db) => {
-            loadVecExtension(db);
-        },
-        create: false,
-    });
+    const db = openManagedDatabase({ path: resolvedPath, create: false });
     try {
-        assertCanonicalIndexGeneration(db, resolvedPath);
+        checkIndexLayout(db, resolvedPath);
         return db;
     }
     catch (error) {
@@ -143,21 +99,29 @@ export function openExistingDatabase(dbPath) {
     }
 }
 /**
- * Read callers must never receive a known-incompatible derived index.  The
- * writable opener owns rebuilding an older generation; a reader can only
- * report the one action that is safe for the direction of the mismatch.
+ * A reader serves an older layout as-is (the FTS readers understand both
+ * layouts, and a missing table degrades at the caller — keyword-only search,
+ * an inline rebuild, or a "run akm index" notice) and names it once per
+ * process; the next writable open migrates it in place. A newer layout is
+ * refused, naming the upgrade ({@link newerIndexLayoutError}).
  */
-function assertCanonicalIndexGeneration(db, resolvedPath) {
-    if (isCanonicalIndexGeneration(db))
+function checkIndexLayout(db, resolvedPath) {
+    let stored;
+    try {
+        const row = db.prepare("SELECT value FROM index_meta WHERE key = 'version'").get();
+        if (!row)
+            return;
+        stored = Number(row.value);
+    }
+    catch {
         return;
-    const classification = classifyIndexGeneration(db);
-    const stored = classification.storedVersion ?? "unknown";
-    if (classification.status === "newer") {
-        throw new ConfigError(`Index database at ${resolvedPath} was built by a newer akm (stored generation ${stored}; ` +
-            `this binary understands ${CANONICAL_INDEX_DB_VERSION}). Upgrade akm to use this index.`, "INDEX_SCHEMA_INCOMPATIBLE", "Upgrade akm to a version that understands this index generation.");
     }
-    throw new ConfigError(`Index database at ${resolvedPath} is not usable with this akm's derived schema (stored generation ${stored}; ` +
-        `this binary understands ${CANONICAL_INDEX_DB_VERSION}). Run 'akm index' to rebuild it.`, "INDEX_SCHEMA_INCOMPATIBLE", "Run `akm index` to rebuild the derived index from the currently materialized sources.");
+    if (!Number.isFinite(stored) || stored === CANONICAL_INDEX_DB_VERSION)
+        return;
+    if (stored > CANONICAL_INDEX_DB_VERSION)
+        throw newerIndexLayoutError(stored, resolvedPath);
+    warnOnce(`index-db-layout:${resolvedPath}`, `Index database at ${resolvedPath} uses an older layout (${stored}; this akm writes ${CANONICAL_INDEX_DB_VERSION}). ` +
+        "Serving it as-is; the next 'akm index' migrates it in place.");
 }
 /**
  * Refuse to treat an UNREADABLE index as a missing one (#791).
@@ -216,10 +180,13 @@ export function openReadonlyExistingDatabase(dbPath, options) {
     // never block — but in the DELETE/TRUNCATE modes the network-FS fallback and
     // AKM_SQLITE_JOURNAL_MODE can select, a concurrent writer makes every read
     // fail instantly with SQLITE_BUSY. busy_timeout is legal on a read-only
-    // connection, so apply just that one.
+    // connection, so apply just that one. `busyTimeoutMs` defaults to the
+    // shared 30s constant; a caller that must never sit behind another akm
+    // process's write lock for long (e.g. `akm info`) can pass a much shorter
+    // bound instead.
     try {
-        db.exec(`PRAGMA busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`);
-        assertCanonicalIndexGeneration(db, resolvedPath);
+        db.exec(`PRAGMA busy_timeout = ${options?.busyTimeoutMs ?? SQLITE_BUSY_TIMEOUT_MS}`);
+        checkIndexLayout(db, resolvedPath);
         return db;
     }
     catch (error) {