akm-cli 0.9.17-alpha.2 → 0.9.17-alpha.4

package/dist/storage/repositories/improve-ledger-repository.js

@@ -0,0 +1,168 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+export const IMPROVE_LEDGER_OUTCOMES = [
+    "proposed",
+    "accepted",
+    "rejected",
+    "quality_rejected",
+    "review_needed",
+    "expired",
+    "unchanged",
+    "failed",
+    "judged_no_action",
+];
+const MS_PER_DAY = 86_400_000;
+const DETAIL_MAX_CHARS = 500;
+/**
+ * Post-rejection windows by source: reflect 14 d, distill 30 d, every other
+ * source 7 d — the constants the rejection backoff always used.
+ */
+export const LEDGER_REJECTION_WINDOW_DAYS = Object.freeze({
+    reflect: 14,
+    distill: 30,
+});
+export const LEDGER_DEFAULT_REJECTION_WINDOW_DAYS = 7;
+/** A proposal nobody reviewed inside the retention window: short grace, not a rejection backoff. */
+export const LEDGER_EXPIRED_GRACE_DAYS = 1;
+/** Revisit cadence for a ref the stage looked at and had nothing to do (or is still pending). */
+export const LEDGER_REVISIT_CADENCE_DAYS = 7;
+/**
+ * Outcomes whose window a fresh signal on the asset (new feedback, a content
+ * change) cannot lift. Every other window is a revisit cadence that a signal
+ * newer than `last_attempt_at` lifts.
+ */
+export const LEDGER_HARD_OUTCOMES = new Set([
+    "rejected",
+    "quality_rejected",
+    "expired",
+]);
+function windowDays(source, outcome) {
+    switch (outcome) {
+        case "rejected":
+        case "quality_rejected":
+            return LEDGER_REJECTION_WINDOW_DAYS[source] ?? LEDGER_DEFAULT_REJECTION_WINDOW_DAYS;
+        case "expired":
+            return LEDGER_EXPIRED_GRACE_DAYS;
+        case "unchanged":
+        case "judged_no_action":
+        case "proposed":
+        case "review_needed":
+            return LEDGER_REVISIT_CADENCE_DAYS;
+        case "accepted":
+        case "failed":
+            return null;
+    }
+}
+/**
+ * The single cadence function: when a `(source, outcome)` recorded at
+ * `fromIso` becomes eligible again, or `null` for "immediately".
+ */
+export function nextEligibleAt(source, outcome, fromIso) {
+    const from = Date.parse(fromIso);
+    if (!Number.isFinite(from))
+        return null;
+    const days = windowDays(source, outcome);
+    return days === null ? null : new Date(from + days * MS_PER_DAY).toISOString();
+}
+/**
+ * Whether the ledger blocks another attempt on this row at `nowIso`.
+ * `signalSinceIso` is the newest signal on the asset (feedback, content
+ * change); a signal newer than the last attempt lifts a soft window but never
+ * a hard one ({@link LEDGER_HARD_OUTCOMES}).
+ */
+export function isLedgerBlocked(row, nowIso, signalSinceIso) {
+    if (!row?.nextEligibleAt)
+        return false;
+    if (row.nextEligibleAt <= nowIso)
+        return false;
+    if (LEDGER_HARD_OUTCOMES.has(row.outcome))
+        return true;
+    return !(signalSinceIso !== undefined && signalSinceIso > row.lastAttemptAt);
+}
+function toRow(row) {
+    return {
+        stashDir: row.stash_dir,
+        ref: row.ref,
+        source: row.source,
+        lastAttemptAt: row.last_attempt_at,
+        // Tolerate an outcome a newer release may add: it still carries a window.
+        outcome: row.outcome,
+        nextEligibleAt: row.next_eligible_at,
+        proposalId: row.proposal_id,
+        detail: row.detail,
+    };
+}
+function trimDetail(detail) {
+    if (detail === undefined)
+        return null;
+    const trimmed = detail.trim();
+    if (trimmed.length === 0)
+        return null;
+    return trimmed.length > DETAIL_MAX_CHARS ? `${trimmed.slice(0, DETAIL_MAX_CHARS - 1)}…` : trimmed;
+}
+/** Record an attempt on `(stashDir, ref, source)`: upsert the row and its cadence. */
+export function recordImproveLedger(db, input) {
+    const row = {
+        stashDir: input.stashDir,
+        ref: input.ref,
+        source: input.source,
+        lastAttemptAt: input.at,
+        outcome: input.outcome,
+        nextEligibleAt: nextEligibleAt(input.source, input.outcome, input.at),
+        proposalId: input.proposalId ?? null,
+        detail: trimDetail(input.detail),
+    };
+    db.prepare(`INSERT INTO improve_ledger
+       (stash_dir, ref, source, last_attempt_at, outcome, next_eligible_at, proposal_id, detail)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT(stash_dir, ref, source) DO UPDATE SET
+       last_attempt_at  = excluded.last_attempt_at,
+       outcome          = excluded.outcome,
+       next_eligible_at = excluded.next_eligible_at,
+       proposal_id      = excluded.proposal_id,
+       detail           = excluded.detail`).run(row.stashDir, row.ref, row.source, row.lastAttemptAt, row.outcome, row.nextEligibleAt, row.proposalId, row.detail);
+    return row;
+}
+/**
+ * Record a decision (accept / reject / expire / revert) on the proposal a row
+ * was minted for. The row is found by `proposal_id` — the proposal's ref may
+ * differ from the ledger key (a distill proposal for `lessons/x` is keyed by
+ * its input `memories/x`) — and `last_attempt_at` is kept: the window starts
+ * at the decision, the attempt happened when it happened. A proposal no row
+ * knows (minted before the ledger existed) gets a row keyed by its own ref.
+ */
+export function recordImproveLedgerDecision(db, input) {
+    const changes = db
+        .prepare(`UPDATE improve_ledger
+       SET outcome = ?, next_eligible_at = ?, detail = ?
+       WHERE stash_dir = ? AND proposal_id = ?`)
+        .run(input.outcome, nextEligibleAt(input.source, input.outcome, input.at), trimDetail(input.detail), input.stashDir, input.proposalId).changes;
+    if (Number(changes) > 0)
+        return;
+    recordImproveLedger(db, {
+        stashDir: input.stashDir,
+        ref: input.ref,
+        source: input.source,
+        outcome: input.outcome,
+        at: input.at,
+        proposalId: input.proposalId,
+        ...(input.detail !== undefined ? { detail: input.detail } : {}),
+    });
+}
+export function getImproveLedgerRow(db, stashDir, ref, source) {
+    const row = db
+        .prepare(`SELECT stash_dir, ref, source, last_attempt_at, outcome, next_eligible_at, proposal_id, detail
+       FROM improve_ledger WHERE stash_dir = ? AND ref = ? AND source = ?`)
+        .get(stashDir, ref, source);
+    return row ? toRow(row) : undefined;
+}
+/** Every row for one stash, optionally narrowed to `sources`. */
+export function listImproveLedgerRows(db, stashDir, sources) {
+    const sourceFilter = sources && sources.length > 0 ? ` AND source IN (${sources.map(() => "?").join(", ")})` : "";
+    const rows = db
+        .prepare(`SELECT stash_dir, ref, source, last_attempt_at, outcome, next_eligible_at, proposal_id, detail
+       FROM improve_ledger WHERE stash_dir = ?${sourceFilter} ORDER BY ref ASC, source ASC`)
+        .all(stashDir, ...(sources && sources.length > 0 ? sources : []));
+    return rows.map(toRow);
+}

package/dist/storage/repositories/index-connection.js

@@ -4,14 +4,13 @@
 /**
  * `index.db` connection lifecycle for the storage layer.
  *
- * Opens/closes the index database, arming the sqlite-vec extension and (for the
- * managed open path) running `ensureSchema`. This module lives BELOW the
- * indexer, so the storage loan helpers (`index-db.ts`, `registry-cache.ts`)
+ * Opens/closes the index database, running `ensureSchema` on the managed
+ * (writable) open path. This module lives BELOW the
+ * indexer, so the storage loan helpers (`index-db.ts`, `registry-index-cache-repository.ts`)
  * import their opener from a sibling here instead of reaching up into the
  * indexer — inverting the old storage→indexer arrow.
  */
 import fs from "node:fs";
-import { createRequire } from "node:module";
 import { ConfigError } from "../../core/errors.js";
 import { classifyPathAccess, describeInaccessiblePath } from "../../core/path-access.js";
 import { getDbPath } from "../../core/paths.js";
@@ -20,9 +19,8 @@ import { openDatabase } from "../database.js";
 import { openManagedDatabase } from "../managed-db.js";
 import { SQLITE_BUSY_TIMEOUT_MS } from "../sqlite-pragmas.js";
 import { openSqliteReadSnapshot, SqliteReadSnapshotUnavailableError } from "../sqlite-read-snapshot.js";
-import { CANONICAL_INDEX_DB_VERSION, classifyIndexGeneration, isCanonicalIndexGeneration } from "./index-entry-schema.js";
-import { ensureSchema } from "./index-schema.js";
-import { loadVecExtension, warnIfVecMissing } from "./index-vec-repository.js";
+import { CANONICAL_INDEX_DB_VERSION } from "./index-entry-schema.js";
+import { ensureSchema, newerIndexLayoutError } from "./index-schema.js";
 /**
  * Whether `error` is SQLite reporting on-disk corruption (`SQLITE_CORRUPT`,
  * "database disk image is malformed") rather than a permission, lock, or
@@ -41,21 +39,11 @@ export function openIndexDatabase(dbPath, options) {
     const spec = {
         path: resolvedPath,
         init: (db) => {
-            // Try to load sqlite-vec extension
-            loadVecExtension(db);
             // Source update uses this narrow lifecycle seam to ATTACH state.db and
             // open its coordinator-owned outer transaction before ensureSchema or
             // any indexer write can mutate the live generation.
             options?.beforeSchema?.(db);
-            // Dim resolution: explicit option wins; otherwise consult the on-disk
-            // config so unparameterised opens (registry providers, graph helpers,
-            // ad-hoc CLI subcommands) honour the operator-declared dimension. Only if
-            // both are absent do we fall through to the no-clobber path, which keeps
-            // ensureSchema from touching `index_meta.embeddingDim` at all.
-            const resolvedDim = options?.embeddingDim ?? resolveConfiguredEmbeddingDim();
-            ensureSchema(db, resolvedDim);
-            // Warn once at init if using JS fallback with many entries
-            warnIfVecMissing(db, { once: true });
+            ensureSchema(db);
         },
     };
     try {
@@ -66,10 +54,9 @@ export function openIndexDatabase(dbPath, options) {
         // (see src/core/state-db.ts's "Why a separate database from index.db"
         // note) — so real on-disk corruption is recovered by deleting the file
         // and rebuilding, not by surfacing a raw SQLITE_CORRUPT to the caller or
-        // quietly falling through to an unreadable index. This mirrors the
-        // existing stale-version-marker rebuild below, one layer further down
-        // (that path opens fine and rewrites tables in place; corruption prevents
-        // even opening, so the file itself has to go first).
+        // quietly falling through to an unreadable index (#865). This is the ONE
+        // from-scratch rebuild: an older layout is migrated in place by
+        // ensureSchema, never dropped.
         if (!isCorruptionError(error))
             throw error;
         warn(`Index database is corrupt at ${resolvedPath} — rebuilding.`);
@@ -85,34 +72,9 @@ export function openIndexDatabase(dbPath, options) {
         return openManagedDatabase(spec);
     }
 }
-/**
- * Read the operator-configured embedding dimension from the on-disk config.
- * Returns `undefined` when no config file is present, when the config has
- * no `embedding.dimension` set, or when reading the config throws (e.g.
- * inside isolated test fixtures with no XDG home). Failure is silent on
- * purpose — every openDatabase() call would otherwise have to handle a
- * config-not-found error path, and the fallback (no-clobber semantics) is
- * already correct.
- */
-function resolveConfiguredEmbeddingDim() {
-    try {
-        const esmRequire = createRequire(import.meta.url);
-        const { loadConfig } = esmRequire("../../core/config/config");
-        const dim = loadConfig().embedding?.dimension;
-        if (typeof dim === "number" && Number.isInteger(dim) && dim > 0 && dim <= 4096) {
-            return dim;
-        }
-        return undefined;
-    }
-    catch {
-        return undefined;
-    }
-}
 export function openExistingDatabase(dbPath) {
-    // Existing-DB callers do not mutate schema or embedding metadata on open.
-    // They do validate the exact current derived generation before returning a
-    // handle, so no current reader can accidentally serve a populated legacy
-    // table and fail later on its first canonical-column query.
+    // Existing-DB callers do not mutate schema or embedding metadata on open;
+    // they serve an older layout as-is and refuse a newer one (see checkIndexLayout).
     //
     // "Existing" is load-bearing: a missing file throws instead of being
     // created. Create-on-open used to leave a schema-less index.db behind (a
@@ -126,15 +88,9 @@ export function openExistingDatabase(dbPath) {
     if (classifyPathAccess(resolvedPath).access === "absent") {
         throw new Error(`Index database not found at ${resolvedPath}. Run 'akm index' to build it.`);
     }
-    const db = openManagedDatabase({
-        path: resolvedPath,
-        init: (db) => {
-            loadVecExtension(db);
-        },
-        create: false,
-    });
+    const db = openManagedDatabase({ path: resolvedPath, create: false });
     try {
-        assertCanonicalIndexGeneration(db, resolvedPath);
+        checkIndexLayout(db, resolvedPath);
         return db;
     }
     catch (error) {
@@ -143,21 +99,29 @@ export function openExistingDatabase(dbPath) {
     }
 }
 /**
- * Read callers must never receive a known-incompatible derived index.  The
- * writable opener owns rebuilding an older generation; a reader can only
- * report the one action that is safe for the direction of the mismatch.
+ * A reader serves an older layout as-is (the FTS readers understand both
+ * layouts, and a missing table degrades at the caller — keyword-only search,
+ * an inline rebuild, or a "run akm index" notice) and names it once per
+ * process; the next writable open migrates it in place. A newer layout is
+ * refused, naming the upgrade ({@link newerIndexLayoutError}).
  */
-function assertCanonicalIndexGeneration(db, resolvedPath) {
-    if (isCanonicalIndexGeneration(db))
+function checkIndexLayout(db, resolvedPath) {
+    let stored;
+    try {
+        const row = db.prepare("SELECT value FROM index_meta WHERE key = 'version'").get();
+        if (!row)
+            return;
+        stored = Number(row.value);
+    }
+    catch {
         return;
-    const classification = classifyIndexGeneration(db);
-    const stored = classification.storedVersion ?? "unknown";
-    if (classification.status === "newer") {
-        throw new ConfigError(`Index database at ${resolvedPath} was built by a newer akm (stored generation ${stored}; ` +
-            `this binary understands ${CANONICAL_INDEX_DB_VERSION}). Upgrade akm to use this index.`, "INDEX_SCHEMA_INCOMPATIBLE", "Upgrade akm to a version that understands this index generation.");
     }
-    throw new ConfigError(`Index database at ${resolvedPath} is not usable with this akm's derived schema (stored generation ${stored}; ` +
-        `this binary understands ${CANONICAL_INDEX_DB_VERSION}). Run 'akm index' to rebuild it.`, "INDEX_SCHEMA_INCOMPATIBLE", "Run `akm index` to rebuild the derived index from the currently materialized sources.");
+    if (!Number.isFinite(stored) || stored === CANONICAL_INDEX_DB_VERSION)
+        return;
+    if (stored > CANONICAL_INDEX_DB_VERSION)
+        throw newerIndexLayoutError(stored, resolvedPath);
+    warnOnce(`index-db-layout:${resolvedPath}`, `Index database at ${resolvedPath} uses an older layout (${stored}; this akm writes ${CANONICAL_INDEX_DB_VERSION}). ` +
+        "Serving it as-is; the next 'akm index' migrates it in place.");
 }
 /**
  * Refuse to treat an UNREADABLE index as a missing one (#791).
@@ -219,7 +183,7 @@ export function openReadonlyExistingDatabase(dbPath, options) {
     // connection, so apply just that one.
     try {
         db.exec(`PRAGMA busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`);
-        assertCanonicalIndexGeneration(db, resolvedPath);
+        checkIndexLayout(db, resolvedPath);
         return db;
     }
     catch (error) {