akm-cli 0.9.0-beta.6 → 0.9.0-rc.0

package/dist/indexer/ensure-index.js

@@ -2,21 +2,29 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
- * Auto-index: silently run an incremental `akm index` when the local index
- * is stale or absent, so that `search`, `show`, and `feedback` always operate
- * against current on-disk state without requiring the user to manually run
- * `akm index` first.
+ * Auto-index bootstrap: silently build the local index inline when it cannot
+ * serve the caller's stash at all (missing DB, no `entries` table, zero rows,
+ * or built for a different stash), so `search`, `show`, and `feedback` work
+ * on first use without a manual `akm index`.
  *
- * This replaces the old filesystem fallbacks that were scattered across
- * `searchLocal()` and `show.ts`, centralizing the "indexed yet?" gap handling
- * behind a single entry point.
+ * Content FRESHNESS is intentionally not this module's job on the read path.
+ * Writers maintain the index (`indexWrittenAssets` for `remember`/extract
+ * session assets; the mutation commands run `akmIndex()` themselves), and the
+ * improve cron / explicit `akm index` do full refreshes. Reads serve whatever
+ * populated index exists. The previous design — a staleness walk plus a
+ * detached background reindex per read — made every read on an actively
+ * written stash spawn a writer that the read's own telemetry then queued
+ * behind (see docs/design/read-path-reindex-contention-findings.md).
+ *
+ * `mode: "blocking"` (improve) still checks staleness and rebuilds inline,
+ * because its planning logic needs a current `entries` table in-process.
  */
 import fs from "node:fs";
 import path from "node:path";
 import { ASSET_SPECS, TYPE_DIRS } from "../core/asset/asset-spec.js";
 import { getDbPath } from "../core/paths.js";
 import { warn } from "../core/warn.js";
-import { closeDatabase, getEntryCount, getMeta, openExistingDatabase } from "./db/db.js";
+import { closeDatabase, getEntryCount, getIndexedFilePaths, getMeta, openExistingDatabase } from "./db/db.js";
 function getIndexableFiles(root, spec) {
     if (!fs.existsSync(root))
         return [];
@@ -52,16 +60,34 @@ function getIndexableFiles(root, spec) {
     }
     return files;
 }
-function hasNewerIndexableFiles(stashDir, builtAt) {
-    if (!builtAt)
-        return true;
-    const builtAtMs = new Date(builtAt).getTime();
-    if (!Number.isFinite(builtAtMs))
-        return true;
+/**
+ * Whether any indexable file under `stashDir` is newer than the last build, or
+ * has never been indexed at all.
+ *
+ * Two independent signals, because neither alone is sufficient:
+ *   1. **mtime > builtAt** — catches in-place *edits* of already-indexed files.
+ *   2. **path not in `indexedPaths`** — catches *newly added* files. This is
+ *      clock-independent on purpose: a freshly-written file can have a
+ *      filesystem mtime that compares as *older* than the wall-clock `builtAt`
+ *      (the two clocks are not perfectly synchronized and `builtAt` is
+ *      millisecond-truncated), so the mtime test alone silently misses
+ *      additions made within ~a millisecond of the previous build.
+ *
+ * `getIndexableFiles` applies each asset type's own relevance filter, so
+ * non-indexed companion files (e.g. `package.json` next to a knowledge doc) are
+ * never considered and do not produce false "new file" positives.
+ */
+function hasNewerIndexableFiles(stashDir, builtAt, indexedPaths) {
+    const builtAtMs = builtAt ? new Date(builtAt).getTime() : Number.NaN;
+    const builtAtUsable = Number.isFinite(builtAtMs);
     for (const [type, spec] of Object.entries(ASSET_SPECS)) {
         const typeRoot = path.join(stashDir, TYPE_DIRS[type] ?? spec.stashDir);
         const files = getIndexableFiles(typeRoot, spec);
         for (const file of files) {
+            if (!indexedPaths.has(file))
+                return true;
+            if (!builtAtUsable)
+                return true;
             try {
                 if (fs.statSync(file).mtimeMs > builtAtMs)
                     return true;
@@ -89,7 +115,7 @@ export function isIndexStale(stashDir) {
         if (entryCount === 0)
             return true;
         const builtAt = getMeta(db, "builtAt");
-        if (hasNewerIndexableFiles(stashDir, builtAt))
+        if (hasNewerIndexableFiles(stashDir, builtAt, getIndexedFilePaths(db)))
             return true;
         const storedStashDir = getMeta(db, "stashDir");
         if (storedStashDir !== stashDir) {
@@ -114,16 +140,44 @@ export function isIndexStale(stashDir) {
     }
 }
 /**
- * Run an incremental index when the local index is stale. Best-effort —
- * failures are logged as warnings but never thrown, so the caller can
- * proceed (and surface a proper "not in index" error if the index is
- * still unusable).
- *
- * Returns `true` if an index run was attempted.
+ * Whether the existing index can serve queries for `stashDir` *right now* —
+ * i.e. the DB file exists, the `entries` table holds rows, and those rows were
+ * built for this stash (it is the stored primary stash or appears in the
+ * stored `stashDirs` set). When this is true the index is at worst
+ * content-stale, so read paths serve it as-is. When it is false the existing
+ * index has nothing relevant to return (no DB, no `entries` table, zero rows,
+ * or built for a different stash), so those cases must rebuild inline.
  */
-export async function ensureIndex(stashDir) {
-    if (!isIndexStale(stashDir))
+function indexCanServeStash(stashDir) {
+    const dbPath = getDbPath();
+    if (!fs.existsSync(dbPath))
+        return false;
+    let db;
+    try {
+        db = openExistingDatabase(dbPath);
+        if (getEntryCount(db) === 0)
+            return false;
+        const storedStashDir = getMeta(db, "stashDir");
+        if (storedStashDir === stashDir)
+            return true;
+        try {
+            const storedDirs = JSON.parse(getMeta(db, "stashDirs") ?? "[]");
+            return storedDirs.includes(stashDir);
+        }
+        catch {
+            return false;
+        }
+    }
+    catch {
+        // No `entries` table (or otherwise unreadable) — cannot serve.
         return false;
+    }
+    finally {
+        if (db)
+            closeDatabase(db);
+    }
+}
+async function runInlineReindex(stashDir) {
     try {
         const { akmIndex } = await import("./indexer.js");
         await akmIndex({ stashDir });
@@ -131,6 +185,31 @@ export async function ensureIndex(stashDir) {
     }
     catch (error) {
         warn("Auto-index failed, proceeding with existing index:", error instanceof Error ? error.message : String(error));
-        return true;
+        return false;
+    }
+}
+/**
+ * Ensure the local index exists and can serve the caller.
+ *
+ * Default mode is `background` — the read-path contract (`search`, `show`,
+ * `feedback`): a populated index built for this stash is served as-is (its
+ * freshness is the writers' job, see module doc); an unusable index rebuilds
+ * inline, since there is nothing to proceed against.
+ *
+ * `mode: "blocking"` additionally treats content-staleness as a rebuild
+ * trigger and waits for it. Use this for callers like `improve` whose
+ * planning logic depends on a current `entries` table in the same process.
+ *
+ * Returns `true` only when an inline index run succeeds.
+ * A rebuild attempt that fails (throws) resolves to `false`.
+ */
+export async function ensureIndex(stashDir, options = {}) {
+    if (options.mode === "blocking") {
+        if (!isIndexStale(stashDir))
+            return false;
+        return runInlineReindex(stashDir);
     }
+    if (indexCanServeStash(stashDir))
+        return false;
+    return runInlineReindex(stashDir);
 }

package/dist/indexer/feedback/utility-policy.js

@@ -0,0 +1,75 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * MemRL feedback → utility policy, extracted from indexer/db/db.ts.
+ *
+ * This is the domain/policy math (arXiv:2601.03192) that decides how a batch of
+ * positive/negative feedback signals moves an asset's utility score. It is pure
+ * — no database access — so the bounded-step behaviour is unit-testable in
+ * isolation; the DB read/write stays with `applyFeedbackToUtilityScore` in db.ts.
+ */
+/**
+ * MemRL learning rate for feedback-driven utility updates (F-5 / #386).
+ *
+ * Follows the bounded-step formula from MemRL (arXiv:2601.03192):
+ *   next = clamp(current + lr × (reward − current), 0, 1)
+ *
+ * This replaces the unbounded `-0.03 × negativeCount` delta that could
+ * silently remove high-utility assets from the improvement loop.
+ */
+export const FEEDBACK_LR = 0.1;
+/**
+ * Positive reward signal for a single positive feedback event.
+ * Reward 1.0 means "fully correct / helpful".
+ */
+const FEEDBACK_REWARD_POSITIVE = 1.0;
+/**
+ * Negative reward signal for a single negative feedback event.
+ * Reward 0.0 means "not helpful" (lowest MemRL signal).
+ */
+const FEEDBACK_REWARD_NEGATIVE = 0.0;
+/**
+ * Utility threshold below which a review-needed escalation is triggered.
+ * When a previously high-utility asset (≥ HIGH_UTILITY_THRESHOLD) drops
+ * below this value, the caller should create an escalation proposal.
+ */
+export const UTILITY_REVIEW_THRESHOLD = 0.5;
+/**
+ * Utility level considered "high" — assets above this are tracked for
+ * threshold-crossing escalation.
+ */
+export const HIGH_UTILITY_THRESHOLD = 0.5;
+/**
+ * Compute the next utility from accumulated feedback counts using the MemRL
+ * bounded-step EMA formula (F-5 / #386, arXiv:2601.03192):
+ *
+ *   reward   = weighted average of positive and negative signals
+ *   nextUtil = clamp(currentUtil + lr × (reward − currentUtil), 0, 1)
+ *
+ * The step is inherently bounded: reward ∈ [0, 1] and currentUtil ∈ [0, 1], so
+ * a single call moves utility by at most {@link FEEDBACK_LR} in either
+ * direction. `reward` is a proportion of the counts, not their magnitude, so
+ * with no positive signals the number of negatives is irrelevant (reward is 0
+ * whether there is 1 negative or 100). Mixing in positives shifts reward and so
+ * the step, but never past the learning-rate bound.
+ *
+ * Pure: no DB access. When both counts are zero, utility is unchanged.
+ */
+export function computeNextUtility(previousUtility, positiveCount, negativeCount) {
+    if (positiveCount === 0 && negativeCount === 0) {
+        return { previousUtility, nextUtility: previousUtility, crossedReviewThreshold: false };
+    }
+    const total = positiveCount + negativeCount;
+    // Weighted reward: proportion of positive signals.
+    const reward = positiveCount > 0 && negativeCount === 0
+        ? FEEDBACK_REWARD_POSITIVE
+        : negativeCount > 0 && positiveCount === 0
+            ? FEEDBACK_REWARD_NEGATIVE
+            : (positiveCount * FEEDBACK_REWARD_POSITIVE + negativeCount * FEEDBACK_REWARD_NEGATIVE) / total;
+    // MemRL bounded-step EMA: lr × (reward − current). |delta| ≤ FEEDBACK_LR.
+    const delta = FEEDBACK_LR * (reward - previousUtility);
+    const nextUtility = Math.max(0, Math.min(1, previousUtility + delta));
+    const crossedReviewThreshold = previousUtility >= HIGH_UTILITY_THRESHOLD && nextUtility < UTILITY_REVIEW_THRESHOLD;
+    return { previousUtility, nextUtility, crossedReviewThreshold };
+}

package/dist/indexer/graph/graph-boost.js

@@ -241,13 +241,18 @@ export function collectGraphRelatedHit(context, filePath) {
  * Find graph files that share entities with the given file.
  *
  * Implementation: SQL self-join on graph_file_entities, scoped by stash_root,
- * grouped by entry_id, ordered by shared-entity count desc. Touches ~50-200
+ * grouped by file_path, ordered by shared-entity count desc. Touches ~50-200
  * rows instead of loading the entire snapshot into memory. Cold-call latency
  * drops from ~30-60ms (full snapshot parse) to ~2-5ms on typical stashes.
  *
+ * #624-P1: the graph tables are keyed on (stash_root, file_path, body_hash) —
+ * NOT entries.id — so candidates are identified by file_path (the unique index
+ * idx_graph_files_path guarantees one graph_files row per path).
+ *
  * The returned `ref` field carries the canonical asset ref (`type:name`)
- * resolved from entries.entry_key when the entry is indexed. Callers should
- * fall back to formatting `path` when `ref` is undefined (orphan graph row).
+ * resolved from entries.entry_key when the file is indexed. Callers should
+ * fall back to formatting `path` when `ref` is undefined (graph row with no
+ * matching entries row).
  */
 export function listRelatedPathsForFile(stashRoot, filePath, limit = 5, db) {
     if (!db) {
@@ -255,113 +260,118 @@ export function listRelatedPathsForFile(stashRoot, filePath, limit = 5, db) {
         // callers pass a handle), so degrade to empty rather than reopening.
         return [];
     }
-    // Resolve target's entry_id from the stash_root + file_path. The graph rows
-    // are keyed on entry_id; without it we can't run the join.
-    let targetEntryId;
+    // Confirm the target file has a graph row; without it there is nothing to
+    // relate. (Identity is file_path within the stash — one row per path.)
     try {
         const row = db
-            .prepare("SELECT entry_id FROM graph_files WHERE stash_root = ? AND file_path = ? LIMIT 1")
+            .prepare("SELECT 1 AS present FROM graph_files WHERE stash_root = ? AND file_path = ? LIMIT 1")
             .get(stashRoot, filePath);
-        targetEntryId = row?.entry_id;
+        if (row === undefined)
+            return [];
     }
     catch {
         return [];
     }
-    if (targetEntryId == null)
-        return [];
     const effectiveLimit = Math.max(1, limit);
-    // Shared-entity count per candidate entry_id.
+    // Shared-entity count per candidate file_path. The target's entities are the
+    // rows for `filePath`; candidates are any OTHER file_path in the stash that
+    // shares a normalized entity.
     let candidateRows;
     try {
         candidateRows = db
-            .prepare(`SELECT gf.entry_id    AS entry_id,
-                gf.file_path   AS file_path,
+            .prepare(`SELECT gf.file_path   AS file_path,
                 gf.file_type   AS file_type,
                 COUNT(*)       AS shared
             FROM graph_file_entities target
             JOIN graph_file_entities e
               ON e.stash_root = target.stash_root
              AND e.entity_norm = target.entity_norm
-             AND e.entry_id  != target.entry_id
+             AND e.file_path  != target.file_path
             JOIN graph_files gf
-              ON gf.entry_id = e.entry_id
-          WHERE target.entry_id  = ?
+              ON gf.stash_root = e.stash_root
+             AND gf.file_path = e.file_path
+             AND gf.body_hash = e.body_hash
+          WHERE target.file_path  = ?
             AND target.stash_root = ?
-          GROUP BY gf.entry_id
+          GROUP BY gf.file_path
           ORDER BY shared DESC, gf.file_path ASC
           LIMIT ?`)
-            .all(targetEntryId, stashRoot, effectiveLimit);
+            .all(filePath, stashRoot, effectiveLimit);
     }
     catch {
         return [];
     }
     if (candidateRows.length === 0)
         return [];
-    const candidateIds = candidateRows.map((r) => r.entry_id);
-    const placeholders = candidateIds.map(() => "?").join(",");
+    const candidatePaths = candidateRows.map((r) => r.file_path);
+    const placeholders = candidatePaths.map(() => "?").join(",");
     // Pull the shared entity names (joined by normalized casing) for display.
     const sharedRows = db
-        .prepare(`SELECT e.entry_id AS entry_id, e.entity AS entity
+        .prepare(`SELECT e.file_path AS file_path, e.entity AS entity
          FROM graph_file_entities e
          JOIN graph_file_entities target
            ON target.stash_root = e.stash_root
            AND target.entity_norm = e.entity_norm
-         WHERE e.entry_id IN (${placeholders})
-           AND target.entry_id = ?
+         WHERE e.file_path IN (${placeholders})
+           AND e.stash_root = ?
+           AND target.file_path = ?
            AND target.stash_root = ?`)
-        .all(...candidateIds, targetEntryId, stashRoot);
-    const sharedByEntry = new Map();
+        .all(...candidatePaths, stashRoot, filePath, stashRoot);
+    const sharedByPath = new Map();
     for (const row of sharedRows) {
-        let bucket = sharedByEntry.get(row.entry_id);
+        let bucket = sharedByPath.get(row.file_path);
         if (!bucket) {
             bucket = new Set();
-            sharedByEntry.set(row.entry_id, bucket);
+            sharedByPath.set(row.file_path, bucket);
         }
         bucket.add(row.entity);
     }
     // Relation count for each candidate (relations where either endpoint
     // matches one of the shared entities).
-    const relationCountByEntry = new Map();
+    const relationCountByPath = new Map();
     const relationRows = db
-        .prepare(`SELECT entry_id, from_entity, to_entity
+        .prepare(`SELECT file_path, from_entity, to_entity
          FROM graph_file_relations
-        WHERE entry_id IN (${placeholders})`)
-        .all(...candidateIds);
+        WHERE file_path IN (${placeholders})
+          AND stash_root = ?`)
+        .all(...candidatePaths, stashRoot);
     for (const row of relationRows) {
-        const shared = sharedByEntry.get(row.entry_id);
+        const shared = sharedByPath.get(row.file_path);
         if (!shared)
             continue;
         if (shared.has(row.from_entity) || shared.has(row.to_entity)) {
-            relationCountByEntry.set(row.entry_id, (relationCountByEntry.get(row.entry_id) ?? 0) + 1);
+            relationCountByPath.set(row.file_path, (relationCountByPath.get(row.file_path) ?? 0) + 1);
         }
     }
     // Optional: ref lookup via entries.entry_key. entry_key is stored as
     // `${stash_dir}:${type}:${name}` — strip the stash-dir prefix to get the
-    // user-facing `type:name`.
-    const refByEntryId = new Map();
+    // user-facing `type:name`. Resolve by (stash_dir, file_path) now that the
+    // graph rows are no longer keyed on entries.id.
+    const refByPath = new Map();
     try {
         const entryRows = db
-            .prepare(`SELECT id, entry_key, stash_dir FROM entries WHERE id IN (${placeholders})`)
-            .all(...candidateIds);
+            .prepare(`SELECT entry_key, stash_dir, file_path FROM entries
+          WHERE file_path IN (${placeholders}) AND stash_dir = ?`)
+            .all(...candidatePaths, stashRoot);
         for (const row of entryRows) {
             const ref = stripStashPrefix(row.entry_key, row.stash_dir);
             if (ref)
-                refByEntryId.set(row.id, ref);
+                refByPath.set(row.file_path, ref);
         }
     }
     catch {
         /* ignore — refs are best-effort */
     }
     return candidateRows.map((row) => {
-        const sharedSet = sharedByEntry.get(row.entry_id) ?? new Set();
+        const sharedSet = sharedByPath.get(row.file_path) ?? new Set();
         const sharedEntities = [...sharedSet].sort((a, b) => a.localeCompare(b));
-        const ref = refByEntryId.get(row.entry_id);
+        const ref = refByPath.get(row.file_path);
         return {
             ...(ref ? { ref } : {}),
             path: row.file_path,
             type: row.file_type,
             sharedEntities,
-            relationCount: relationCountByEntry.get(row.entry_id) ?? 0,
+            relationCount: relationCountByPath.get(row.file_path) ?? 0,
         };
     });
 }