akm-cli 0.9.0-beta.9 → 0.9.0-rc.1

package/dist/indexer/ensure-index.js

@@ -2,23 +2,29 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
- * Auto-index: silently run an incremental `akm index` when the local index
- * is stale or absent, so that `search`, `show`, and `feedback` always operate
- * against current on-disk state without requiring the user to manually run
- * `akm index` first.
+ * Auto-index bootstrap: silently build the local index inline when it cannot
+ * serve the caller's stash at all (missing DB, no `entries` table, zero rows,
+ * or built for a different stash), so `search`, `show`, and `feedback` work
+ * on first use without a manual `akm index`.
  *
- * This replaces the old filesystem fallbacks that were scattered across
- * `searchLocal()` and `show.ts`, centralizing the "indexed yet?" gap handling
- * behind a single entry point.
+ * Content FRESHNESS is intentionally not this module's job on the read path.
+ * Writers maintain the index (`indexWrittenAssets` for `remember`/extract
+ * session assets; the mutation commands run `akmIndex()` themselves), and the
+ * improve cron / explicit `akm index` do full refreshes. Reads serve whatever
+ * populated index exists. The previous design — a staleness walk plus a
+ * detached background reindex per read — made every read on an actively
+ * written stash spawn a writer that the read's own telemetry then queued
+ * behind (see docs/design/read-path-reindex-contention-findings.md).
+ *
+ * `mode: "blocking"` (improve) still checks staleness and rebuilds inline,
+ * because its planning logic needs a current `entries` table in-process.
  */
-import { spawn } from "node:child_process";
 import fs from "node:fs";
 import path from "node:path";
 import { ASSET_SPECS, TYPE_DIRS } from "../core/asset/asset-spec.js";
-import { getDataDir, getDbPath } from "../core/paths.js";
+import { getDbPath } from "../core/paths.js";
 import { warn } from "../core/warn.js";
 import { closeDatabase, getEntryCount, getIndexedFilePaths, getMeta, openExistingDatabase } from "./db/db.js";
-import { acquireIndexWriterLease, handoffIndexWriterLeaseToPid } from "./index-writer-lock.js";
 function getIndexableFiles(root, spec) {
     if (!fs.existsSync(root))
         return [];
@@ -138,12 +144,9 @@ export function isIndexStale(stashDir) {
  * i.e. the DB file exists, the `entries` table holds rows, and those rows were
  * built for this stash (it is the stored primary stash or appears in the
  * stored `stashDirs` set). When this is true the index is at worst
- * content-stale, so the `#607` background-reindex optimization is safe: the
- * caller gets slightly-stale-but-relevant results immediately. When it is
- * false the existing index has nothing relevant to return (no DB, no `entries`
- * table, zero rows, or built for a different stash), so a background reindex
- * would leave the caller empty until the next read — those cases must rebuild
- * inline.
+ * content-stale, so read paths serve it as-is. When it is false the existing
+ * index has nothing relevant to return (no DB, no `entries` table, zero rows,
+ * or built for a different stash), so those cases must rebuild inline.
  */
 function indexCanServeStash(stashDir) {
     const dbPath = getDbPath();
@@ -174,43 +177,6 @@ function indexCanServeStash(stashDir) {
             closeDatabase(db);
     }
 }
-/**
- * Spawn a background `akm index` process. Non-blocking — returns immediately.
- * Background callers share the same global index-writer lease as foreground
- * writers, so stale-read-triggered auto-index attempts coalesce safely.
- */
-async function spawnBackgroundReindex(_stashDir) {
-    const dataDir = getDataDir();
-    const logFile = path.join(dataDir, "logs", "index-background.log");
-    fs.mkdirSync(path.dirname(logFile), { recursive: true });
-    const lease = await acquireIndexWriterLease({ mode: "try", purpose: "background-reindex-spawn" });
-    if (!lease)
-        return;
-    const akmBin = process.argv[0];
-    const akmScript = process.argv[1];
-    try {
-        const child = spawn(akmBin, [akmScript, "index", "--background"], {
-            detached: true,
-            stdio: ["ignore", fs.openSync(logFile, "a"), fs.openSync(logFile, "a")],
-            env: { ...process.env },
-        });
-        if (!child.pid) {
-            lease.release();
-            return;
-        }
-        handoffIndexWriterLeaseToPid(lease, child.pid, "background-reindex");
-        try {
-            child.unref();
-        }
-        catch {
-            // ignore
-        }
-    }
-    catch (error) {
-        lease.release();
-        throw error;
-    }
-}
 async function runInlineReindex(stashDir) {
     try {
         const { akmIndex } = await import("./indexer.js");
@@ -219,53 +185,31 @@ async function runInlineReindex(stashDir) {
     }
     catch (error) {
         warn("Auto-index failed, proceeding with existing index:", error instanceof Error ? error.message : String(error));
-        return true;
+        return false;
     }
 }
 /**
- * Ensure the local index exists and is fresh enough for the caller's needs.
+ * Ensure the local index exists and can serve the caller.
  *
- * Default mode is `background`, which preserves the low-latency behavior used
- * by read paths (`search`, `show`, `feedback`): when a populated index is
- * merely stale, spawn a detached reindex and proceed against the existing
- * index. When the index is entirely absent (no DB / no `entries` table / zero
- * rows) the rebuild runs inline regardless of mode, since there is nothing to
- * proceed against.
+ * Default mode is `background` — the read-path contract (`search`, `show`,
+ * `feedback`): a populated index built for this stash is served as-is (its
+ * freshness is the writers' job, see module doc); an unusable index rebuilds
+ * inline, since there is nothing to proceed against.
  *
- * `mode: "blocking"` waits for the rebuild to finish before returning. Use
- * this for callers like `improve` whose planning logic depends on a populated
- * `entries` table in the same process.
+ * `mode: "blocking"` additionally treats content-staleness as a rebuild
+ * trigger and waits for it. Use this for callers like `improve` whose
+ * planning logic depends on a current `entries` table in the same process.
  *
- * Returns `true` if an index run was attempted.
+ * Returns `true` only when an inline index run succeeds.
+ * A rebuild attempt that fails (throws) resolves to `false`.
  */
 export async function ensureIndex(stashDir, options = {}) {
-    if (!isIndexStale(stashDir))
-        return false;
-    // Blocking when explicitly requested, or whenever the existing index cannot
-    // serve this stash (absent DB, no `entries` table, zero rows, or built for a
-    // different stash): a background reindex returns immediately and would leave
-    // a first-time caller (search, curate, wiki, show, feedback) with empty
-    // results. Building inline is a one-off cost; a populated index for this
-    // stash that is merely content-stale still refreshes in the background.
-    if (options.mode === "blocking" || !indexCanServeStash(stashDir)) {
-        return runInlineReindex(stashDir);
-    }
-    // The background path re-invokes the akm CLI as a detached child via
-    // `process.argv[1]`. That is only the akm entrypoint when THIS process is the
-    // akm CLI itself — which the CLI startup block signals with AKM_CLI_ENTRY=1.
-    // In any other host (the in-process test runner, a library embedding akm),
-    // argv[1] points at the host (e.g. the test runner), so spawning it would
-    // launch the wrong program and orphan it. Build inline there instead — same
-    // resulting index, no detached process.
-    if (process.env.AKM_CLI_ENTRY !== "1") {
+    if (options.mode === "blocking") {
+        if (!isIndexStale(stashDir))
+            return false;
         return runInlineReindex(stashDir);
     }
-    try {
-        await spawnBackgroundReindex(stashDir);
-        return true;
-    }
-    catch (error) {
-        warn("Background reindex spawn failed, proceeding with existing index:", error instanceof Error ? error.message : String(error));
-        return true;
-    }
+    if (indexCanServeStash(stashDir))
+        return false;
+    return runInlineReindex(stashDir);
 }

package/dist/indexer/feedback/utility-policy.js

@@ -0,0 +1,75 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * MemRL feedback → utility policy, extracted from indexer/db/db.ts.
+ *
+ * This is the domain/policy math (arXiv:2601.03192) that decides how a batch of
+ * positive/negative feedback signals moves an asset's utility score. It is pure
+ * — no database access — so the bounded-step behaviour is unit-testable in
+ * isolation; the DB read/write stays with `applyFeedbackToUtilityScore` in db.ts.
+ */
+/**
+ * MemRL learning rate for feedback-driven utility updates (F-5 / #386).
+ *
+ * Follows the bounded-step formula from MemRL (arXiv:2601.03192):
+ *   next = clamp(current + lr × (reward − current), 0, 1)
+ *
+ * This replaces the unbounded `-0.03 × negativeCount` delta that could
+ * silently remove high-utility assets from the improvement loop.
+ */
+export const FEEDBACK_LR = 0.1;
+/**
+ * Positive reward signal for a single positive feedback event.
+ * Reward 1.0 means "fully correct / helpful".
+ */
+const FEEDBACK_REWARD_POSITIVE = 1.0;
+/**
+ * Negative reward signal for a single negative feedback event.
+ * Reward 0.0 means "not helpful" (lowest MemRL signal).
+ */
+const FEEDBACK_REWARD_NEGATIVE = 0.0;
+/**
+ * Utility threshold below which a review-needed escalation is triggered.
+ * When a previously high-utility asset (≥ HIGH_UTILITY_THRESHOLD) drops
+ * below this value, the caller should create an escalation proposal.
+ */
+export const UTILITY_REVIEW_THRESHOLD = 0.5;
+/**
+ * Utility level considered "high" — assets above this are tracked for
+ * threshold-crossing escalation.
+ */
+export const HIGH_UTILITY_THRESHOLD = 0.5;
+/**
+ * Compute the next utility from accumulated feedback counts using the MemRL
+ * bounded-step EMA formula (F-5 / #386, arXiv:2601.03192):
+ *
+ *   reward   = weighted average of positive and negative signals
+ *   nextUtil = clamp(currentUtil + lr × (reward − currentUtil), 0, 1)
+ *
+ * The step is inherently bounded: reward ∈ [0, 1] and currentUtil ∈ [0, 1], so
+ * a single call moves utility by at most {@link FEEDBACK_LR} in either
+ * direction. `reward` is a proportion of the counts, not their magnitude, so
+ * with no positive signals the number of negatives is irrelevant (reward is 0
+ * whether there is 1 negative or 100). Mixing in positives shifts reward and so
+ * the step, but never past the learning-rate bound.
+ *
+ * Pure: no DB access. When both counts are zero, utility is unchanged.
+ */
+export function computeNextUtility(previousUtility, positiveCount, negativeCount) {
+    if (positiveCount === 0 && negativeCount === 0) {
+        return { previousUtility, nextUtility: previousUtility, crossedReviewThreshold: false };
+    }
+    const total = positiveCount + negativeCount;
+    // Weighted reward: proportion of positive signals.
+    const reward = positiveCount > 0 && negativeCount === 0
+        ? FEEDBACK_REWARD_POSITIVE
+        : negativeCount > 0 && positiveCount === 0
+            ? FEEDBACK_REWARD_NEGATIVE
+            : (positiveCount * FEEDBACK_REWARD_POSITIVE + negativeCount * FEEDBACK_REWARD_NEGATIVE) / total;
+    // MemRL bounded-step EMA: lr × (reward − current). |delta| ≤ FEEDBACK_LR.
+    const delta = FEEDBACK_LR * (reward - previousUtility);
+    const nextUtility = Math.max(0, Math.min(1, previousUtility + delta));
+    const crossedReviewThreshold = previousUtility >= HIGH_UTILITY_THRESHOLD && nextUtility < UTILITY_REVIEW_THRESHOLD;
+    return { previousUtility, nextUtility, crossedReviewThreshold };
+}

package/dist/indexer/graph/graph-boost.js

@@ -241,13 +241,18 @@ export function collectGraphRelatedHit(context, filePath) {
  * Find graph files that share entities with the given file.
  *
  * Implementation: SQL self-join on graph_file_entities, scoped by stash_root,
- * grouped by entry_id, ordered by shared-entity count desc. Touches ~50-200
+ * grouped by file_path, ordered by shared-entity count desc. Touches ~50-200
  * rows instead of loading the entire snapshot into memory. Cold-call latency
  * drops from ~30-60ms (full snapshot parse) to ~2-5ms on typical stashes.
  *
+ * #624-P1: the graph tables are keyed on (stash_root, file_path, body_hash) —
+ * NOT entries.id — so candidates are identified by file_path (the unique index
+ * idx_graph_files_path guarantees one graph_files row per path).
+ *
  * The returned `ref` field carries the canonical asset ref (`type:name`)
- * resolved from entries.entry_key when the entry is indexed. Callers should
- * fall back to formatting `path` when `ref` is undefined (orphan graph row).
+ * resolved from entries.entry_key when the file is indexed. Callers should
+ * fall back to formatting `path` when `ref` is undefined (graph row with no
+ * matching entries row).
  */
 export function listRelatedPathsForFile(stashRoot, filePath, limit = 5, db) {
     if (!db) {
@@ -255,113 +260,118 @@ export function listRelatedPathsForFile(stashRoot, filePath, limit = 5, db) {
         // callers pass a handle), so degrade to empty rather than reopening.
         return [];
     }
-    // Resolve target's entry_id from the stash_root + file_path. The graph rows
-    // are keyed on entry_id; without it we can't run the join.
-    let targetEntryId;
+    // Confirm the target file has a graph row; without it there is nothing to
+    // relate. (Identity is file_path within the stash — one row per path.)
     try {
         const row = db
-            .prepare("SELECT entry_id FROM graph_files WHERE stash_root = ? AND file_path = ? LIMIT 1")
+            .prepare("SELECT 1 AS present FROM graph_files WHERE stash_root = ? AND file_path = ? LIMIT 1")
             .get(stashRoot, filePath);
-        targetEntryId = row?.entry_id;
+        if (row === undefined)
+            return [];
     }
     catch {
         return [];
     }
-    if (targetEntryId == null)
-        return [];
     const effectiveLimit = Math.max(1, limit);
-    // Shared-entity count per candidate entry_id.
+    // Shared-entity count per candidate file_path. The target's entities are the
+    // rows for `filePath`; candidates are any OTHER file_path in the stash that
+    // shares a normalized entity.
     let candidateRows;
     try {
         candidateRows = db
-            .prepare(`SELECT gf.entry_id    AS entry_id,
-                gf.file_path   AS file_path,
+            .prepare(`SELECT gf.file_path   AS file_path,
                 gf.file_type   AS file_type,
                 COUNT(*)       AS shared
             FROM graph_file_entities target
             JOIN graph_file_entities e
               ON e.stash_root = target.stash_root
              AND e.entity_norm = target.entity_norm
-             AND e.entry_id  != target.entry_id
+             AND e.file_path  != target.file_path
             JOIN graph_files gf
-              ON gf.entry_id = e.entry_id
-          WHERE target.entry_id  = ?
+              ON gf.stash_root = e.stash_root
+             AND gf.file_path = e.file_path
+             AND gf.body_hash = e.body_hash
+          WHERE target.file_path  = ?
             AND target.stash_root = ?
-          GROUP BY gf.entry_id
+          GROUP BY gf.file_path
           ORDER BY shared DESC, gf.file_path ASC
           LIMIT ?`)
-            .all(targetEntryId, stashRoot, effectiveLimit);
+            .all(filePath, stashRoot, effectiveLimit);
     }
     catch {
         return [];
     }
     if (candidateRows.length === 0)
         return [];
-    const candidateIds = candidateRows.map((r) => r.entry_id);
-    const placeholders = candidateIds.map(() => "?").join(",");
+    const candidatePaths = candidateRows.map((r) => r.file_path);
+    const placeholders = candidatePaths.map(() => "?").join(",");
     // Pull the shared entity names (joined by normalized casing) for display.
     const sharedRows = db
-        .prepare(`SELECT e.entry_id AS entry_id, e.entity AS entity
+        .prepare(`SELECT e.file_path AS file_path, e.entity AS entity
          FROM graph_file_entities e
          JOIN graph_file_entities target
            ON target.stash_root = e.stash_root
            AND target.entity_norm = e.entity_norm
-         WHERE e.entry_id IN (${placeholders})
-           AND target.entry_id = ?
+         WHERE e.file_path IN (${placeholders})
+           AND e.stash_root = ?
+           AND target.file_path = ?
            AND target.stash_root = ?`)
-        .all(...candidateIds, targetEntryId, stashRoot);
-    const sharedByEntry = new Map();
+        .all(...candidatePaths, stashRoot, filePath, stashRoot);
+    const sharedByPath = new Map();
     for (const row of sharedRows) {
-        let bucket = sharedByEntry.get(row.entry_id);
+        let bucket = sharedByPath.get(row.file_path);
         if (!bucket) {
             bucket = new Set();
-            sharedByEntry.set(row.entry_id, bucket);
+            sharedByPath.set(row.file_path, bucket);
         }
         bucket.add(row.entity);
     }
     // Relation count for each candidate (relations where either endpoint
     // matches one of the shared entities).
-    const relationCountByEntry = new Map();
+    const relationCountByPath = new Map();
     const relationRows = db
-        .prepare(`SELECT entry_id, from_entity, to_entity
+        .prepare(`SELECT file_path, from_entity, to_entity
          FROM graph_file_relations
-        WHERE entry_id IN (${placeholders})`)
-        .all(...candidateIds);
+        WHERE file_path IN (${placeholders})
+          AND stash_root = ?`)
+        .all(...candidatePaths, stashRoot);
     for (const row of relationRows) {
-        const shared = sharedByEntry.get(row.entry_id);
+        const shared = sharedByPath.get(row.file_path);
         if (!shared)
             continue;
         if (shared.has(row.from_entity) || shared.has(row.to_entity)) {
-            relationCountByEntry.set(row.entry_id, (relationCountByEntry.get(row.entry_id) ?? 0) + 1);
+            relationCountByPath.set(row.file_path, (relationCountByPath.get(row.file_path) ?? 0) + 1);
         }
     }
     // Optional: ref lookup via entries.entry_key. entry_key is stored as
     // `${stash_dir}:${type}:${name}` — strip the stash-dir prefix to get the
-    // user-facing `type:name`.
-    const refByEntryId = new Map();
+    // user-facing `type:name`. Resolve by (stash_dir, file_path) now that the
+    // graph rows are no longer keyed on entries.id.
+    const refByPath = new Map();
     try {
         const entryRows = db
-            .prepare(`SELECT id, entry_key, stash_dir FROM entries WHERE id IN (${placeholders})`)
-            .all(...candidateIds);
+            .prepare(`SELECT entry_key, stash_dir, file_path FROM entries
+          WHERE file_path IN (${placeholders}) AND stash_dir = ?`)
+            .all(...candidatePaths, stashRoot);
         for (const row of entryRows) {
             const ref = stripStashPrefix(row.entry_key, row.stash_dir);
             if (ref)
-                refByEntryId.set(row.id, ref);
+                refByPath.set(row.file_path, ref);
         }
     }
     catch {
         /* ignore — refs are best-effort */
     }
     return candidateRows.map((row) => {
-        const sharedSet = sharedByEntry.get(row.entry_id) ?? new Set();
+        const sharedSet = sharedByPath.get(row.file_path) ?? new Set();
         const sharedEntities = [...sharedSet].sort((a, b) => a.localeCompare(b));
-        const ref = refByEntryId.get(row.entry_id);
+        const ref = refByPath.get(row.file_path);
         return {
             ...(ref ? { ref } : {}),
             path: row.file_path,
             type: row.file_type,
             sharedEntities,
-            relationCount: relationCountByEntry.get(row.entry_id) ?? 0,
+            relationCount: relationCountByPath.get(row.file_path) ?? 0,
         };
     });
 }