akm-cli 0.9.0-beta.3 → 0.9.0-beta.31

package/dist/indexer/graph/graph-boost.js

@@ -241,13 +241,18 @@ export function collectGraphRelatedHit(context, filePath) {
  * Find graph files that share entities with the given file.
  *
  * Implementation: SQL self-join on graph_file_entities, scoped by stash_root,
- * grouped by entry_id, ordered by shared-entity count desc. Touches ~50-200
+ * grouped by file_path, ordered by shared-entity count desc. Touches ~50-200
  * rows instead of loading the entire snapshot into memory. Cold-call latency
  * drops from ~30-60ms (full snapshot parse) to ~2-5ms on typical stashes.
  *
+ * #624-P1: the graph tables are keyed on (stash_root, file_path, body_hash) —
+ * NOT entries.id — so candidates are identified by file_path (the unique index
+ * idx_graph_files_path guarantees one graph_files row per path).
+ *
  * The returned `ref` field carries the canonical asset ref (`type:name`)
- * resolved from entries.entry_key when the entry is indexed. Callers should
- * fall back to formatting `path` when `ref` is undefined (orphan graph row).
+ * resolved from entries.entry_key when the file is indexed. Callers should
+ * fall back to formatting `path` when `ref` is undefined (graph row with no
+ * matching entries row).
  */
 export function listRelatedPathsForFile(stashRoot, filePath, limit = 5, db) {
     if (!db) {
@@ -255,113 +260,118 @@ export function listRelatedPathsForFile(stashRoot, filePath, limit = 5, db) {
         // callers pass a handle), so degrade to empty rather than reopening.
         return [];
     }
-    // Resolve target's entry_id from the stash_root + file_path. The graph rows
-    // are keyed on entry_id; without it we can't run the join.
-    let targetEntryId;
+    // Confirm the target file has a graph row; without it there is nothing to
+    // relate. (Identity is file_path within the stash — one row per path.)
     try {
         const row = db
-            .prepare("SELECT entry_id FROM graph_files WHERE stash_root = ? AND file_path = ? LIMIT 1")
+            .prepare("SELECT 1 AS present FROM graph_files WHERE stash_root = ? AND file_path = ? LIMIT 1")
             .get(stashRoot, filePath);
-        targetEntryId = row?.entry_id;
+        if (row === undefined)
+            return [];
     }
     catch {
         return [];
     }
-    if (targetEntryId == null)
-        return [];
     const effectiveLimit = Math.max(1, limit);
-    // Shared-entity count per candidate entry_id.
+    // Shared-entity count per candidate file_path. The target's entities are the
+    // rows for `filePath`; candidates are any OTHER file_path in the stash that
+    // shares a normalized entity.
     let candidateRows;
     try {
         candidateRows = db
-            .prepare(`SELECT gf.entry_id    AS entry_id,
-                gf.file_path   AS file_path,
+            .prepare(`SELECT gf.file_path   AS file_path,
                 gf.file_type   AS file_type,
                 COUNT(*)       AS shared
             FROM graph_file_entities target
             JOIN graph_file_entities e
               ON e.stash_root = target.stash_root
              AND e.entity_norm = target.entity_norm
-             AND e.entry_id  != target.entry_id
+             AND e.file_path  != target.file_path
             JOIN graph_files gf
-              ON gf.entry_id = e.entry_id
-          WHERE target.entry_id  = ?
+              ON gf.stash_root = e.stash_root
+             AND gf.file_path = e.file_path
+             AND gf.body_hash = e.body_hash
+          WHERE target.file_path  = ?
             AND target.stash_root = ?
-          GROUP BY gf.entry_id
+          GROUP BY gf.file_path
           ORDER BY shared DESC, gf.file_path ASC
           LIMIT ?`)
-            .all(targetEntryId, stashRoot, effectiveLimit);
+            .all(filePath, stashRoot, effectiveLimit);
     }
     catch {
         return [];
     }
     if (candidateRows.length === 0)
         return [];
-    const candidateIds = candidateRows.map((r) => r.entry_id);
-    const placeholders = candidateIds.map(() => "?").join(",");
+    const candidatePaths = candidateRows.map((r) => r.file_path);
+    const placeholders = candidatePaths.map(() => "?").join(",");
     // Pull the shared entity names (joined by normalized casing) for display.
     const sharedRows = db
-        .prepare(`SELECT e.entry_id AS entry_id, e.entity AS entity
+        .prepare(`SELECT e.file_path AS file_path, e.entity AS entity
          FROM graph_file_entities e
          JOIN graph_file_entities target
            ON target.stash_root = e.stash_root
            AND target.entity_norm = e.entity_norm
-         WHERE e.entry_id IN (${placeholders})
-           AND target.entry_id = ?
+         WHERE e.file_path IN (${placeholders})
+           AND e.stash_root = ?
+           AND target.file_path = ?
            AND target.stash_root = ?`)
-        .all(...candidateIds, targetEntryId, stashRoot);
-    const sharedByEntry = new Map();
+        .all(...candidatePaths, stashRoot, filePath, stashRoot);
+    const sharedByPath = new Map();
     for (const row of sharedRows) {
-        let bucket = sharedByEntry.get(row.entry_id);
+        let bucket = sharedByPath.get(row.file_path);
         if (!bucket) {
             bucket = new Set();
-            sharedByEntry.set(row.entry_id, bucket);
+            sharedByPath.set(row.file_path, bucket);
         }
         bucket.add(row.entity);
     }
     // Relation count for each candidate (relations where either endpoint
     // matches one of the shared entities).
-    const relationCountByEntry = new Map();
+    const relationCountByPath = new Map();
     const relationRows = db
-        .prepare(`SELECT entry_id, from_entity, to_entity
+        .prepare(`SELECT file_path, from_entity, to_entity
          FROM graph_file_relations
-        WHERE entry_id IN (${placeholders})`)
-        .all(...candidateIds);
+        WHERE file_path IN (${placeholders})
+          AND stash_root = ?`)
+        .all(...candidatePaths, stashRoot);
     for (const row of relationRows) {
-        const shared = sharedByEntry.get(row.entry_id);
+        const shared = sharedByPath.get(row.file_path);
         if (!shared)
             continue;
         if (shared.has(row.from_entity) || shared.has(row.to_entity)) {
-            relationCountByEntry.set(row.entry_id, (relationCountByEntry.get(row.entry_id) ?? 0) + 1);
+            relationCountByPath.set(row.file_path, (relationCountByPath.get(row.file_path) ?? 0) + 1);
         }
     }
     // Optional: ref lookup via entries.entry_key. entry_key is stored as
     // `${stash_dir}:${type}:${name}` — strip the stash-dir prefix to get the
-    // user-facing `type:name`.
-    const refByEntryId = new Map();
+    // user-facing `type:name`. Resolve by (stash_dir, file_path) now that the
+    // graph rows are no longer keyed on entries.id.
+    const refByPath = new Map();
     try {
         const entryRows = db
-            .prepare(`SELECT id, entry_key, stash_dir FROM entries WHERE id IN (${placeholders})`)
-            .all(...candidateIds);
+            .prepare(`SELECT entry_key, stash_dir, file_path FROM entries
+          WHERE file_path IN (${placeholders}) AND stash_dir = ?`)
+            .all(...candidatePaths, stashRoot);
         for (const row of entryRows) {
             const ref = stripStashPrefix(row.entry_key, row.stash_dir);
             if (ref)
-                refByEntryId.set(row.id, ref);
+                refByPath.set(row.file_path, ref);
         }
     }
     catch {
         /* ignore — refs are best-effort */
     }
     return candidateRows.map((row) => {
-        const sharedSet = sharedByEntry.get(row.entry_id) ?? new Set();
+        const sharedSet = sharedByPath.get(row.file_path) ?? new Set();
         const sharedEntities = [...sharedSet].sort((a, b) => a.localeCompare(b));
-        const ref = refByEntryId.get(row.entry_id);
+        const ref = refByPath.get(row.file_path);
         return {
             ...(ref ? { ref } : {}),
             path: row.file_path,
             type: row.file_type,
             sharedEntities,
-            relationCount: relationCountByEntry.get(row.entry_id) ?? 0,
+            relationCount: relationCountByPath.get(row.file_path) ?? 0,
         };
     });
 }

package/dist/indexer/graph/graph-extraction.js

@@ -40,13 +40,14 @@ import path from "node:path";
 import { TYPE_DIRS } from "../../core/asset/asset-spec.js";
 import { parseFrontmatter } from "../../core/asset/frontmatter.js";
 import { concurrentMap } from "../../core/concurrent.js";
-import { getIndexPassConfig, resolveBatchSize } from "../../core/config/config.js";
+import { getIndexPassConfig, loadConfig, resolveBatchSize } from "../../core/config/config.js";
+import { rethrowIfTestIsolationError } from "../../core/errors.js";
 import { warn, warnVerbose } from "../../core/warn.js";
 import { isProcessEnabled } from "../../llm/feature-gate.js";
 import * as graphExtract from "../../llm/graph-extract.js";
 import { resolveIndexPassLLM } from "../../llm/index-passes.js";
 import { computeBodyHash, GRAPH_SCHEMA_VERSION, getLlmCacheEntriesByRefs, getLlmCacheEntry, upsertLlmCacheEntry, } from "../db/db.js";
-import { loadStoredGraphSnapshot, replaceStoredGraph } from "../db/graph-db.js";
+import { drainExtractionQueue, loadStoredGraphSnapshot, replaceStoredGraph } from "../db/graph-db.js";
 import { walkMarkdownFiles } from "../walk/walker.js";
 import { deduplicateGraph } from "./graph-dedup.js";
 /** Schema version for the persisted artifact — bumps trigger a full rebuild. */
@@ -92,6 +93,12 @@ function computeGraphQualityTelemetry(consideredFiles, extractedFiles, entityCou
     };
 }
 export const DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES = ["memory", "knowledge"];
+/**
+ * Max number of lazy-extraction queue rows drained per pass (#624-P3). Bounds
+ * per-run work so a large backlog is spread across runs rather than processed
+ * all at once. Generous default — the queue is normally near-empty.
+ */
+const GRAPH_EXTRACTION_QUEUE_DRAIN_LIMIT = 100;
 const SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES = new Set([
     "memory",
     "knowledge",
@@ -291,8 +298,29 @@ export async function runGraphExtractionPass(ctx) {
         warnVerbose("graph extraction: skipped because no primary stash source is available.");
         return { ...EMPTY_RESULT };
     }
+    // #624-P3: drain the lazy-extraction queue BEFORE the ranked sweep, highest
+    // priority first. Queued paths are extracted individually (per-file merge,
+    // other files untouched) so they are processed even when they fall outside
+    // the normal candidate set. Default (empty queue) is a byte-identical no-op:
+    // drainExtractionQueue returns [] and the loop body never runs.
+    if (db) {
+        const drained = drainExtractionQueue(db, primary.path, GRAPH_EXTRACTION_QUEUE_DRAIN_LIMIT);
+        for (const queued of drained) {
+            if (signal?.aborted)
+                break;
+            await extractGraphForSingleFile(db, primary.path, queued.filePath, queued.bodyHash, { config, signal });
+        }
+    }
     const includeTypes = getGraphExtractionIncludeTypes(config);
-    const eligible = collectEligibleFiles(primary.path, includeTypes).filter((candidate) => !options.candidatePaths || options.candidatePaths.has(candidate.absPath));
+    let eligible = collectEligibleFiles(primary.path, includeTypes).filter((candidate) => !options.candidatePaths || options.candidatePaths.has(candidate.absPath));
+    // P2 (#624): when topN is set and a DB is available, rank the (already
+    // candidate-filtered) eligible set by utility_scores DESC and keep only the
+    // top-N. Default (topN unset) is byte-identical to today — no ranking query
+    // is issued and the eligible set is untouched. Ranking composes WITH the
+    // candidatePaths filter: scoped-then-ranked-then-sliced.
+    if (db && options.topN != null && options.topN >= 0) {
+        eligible = rankCandidatesByUtility(db, eligible, primary.path).slice(0, options.topN);
+    }
     const considered = eligible.length;
     if (considered === 0) {
         const scoped = options.candidatePaths ? ` matching ${options.candidatePaths.size} candidate path(s)` : "";
@@ -628,6 +656,178 @@ export async function runGraphExtractionPass(ctx) {
         warnings,
     };
 }
+/**
+ * Infer the asset type (`memory`, `knowledge`, …) for a path from the stash
+ * directory segment it lives under. Returns the matching include-type, or
+ * `undefined` when the path is not under a known graph-eligible type dir.
+ */
+function inferGraphTypeForPath(stashRoot, absPath) {
+    const rel = path.relative(stashRoot, absPath);
+    const firstSeg = rel.split(path.sep)[0];
+    if (!firstSeg)
+        return undefined;
+    for (const type of SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES) {
+        if (TYPE_DIRS[type] === firstSeg)
+            return type;
+    }
+    return undefined;
+}
+/**
+ * #624-P3 — extract graph data for a SINGLE file and merge it into the stored
+ * graph WITHOUT clobbering other files' rows.
+ *
+ * Re-reads the body from disk at call time (the queued body_hash is NOT trusted
+ * blindly — the file may have been deleted or changed since enqueue) and skips
+ * silently (returns `false`) when the file is gone or empty. Resolves the LLM
+ * via {@link resolveIndexPassLLM} (model-available guard: returns `false` when
+ * no provider is configured) UNLESS `opts.llmOverride` is supplied, in which
+ * case the override is the extractor seam (used by tests and by callers that
+ * already hold a resolved model).
+ *
+ * Returns `true` when a graph row was written for the file, `false` on any
+ * skip (missing file, empty body, unknown type, no model, or extraction error).
+ */
+export async function extractGraphForSingleFile(db, stashRoot, filePath, bodyHash, opts) {
+    try {
+        // Re-read from disk — never trust a stale queued body.
+        let raw;
+        try {
+            raw = fs.readFileSync(filePath, "utf8");
+        }
+        catch {
+            return false; // file gone / unreadable → silent skip
+        }
+        const parsed = parseFrontmatter(raw);
+        const body = parsed.content.trim();
+        if (!body)
+            return false;
+        const type = inferGraphTypeForPath(stashRoot, filePath) ?? "memory";
+        const effectiveHash = bodyHash ?? computeBodyHash(body);
+        // Extract — via the injected seam, or the real per-asset path.
+        let extraction;
+        if (opts?.llmOverride) {
+            const out = await opts.llmOverride(body);
+            extraction = {
+                entities: out.entities,
+                relations: out.relations,
+                ...(out.confidence !== undefined ? { confidence: out.confidence } : {}),
+            };
+        }
+        else {
+            const config = opts?.config ?? loadConfig();
+            if (!isProcessEnabled("index", "graph_extraction", config))
+                return false;
+            const llmConfig = resolveIndexPassLLM("graph", config);
+            if (!llmConfig)
+                return false; // model-available guard
+            const result = await graphExtract.extractGraphFromBody(llmConfig, body, opts?.signal, config);
+            extraction = {
+                entities: result.entities,
+                relations: result.relations,
+                ...(result.confidence !== undefined ? { confidence: result.confidence } : {}),
+            };
+        }
+        const entities = [...new Set(extraction.entities.map((e) => e.trim()).filter(Boolean))];
+        const relations = extraction.relations
+            .map((r) => ({
+            from: r.from.trim(),
+            to: r.to.trim(),
+            ...(r.type ? { type: r.type.trim() } : {}),
+            ...(normalizeConfidence(r.confidence) !== undefined ? { confidence: normalizeConfidence(r.confidence) } : {}),
+        }))
+            .filter((r) => r.from && r.to);
+        const node = {
+            path: filePath,
+            type,
+            bodyHash: effectiveHash,
+            entities,
+            relations,
+            ...(normalizeConfidence(extraction.confidence) !== undefined
+                ? { confidence: normalizeConfidence(extraction.confidence) }
+                : {}),
+            status: entities.length > 0 ? "extracted" : "empty",
+            reason: entities.length > 0 ? "none" : "no_graph_content",
+            extractionRunId: crypto.randomUUID(),
+        };
+        // Merge with the previously-stored nodes, scoping the refresh to JUST this
+        // path so other files' rows are preserved (and graph_meta counts refresh).
+        const previousGraph = loadGraphFile(stashRoot, db);
+        const candidatePaths = new Set([filePath]);
+        const mergedNodes = mergeGraphNodes(previousGraph.files, [node], candidatePaths);
+        const assetRefs = mergedNodes.map((n) => n.path);
+        const deduped = deduplicateGraph(mergedNodes.map((n) => ({ entities: n.entities, relations: n.relations })), assetRefs);
+        const qualityExtracted = mergedNodes.filter((n) => n.status === "extracted" && n.entities.length > 0).length;
+        const quality = computeGraphQualityTelemetry(mergedNodes.length, qualityExtracted, deduped.entities.length, deduped.relations.length);
+        const graph = {
+            schemaVersion: GRAPH_FILE_SCHEMA_VERSION,
+            generatedAt: new Date().toISOString(),
+            stashRoot,
+            files: mergedNodes,
+            entities: deduped.entities,
+            relations: deduped.relations,
+            quality,
+            ...(previousGraph.telemetry ? { telemetry: previousGraph.telemetry } : {}),
+        };
+        return writeGraphFile(stashRoot, graph, db);
+    }
+    catch (err) {
+        rethrowIfTestIsolationError(err);
+        return false;
+    }
+}
+// ── Eligible-file detection ─────────────────────────────────────────────────
+/**
+ * Rank eligible graph-extraction candidates by their entry `utility_scores`,
+ * highest first, for the incremental high-signal-first sweep (P2 of #624).
+ *
+ * The join is READ-ONLY (`entries.file_path = candidate.absPath`, then
+ * `entries.id -> utility_scores.entry_id`) and does NOT re-couple the graph
+ * rows to `entries`. It reads the GLOBAL `utility_scores` table (not the
+ * per-scope `utility_scores_scoped`), so ranking is corpus-wide; `stashRoot`
+ * is accepted for call-site symmetry/future scoping but is not used to filter
+ * (the global table has no `stash_root` column).
+ *
+ * Candidates with no matching `entries` row, or an entry with no
+ * `utility_scores` row, get an effective utility of 0 (LEFT JOIN + COALESCE)
+ * and sort LAST — they are deprioritized, never dropped, so a `topN >= total`
+ * slice still includes them and they remain reachable on later runs.
+ *
+ * Ties (equal utility) break by `file_path` ASC for deterministic output.
+ * Returns a NEW array; the input is not mutated. SQLite's ~999 bound-parameter
+ * cap is respected by chunking the `IN (...)` lookup at 500.
+ *
+ * Exported for direct unit testing.
+ */
+export function rankCandidatesByUtility(db, candidates, _stashRoot) {
+    // Cannot rank without a DB → return the input unranked rather than throw.
+    // Keeps the DB-less code path (reuse-from-memory) working when topN is set.
+    if (!db || candidates.length === 0)
+        return candidates;
+    const utilityByPath = new Map();
+    const CHUNK = 500;
+    for (let start = 0; start < candidates.length; start += CHUNK) {
+        const chunk = candidates.slice(start, start + CHUNK);
+        const paths = chunk.map((c) => c.absPath);
+        const placeholders = paths.map(() => "?").join(", ");
+        const rows = db
+            .prepare(`SELECT e.file_path AS file_path, COALESCE(MAX(u.utility), 0) AS utility
+           FROM entries e
+           LEFT JOIN utility_scores u ON u.entry_id = e.id
+          WHERE e.file_path IN (${placeholders})
+          GROUP BY e.file_path`)
+            .all(...paths);
+        for (const row of rows) {
+            utilityByPath.set(row.file_path, row.utility ?? 0);
+        }
+    }
+    return [...candidates].sort((a, b) => {
+        const ua = utilityByPath.get(a.absPath) ?? 0;
+        const ub = utilityByPath.get(b.absPath) ?? 0;
+        if (ub !== ua)
+            return ub - ua; // utility DESC
+        return a.absPath < b.absPath ? -1 : a.absPath > b.absPath ? 1 : 0; // tie-break: path ASC
+    });
+}
 /**
  * Scan the primary stash for `memory:` and `knowledge:` markdown files
  * suitable for graph extraction. The directory layout convention is the

package/dist/indexer/index-writer-lock.js

@@ -0,0 +1,99 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import fs from "node:fs";
+import path from "node:path";
+import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "../core/file-lock.js";
+import { getDbPath, getIndexWriterLockPath } from "../core/paths.js";
+const INDEX_WRITER_LOCK_STALE_AFTER_MS = 12 * 60 * 60 * 1000;
+const INDEX_WRITER_WAIT_MS = 100;
+const heldLocks = new Map();
+function buildPayload(purpose, pid = process.pid) {
+    return JSON.stringify({
+        pid,
+        purpose,
+        dbPath: getDbPath(),
+        startedAt: new Date().toISOString(),
+    });
+}
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function throwIfAborted(signal) {
+    if (!signal?.aborted)
+        return;
+    throw signal.reason instanceof Error ? signal.reason : new Error("index writer wait aborted");
+}
+function releaseHeldLock(lockPath) {
+    const held = heldLocks.get(lockPath);
+    if (!held)
+        return;
+    held.depth -= 1;
+    if (held.depth > 0)
+        return;
+    heldLocks.delete(lockPath);
+    process.off("exit", held.exitHandler);
+    releaseLockIfOwned(lockPath, process.pid);
+}
+function retainHeldLock(lockPath) {
+    const existing = heldLocks.get(lockPath);
+    if (existing) {
+        existing.depth += 1;
+        return { lockPath, release: () => releaseHeldLock(lockPath) };
+    }
+    const exitHandler = () => releaseLockIfOwned(lockPath, process.pid);
+    process.on("exit", exitHandler);
+    heldLocks.set(lockPath, { depth: 1, exitHandler });
+    return { lockPath, release: () => releaseHeldLock(lockPath) };
+}
+function detachHeldLock(lockPath) {
+    const held = heldLocks.get(lockPath);
+    if (!held)
+        return;
+    heldLocks.delete(lockPath);
+    process.off("exit", held.exitHandler);
+}
+export async function acquireIndexWriterLease(options) {
+    const mode = options.mode ?? "wait";
+    const lockPath = getIndexWriterLockPath();
+    fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+    if (heldLocks.has(lockPath)) {
+        return retainHeldLock(lockPath);
+    }
+    while (true) {
+        throwIfAborted(options.signal);
+        if (tryAcquireLockSync(lockPath, buildPayload(options.purpose))) {
+            return retainHeldLock(lockPath);
+        }
+        const probe = probeLock(lockPath, { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
+        if (probe.state === "held" && probe.holderPid === process.pid) {
+            return retainHeldLock(lockPath);
+        }
+        if (probe.state === "stale") {
+            releaseLock(lockPath);
+            continue;
+        }
+        if (mode === "try")
+            return undefined;
+        await delay(INDEX_WRITER_WAIT_MS);
+    }
+}
+export async function withIndexWriterLease(options, run) {
+    const lease = await acquireIndexWriterLease(options);
+    if (!lease) {
+        throw new Error(`index writer lease unavailable for ${options.purpose}`);
+    }
+    try {
+        return await run();
+    }
+    finally {
+        lease.release();
+    }
+}
+export function handoffIndexWriterLeaseToPid(lease, pid, purpose) {
+    fs.writeFileSync(lease.lockPath, buildPayload(purpose, pid), "utf8");
+    detachHeldLock(lease.lockPath);
+}
+export function probeIndexWriterLease() {
+    return probeLock(getIndexWriterLockPath(), { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
+}