akm-cli 0.9.0-beta.9 → 0.9.0-rc.0

package/dist/indexer/graph/graph-extraction.js

@@ -40,13 +40,15 @@ import path from "node:path";
 import { TYPE_DIRS } from "../../core/asset/asset-spec.js";
 import { parseFrontmatter } from "../../core/asset/frontmatter.js";
 import { concurrentMap } from "../../core/concurrent.js";
-import { getIndexPassConfig, resolveBatchSize } from "../../core/config/config.js";
+import { getIndexPassConfig, loadConfig, resolveBatchSize } from "../../core/config/config.js";
+import { rethrowIfTestIsolationError } from "../../core/errors.js";
 import { warn, warnVerbose } from "../../core/warn.js";
 import { isProcessEnabled } from "../../llm/feature-gate.js";
 import * as graphExtract from "../../llm/graph-extract.js";
 import { resolveIndexPassLLM } from "../../llm/index-passes.js";
-import { computeBodyHash, GRAPH_SCHEMA_VERSION, getLlmCacheEntriesByRefs, getLlmCacheEntry, upsertLlmCacheEntry, } from "../db/db.js";
-import { loadStoredGraphSnapshot, replaceStoredGraph } from "../db/graph-db.js";
+import { computeBodyHash, getLlmCacheEntriesByRefs, getLlmCacheEntry, upsertLlmCacheEntry, } from "../db/db.js";
+import { drainExtractionQueue, loadStoredGraphSnapshot, replaceStoredGraph } from "../db/graph-db.js";
+import { GRAPH_SCHEMA_VERSION } from "../db/schema.js";
 import { walkMarkdownFiles } from "../walk/walker.js";
 import { deduplicateGraph } from "./graph-dedup.js";
 /** Schema version for the persisted artifact — bumps trigger a full rebuild. */
@@ -92,6 +94,12 @@ function computeGraphQualityTelemetry(consideredFiles, extractedFiles, entityCou
     };
 }
 export const DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES = ["memory", "knowledge"];
+/**
+ * Max number of lazy-extraction queue rows drained per pass (#624-P3). Bounds
+ * per-run work so a large backlog is spread across runs rather than processed
+ * all at once. Generous default — the queue is normally near-empty.
+ */
+const GRAPH_EXTRACTION_QUEUE_DRAIN_LIMIT = 100;
 const SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES = new Set([
     "memory",
     "knowledge",
@@ -291,8 +299,29 @@ export async function runGraphExtractionPass(ctx) {
         warnVerbose("graph extraction: skipped because no primary stash source is available.");
         return { ...EMPTY_RESULT };
     }
+    // #624-P3: drain the lazy-extraction queue BEFORE the ranked sweep, highest
+    // priority first. Queued paths are extracted individually (per-file merge,
+    // other files untouched) so they are processed even when they fall outside
+    // the normal candidate set. Default (empty queue) is a byte-identical no-op:
+    // drainExtractionQueue returns [] and the loop body never runs.
+    if (db) {
+        const drained = drainExtractionQueue(db, primary.path, GRAPH_EXTRACTION_QUEUE_DRAIN_LIMIT);
+        for (const queued of drained) {
+            if (signal?.aborted)
+                break;
+            await extractGraphForSingleFile(db, primary.path, queued.filePath, queued.bodyHash, { config, signal });
+        }
+    }
     const includeTypes = getGraphExtractionIncludeTypes(config);
-    const eligible = collectEligibleFiles(primary.path, includeTypes).filter((candidate) => !options.candidatePaths || options.candidatePaths.has(candidate.absPath));
+    let eligible = collectEligibleFiles(primary.path, includeTypes).filter((candidate) => !options.candidatePaths || options.candidatePaths.has(candidate.absPath));
+    // P2 (#624): when topN is set and a DB is available, rank the (already
+    // candidate-filtered) eligible set by utility_scores DESC and keep only the
+    // top-N. Default (topN unset) is byte-identical to today — no ranking query
+    // is issued and the eligible set is untouched. Ranking composes WITH the
+    // candidatePaths filter: scoped-then-ranked-then-sliced.
+    if (db && options.topN != null && options.topN >= 0) {
+        eligible = rankCandidatesByUtility(db, eligible, primary.path).slice(0, options.topN);
+    }
     const considered = eligible.length;
     if (considered === 0) {
         const scoped = options.candidatePaths ? ` matching ${options.candidatePaths.size} candidate path(s)` : "";
@@ -344,6 +373,7 @@ export async function runGraphExtractionPass(ctx) {
         failureCount: 0,
         htmlErrorCount: 0,
         retryAttempts: 0,
+        nonArrayBatchFailures: 0,
     };
     const canReusePreviousGraph = previousGraph.telemetry?.extractorId === extractorId;
     const runtimeTelemetry = {
@@ -597,6 +627,7 @@ export async function runGraphExtractionPass(ctx) {
     telemetry.failureCount = runtimeTelemetry.failureCount ?? 0;
     telemetry.htmlErrorCount = runtimeTelemetry.htmlErrorCount ?? 0;
     telemetry.retryAttempts = runtimeTelemetry.retryAttempts ?? 0;
+    telemetry.nonArrayBatchFailures = runtimeTelemetry.nonArrayBatchFailures ?? 0;
     const qualityConsidered = mergedNodes.length;
     const qualityExtracted = mergedNodes.filter((node) => node.status === "extracted" && node.entities.length > 0).length;
     const quality = computeGraphQualityTelemetry(qualityConsidered, qualityExtracted, deduped.entities.length, deduped.relations.length);
@@ -628,6 +659,178 @@ export async function runGraphExtractionPass(ctx) {
         warnings,
     };
 }
+/**
+ * Infer the asset type (`memory`, `knowledge`, …) for a path from the stash
+ * directory segment it lives under. Returns the matching include-type, or
+ * `undefined` when the path is not under a known graph-eligible type dir.
+ */
+function inferGraphTypeForPath(stashRoot, absPath) {
+    const rel = path.relative(stashRoot, absPath);
+    const firstSeg = rel.split(path.sep)[0];
+    if (!firstSeg)
+        return undefined;
+    for (const type of SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES) {
+        if (TYPE_DIRS[type] === firstSeg)
+            return type;
+    }
+    return undefined;
+}
+/**
+ * #624-P3 — extract graph data for a SINGLE file and merge it into the stored
+ * graph WITHOUT clobbering other files' rows.
+ *
+ * Re-reads the body from disk at call time (the queued body_hash is NOT trusted
+ * blindly — the file may have been deleted or changed since enqueue) and skips
+ * silently (returns `false`) when the file is gone or empty. Resolves the LLM
+ * via {@link resolveIndexPassLLM} (model-available guard: returns `false` when
+ * no provider is configured) UNLESS `opts.llmOverride` is supplied, in which
+ * case the override is the extractor seam (used by tests and by callers that
+ * already hold a resolved model).
+ *
+ * Returns `true` when a graph row was written for the file, `false` on any
+ * skip (missing file, empty body, unknown type, no model, or extraction error).
+ */
+export async function extractGraphForSingleFile(db, stashRoot, filePath, bodyHash, opts) {
+    try {
+        // Re-read from disk — never trust a stale queued body.
+        let raw;
+        try {
+            raw = fs.readFileSync(filePath, "utf8");
+        }
+        catch {
+            return false; // file gone / unreadable → silent skip
+        }
+        const parsed = parseFrontmatter(raw);
+        const body = parsed.content.trim();
+        if (!body)
+            return false;
+        const type = inferGraphTypeForPath(stashRoot, filePath) ?? "memory";
+        const effectiveHash = bodyHash ?? computeBodyHash(body);
+        // Extract — via the injected seam, or the real per-asset path.
+        let extraction;
+        if (opts?.llmOverride) {
+            const out = await opts.llmOverride(body);
+            extraction = {
+                entities: out.entities,
+                relations: out.relations,
+                ...(out.confidence !== undefined ? { confidence: out.confidence } : {}),
+            };
+        }
+        else {
+            const config = opts?.config ?? loadConfig();
+            if (!isProcessEnabled("index", "graph_extraction", config))
+                return false;
+            const llmConfig = resolveIndexPassLLM("graph", config);
+            if (!llmConfig)
+                return false; // model-available guard
+            const result = await graphExtract.extractGraphFromBody(llmConfig, body, opts?.signal, config);
+            extraction = {
+                entities: result.entities,
+                relations: result.relations,
+                ...(result.confidence !== undefined ? { confidence: result.confidence } : {}),
+            };
+        }
+        const entities = [...new Set(extraction.entities.map((e) => e.trim()).filter(Boolean))];
+        const relations = extraction.relations
+            .map((r) => ({
+            from: r.from.trim(),
+            to: r.to.trim(),
+            ...(r.type ? { type: r.type.trim() } : {}),
+            ...(normalizeConfidence(r.confidence) !== undefined ? { confidence: normalizeConfidence(r.confidence) } : {}),
+        }))
+            .filter((r) => r.from && r.to);
+        const node = {
+            path: filePath,
+            type,
+            bodyHash: effectiveHash,
+            entities,
+            relations,
+            ...(normalizeConfidence(extraction.confidence) !== undefined
+                ? { confidence: normalizeConfidence(extraction.confidence) }
+                : {}),
+            status: entities.length > 0 ? "extracted" : "empty",
+            reason: entities.length > 0 ? "none" : "no_graph_content",
+            extractionRunId: crypto.randomUUID(),
+        };
+        // Merge with the previously-stored nodes, scoping the refresh to JUST this
+        // path so other files' rows are preserved (and graph_meta counts refresh).
+        const previousGraph = loadGraphFile(stashRoot, db);
+        const candidatePaths = new Set([filePath]);
+        const mergedNodes = mergeGraphNodes(previousGraph.files, [node], candidatePaths);
+        const assetRefs = mergedNodes.map((n) => n.path);
+        const deduped = deduplicateGraph(mergedNodes.map((n) => ({ entities: n.entities, relations: n.relations })), assetRefs);
+        const qualityExtracted = mergedNodes.filter((n) => n.status === "extracted" && n.entities.length > 0).length;
+        const quality = computeGraphQualityTelemetry(mergedNodes.length, qualityExtracted, deduped.entities.length, deduped.relations.length);
+        const graph = {
+            schemaVersion: GRAPH_FILE_SCHEMA_VERSION,
+            generatedAt: new Date().toISOString(),
+            stashRoot,
+            files: mergedNodes,
+            entities: deduped.entities,
+            relations: deduped.relations,
+            quality,
+            ...(previousGraph.telemetry ? { telemetry: previousGraph.telemetry } : {}),
+        };
+        return writeGraphFile(stashRoot, graph, db);
+    }
+    catch (err) {
+        rethrowIfTestIsolationError(err);
+        return false;
+    }
+}
+// ── Eligible-file detection ─────────────────────────────────────────────────
+/**
+ * Rank eligible graph-extraction candidates by their entry `utility_scores`,
+ * highest first, for the incremental high-signal-first sweep (P2 of #624).
+ *
+ * The join is READ-ONLY (`entries.file_path = candidate.absPath`, then
+ * `entries.id -> utility_scores.entry_id`) and does NOT re-couple the graph
+ * rows to `entries`. It reads the GLOBAL `utility_scores` table (not the
+ * per-scope `utility_scores_scoped`), so ranking is corpus-wide; `stashRoot`
+ * is accepted for call-site symmetry/future scoping but is not used to filter
+ * (the global table has no `stash_root` column).
+ *
+ * Candidates with no matching `entries` row, or an entry with no
+ * `utility_scores` row, get an effective utility of 0 (LEFT JOIN + COALESCE)
+ * and sort LAST — they are deprioritized, never dropped, so a `topN >= total`
+ * slice still includes them and they remain reachable on later runs.
+ *
+ * Ties (equal utility) break by `file_path` ASC for deterministic output.
+ * Returns a NEW array; the input is not mutated. SQLite's ~999 bound-parameter
+ * cap is respected by chunking the `IN (...)` lookup at 500.
+ *
+ * Exported for direct unit testing.
+ */
+export function rankCandidatesByUtility(db, candidates, _stashRoot) {
+    // Cannot rank without a DB → return the input unranked rather than throw.
+    // Keeps the DB-less code path (reuse-from-memory) working when topN is set.
+    if (!db || candidates.length === 0)
+        return candidates;
+    const utilityByPath = new Map();
+    const CHUNK = 500;
+    for (let start = 0; start < candidates.length; start += CHUNK) {
+        const chunk = candidates.slice(start, start + CHUNK);
+        const paths = chunk.map((c) => c.absPath);
+        const placeholders = paths.map(() => "?").join(", ");
+        const rows = db
+            .prepare(`SELECT e.file_path AS file_path, COALESCE(MAX(u.utility), 0) AS utility
+           FROM entries e
+           LEFT JOIN utility_scores u ON u.entry_id = e.id
+          WHERE e.file_path IN (${placeholders})
+          GROUP BY e.file_path`)
+            .all(...paths);
+        for (const row of rows) {
+            utilityByPath.set(row.file_path, row.utility ?? 0);
+        }
+    }
+    return [...candidates].sort((a, b) => {
+        const ua = utilityByPath.get(a.absPath) ?? 0;
+        const ub = utilityByPath.get(b.absPath) ?? 0;
+        if (ub !== ua)
+            return ub - ua; // utility DESC
+        return a.absPath < b.absPath ? -1 : a.absPath > b.absPath ? 1 : 0; // tie-break: path ASC
+    });
+}
 /**
  * Scan the primary stash for `memory:` and `knowledge:` markdown files
  * suitable for graph extraction. The directory layout convention is the

package/dist/indexer/index-writer-lock.js

@@ -7,6 +7,7 @@ import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "
 import { getDbPath, getIndexWriterLockPath } from "../core/paths.js";
 const INDEX_WRITER_LOCK_STALE_AFTER_MS = 12 * 60 * 60 * 1000;
 const INDEX_WRITER_WAIT_MS = 100;
+const DEFAULT_INDEX_WRITER_MAX_WAIT_MS = 10 * 60 * 1000;
 const heldLocks = new Map();
 function buildPayload(purpose, pid = process.pid) {
     return JSON.stringify({
@@ -46,27 +47,26 @@ function retainHeldLock(lockPath) {
     heldLocks.set(lockPath, { depth: 1, exitHandler });
     return { lockPath, release: () => releaseHeldLock(lockPath) };
 }
-function detachHeldLock(lockPath) {
-    const held = heldLocks.get(lockPath);
-    if (!held)
-        return;
-    heldLocks.delete(lockPath);
-    process.off("exit", held.exitHandler);
-}
 export async function acquireIndexWriterLease(options) {
     const mode = options.mode ?? "wait";
     const lockPath = getIndexWriterLockPath();
+    const startedAt = Date.now();
+    const maxWaitMs = options.maxWaitMs ?? DEFAULT_INDEX_WRITER_MAX_WAIT_MS;
     fs.mkdirSync(path.dirname(lockPath), { recursive: true });
     if (heldLocks.has(lockPath)) {
+        options.onAcquired?.({ waitedMs: 0 });
         return retainHeldLock(lockPath);
     }
+    let lastWaitNoticeMs = 0;
     while (true) {
         throwIfAborted(options.signal);
         if (tryAcquireLockSync(lockPath, buildPayload(options.purpose))) {
+            options.onAcquired?.({ waitedMs: Date.now() - startedAt });
             return retainHeldLock(lockPath);
         }
         const probe = probeLock(lockPath, { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
         if (probe.state === "held" && probe.holderPid === process.pid) {
+            options.onAcquired?.({ waitedMs: Date.now() - startedAt });
             return retainHeldLock(lockPath);
         }
         if (probe.state === "stale") {
@@ -75,6 +75,17 @@ export async function acquireIndexWriterLease(options) {
         }
         if (mode === "try")
             return undefined;
+        // Held by another live process. Time out only *after* a real acquisition
+        // attempt, so a caller with maxWaitMs:0 still gets one chance at a free lock
+        // instead of throwing before it ever tries.
+        if (maxWaitMs >= 0 && Date.now() - startedAt >= maxWaitMs) {
+            throw new Error(`timed out waiting for index writer lease for ${options.purpose}`);
+        }
+        const waitedMs = Date.now() - startedAt;
+        if (waitedMs - lastWaitNoticeMs >= 15000) {
+            options.onWait?.({ waitedMs });
+            lastWaitNoticeMs = waitedMs;
+        }
         await delay(INDEX_WRITER_WAIT_MS);
     }
 }
@@ -90,10 +101,6 @@ export async function withIndexWriterLease(options, run) {
         lease.release();
     }
 }
-export function handoffIndexWriterLeaseToPid(lease, pid, purpose) {
-    fs.writeFileSync(lease.lockPath, buildPayload(purpose, pid), "utf8");
-    detachHeldLock(lease.lockPath);
-}
 export function probeIndexWriterLease() {
     return probeLock(getIndexWriterLockPath(), { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
 }

package/dist/indexer/index-written-assets.js

@@ -0,0 +1,105 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Write-path indexing: targeted single-file index updates for asset writers.
+ *
+ * The index is maintained eagerly by every first-class mutation command
+ * (`source add`, `wiki`, `workflow`, `setup` all run `akmIndex()` after
+ * writing). The memory write paths — `akm remember` / `writeMarkdownAsset`
+ * and extract's session assets — historically did not, which is why reads
+ * used to compensate with stale-triggered background reindexes (the
+ * lock-contention footgun removed alongside this module's introduction; see
+ * docs/design/read-path-reindex-contention-findings.md §7).
+ *
+ * This is NOT a general reindex. It upserts exactly the files the caller just
+ * wrote: frontmatter/metadata via the shared matcher pipeline, the `entries`
+ * row, and an incremental FTS refresh. Embeddings, index-time LLM passes,
+ * graph extraction, `builtAt`, and the per-dir walk cache are all deliberately
+ * untouched — the next full run heals them (the opportunistic-recovery
+ * strategy of docs/technical/index-consistency-adr.md).
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { getDbPath } from "../core/paths.js";
+import { warnVerbose } from "../core/warn.js";
+import { closeDatabase, getEntryCount, openExistingDatabase, rebuildFts, upsertEntry } from "./db/db.js";
+import { generateMetadataFlat } from "./passes/metadata.js";
+import { buildSearchText } from "./search/search-fields.js";
+/**
+ * Busy-timeout (ms) for write-path index upserts. A real write — unlike the
+ * 250ms telemetry inserts — but it must not hang `akm remember` for the full
+ * default 30s behind a running full reindex. When it times out, the upsert is
+ * skipped and the asset becomes searchable after that reindex instead.
+ */
+export const WRITE_PATH_INDEX_BUSY_TIMEOUT_MS = 5_000;
+/**
+ * Index the given just-written asset files into the existing local index.
+ *
+ * FAIL-OPEN at every step: any error (index.db absent, empty, locked past the
+ * busy timeout, unparseable file) is reduced to a verbose-only warning and the
+ * write command succeeds untouched. The degraded outcome is exactly the
+ * pre-write-path-indexing behavior: the asset appears after the next full
+ * `akm index` / improve-cron run.
+ *
+ * An absent or empty index is skipped on purpose — bootstrap belongs to the
+ * first read (`ensureIndex`) or an explicit `akm index`, which also cover
+ * embeddings and the other passes this fast path skips.
+ */
+export async function indexWrittenAssets(stashDir, filePaths) {
+    try {
+        const dbPath = getDbPath();
+        if (!fs.existsSync(dbPath))
+            return;
+        // The full walk never descends into dot-directories (they hold state like
+        // `.meta/`, `.stash.json`), and `shouldIndexStashFile` relies on the walker
+        // for that — mirror it here so this fast path indexes exactly what a full
+        // run would.
+        const files = filePaths.filter((f) => {
+            if (!fs.existsSync(f))
+                return false;
+            const rel = path.relative(stashDir, f);
+            return !rel.split(/[\\/]+/).some((segment) => segment.startsWith("."));
+        });
+        if (files.length === 0)
+            return;
+        // Generate metadata BEFORE opening the DB so the write window stays
+        // short. One call per file keeps the entry↔path pairing exact.
+        const pairs = [];
+        for (const file of files) {
+            const generated = await generateMetadataFlat(stashDir, [file]);
+            const entry = generated.entries[0];
+            // Workflows carry a side-table document upsert this fast path doesn't
+            // do; no current caller writes them, but guard so one never lands
+            // half-indexed.
+            if (entry && entry.type !== "workflow")
+                pairs.push({ file, entry });
+        }
+        if (pairs.length === 0)
+            return;
+        const db = openExistingDatabase(dbPath);
+        try {
+            db.exec(`PRAGMA busy_timeout = ${WRITE_PATH_INDEX_BUSY_TIMEOUT_MS}`);
+            if (getEntryCount(db) === 0)
+                return;
+            for (const { file, entry } of pairs) {
+                const entryKey = `${stashDir}:${entry.type}:${entry.name}`;
+                let entryWithSize = entry;
+                try {
+                    entryWithSize = { ...entry, fileSize: fs.statSync(file).size };
+                }
+                catch {
+                    // stat raced a delete — index without the size, like the full walk does.
+                }
+                upsertEntry(db, entryKey, path.dirname(file), file, stashDir, entryWithSize, buildSearchText(entry));
+            }
+            rebuildFts(db, { incremental: true });
+        }
+        finally {
+            closeDatabase(db);
+        }
+    }
+    catch (error) {
+        warnVerbose("Write-path index update skipped (asset appears after the next full index):", error instanceof Error ? error.message : String(error));
+    }
+}