akm-cli 0.9.16 → 0.9.17-alpha.2

package/dist/indexer/ensure-index.js

@@ -21,11 +21,13 @@
  */
 import fs from "node:fs";
 import path from "node:path";
+import { hashContent } from "../core/adapter/adapters/shared.js";
 import { placementSpecList } from "../core/asset/asset-placement.js";
 import { classifyPathAccess } from "../core/path-access.js";
 import { getDbPath } from "../core/paths.js";
+import { warnVerbose } from "../core/warn.js";
 import { assertIndexPathReadable, closeDatabase, openExistingDatabase } from "../storage/repositories/index-connection.js";
-import { getEntryCount, getIndexedFilePaths } from "../storage/repositories/index-entries-repository.js";
+import { getEntryCount, getIndexedFileHashes, getIndexedFilePaths, } from "../storage/repositories/index-entries-repository.js";
 import { isCanonicalIndexGeneration } from "../storage/repositories/index-entry-schema.js";
 import { getMeta } from "../storage/repositories/index-meta-repository.js";
 import { warnOnBundleRenameDrift } from "./bundle-identity-guard.js";
@@ -77,11 +79,20 @@ function getIndexableFiles(root, spec) {
  *      millisecond-truncated), so the mtime test alone silently misses
  *      additions made within ~a millisecond of the previous build.
  *
+ * A file whose mtime is newer than `builtAt` is NOT automatically stale
+ * (R6): `indexWrittenAssets` upserts a fresh `content_hash` without bumping
+ * `builtAt`, so a file `ensureIndex` itself just incrementally re-indexed
+ * (for example a proposal triage just promoted into `knowledge/`) would
+ * otherwise keep tripping this check on every subsequent call, forcing the
+ * full rescan the write-path fast path exists to avoid. Only files newer
+ * than `builtAt` are hashed here, so this stays cheap — the common case is
+ * zero or a handful of such files.
+ *
  * `getIndexableFiles` applies each asset type's own relevance filter, so
  * non-indexed companion files (e.g. `package.json` next to a knowledge doc) are
  * never considered and do not produce false "new file" positives.
  */
-function hasNewerIndexableFiles(stashDir, builtAt, indexedPaths) {
+function hasNewerIndexableFiles(stashDir, builtAt, indexedPaths, indexedHashes) {
     const builtAtMs = builtAt ? new Date(builtAt).getTime() : Number.NaN;
     const builtAtUsable = Number.isFinite(builtAtMs);
     for (const spec of placementSpecList()) {
@@ -92,13 +103,31 @@ function hasNewerIndexableFiles(stashDir, builtAt, indexedPaths) {
                 return true;
             if (!builtAtUsable)
                 return true;
+            let mtimeMs;
             try {
-                if (fs.statSync(file).mtimeMs > builtAtMs)
-                    return true;
+                mtimeMs = fs.statSync(file).mtimeMs;
+            }
+            catch {
+                return true;
+            }
+            if (mtimeMs <= builtAtMs)
+                continue;
+            // Newer than the last full build — only stale if its current content
+            // actually differs from what is indexed. No stored hash means the row
+            // predates content-hash tracking (or a hash-less enrichment pass), so
+            // fall back to the conservative mtime-stale answer.
+            const indexedHash = indexedHashes.get(file);
+            if (indexedHash === undefined)
+                return true;
+            let currentHash;
+            try {
+                currentHash = hashContent(fs.readFileSync(file, "utf8"));
             }
             catch {
                 return true;
             }
+            if (currentHash !== indexedHash)
+                return true;
         }
     }
     return false;
@@ -125,7 +154,7 @@ export function isIndexStale(stashDir) {
         if (entryCount === 0)
             return true;
         const builtAt = getMeta(db, "builtAt");
-        if (hasNewerIndexableFiles(stashDir, builtAt, getIndexedFilePaths(db)))
+        if (hasNewerIndexableFiles(stashDir, builtAt, getIndexedFilePaths(db), getIndexedFileHashes(db)))
             return true;
         const storedStashDir = getMeta(db, "stashDir");
         if (storedStashDir !== stashDir) {
@@ -193,12 +222,24 @@ function indexCanServeStash(stashDir) {
 }
 async function runInlineReindex(stashDir, options = {}) {
     const { akmIndex } = await import("./indexer.js");
-    await akmIndex({
+    const startedMs = Date.now();
+    const response = await akmIndex({
         stashDir,
         implicit: true,
         ...(options.signal ? { signal: options.signal } : {}),
         ...(options.hydrateSources === false ? { hydrateSources: false } : {}),
     });
+    // R6: the implicit reindex's cost was previously discarded entirely
+    // (`await akmIndex(...)` and nothing else), making a 27-minute blocking
+    // rebuild invisible to both the operator and the improve result. Fall back
+    // to a wall-clock measurement when the response carries no `timing` block.
+    const durationMs = response.timing?.totalMs ?? Date.now() - startedMs;
+    const timing = response.timing;
+    warnVerbose(`[ensure-index] implicit reindex completed in ${durationMs}ms` +
+        (timing
+            ? ` (walk=${timing.walkMs}ms llm=${timing.llmMs}ms embed=${timing.embedMs}ms finalize=${timing.finalizeMs}ms)`
+            : ""));
+    options.onReindexTiming?.({ durationMs, timing });
     return true;
 }
 /**
@@ -226,7 +267,10 @@ export async function ensureIndex(stashDir, options = {}) {
         // materialization point — hydrate cache-backed sources as usual.
         if (!isIndexStale(stashDir))
             return false;
-        return runInlineReindex(stashDir, { ...(options.signal ? { signal: options.signal } : {}) });
+        return runInlineReindex(stashDir, {
+            ...(options.signal ? { signal: options.signal } : {}),
+            ...(options.onReindexTiming ? { onReindexTiming: options.onReindexTiming } : {}),
+        });
     }
     // Background = the READ path (`show` auto-index): query time must never clone/
     // pull/fetch (spec §14.3 / D11). Build from already-materialized content only;
@@ -236,5 +280,6 @@ export async function ensureIndex(stashDir, options = {}) {
     return runInlineReindex(stashDir, {
         ...(options.signal ? { signal: options.signal } : {}),
         hydrateSources: false,
+        ...(options.onReindexTiming ? { onReindexTiming: options.onReindexTiming } : {}),
     });
 }

package/dist/indexer/graph/graph-extraction.js

@@ -128,7 +128,7 @@ function normalizeConfidence(raw) {
         return undefined;
     return Math.max(0, Math.min(1, raw));
 }
-function getGraphExtractorId(config) {
+export function getGraphExtractorId(config) {
     const fingerprint = computeBodyHash(JSON.stringify({
         promptVersion: graphExtract.GRAPH_EXTRACT_PROMPT_VERSION,
         model: config.model,
@@ -152,6 +152,36 @@ function buildLowQualityWarnings(quality, telemetry) {
     }
     return warnings;
 }
+/**
+ * Failure-rate abort for the extraction run (R2), modelled on consolidate's
+ * chunk-level guard (`ABORT_MIN_CHUNKS`/`ABORT_FAILURE_RATE` in
+ * consolidate.ts, C-6/#392): rate-based over a minimum sample so a couple of
+ * transient per-file failures cannot abort a run that would otherwise
+ * recover, while a systemically dead provider stops burning through the
+ * rest of the eligible set. The existing "one failure must not abort the
+ * rest" behaviour for individual files is untouched — this only stops
+ * further model calls once the failure rate itself is the signal.
+ */
+const GRAPH_EXTRACTION_ABORT_MIN_ATTEMPTS = 4;
+const GRAPH_EXTRACTION_ABORT_FAILURE_RATE = 0.5;
+/** Records one attempted (non-cache-hit) model call and flips `aborted` once the failure-rate threshold is crossed. */
+function recordGraphExtractionAttempt(state, failed) {
+    if (state.aborted)
+        return;
+    state.attempts += 1;
+    if (failed)
+        state.failures += 1;
+    if (state.attempts < GRAPH_EXTRACTION_ABORT_MIN_ATTEMPTS)
+        return;
+    const failureRate = state.failures / state.attempts;
+    if (failureRate < GRAPH_EXTRACTION_ABORT_FAILURE_RATE)
+        return;
+    state.aborted = true;
+    state.message =
+        `graph extraction aborted — failure rate ${(failureRate * 100).toFixed(0)}% over ${state.attempts} ` +
+            `attempt(s) (>= ${GRAPH_EXTRACTION_ABORT_FAILURE_RATE * 100}% threshold). LLM may be unavailable.`;
+    warn(state.message);
+}
 export function getGraphExtractionIncludeTypes(config) {
     const configured = getIndexPassConfig(config.index, "graph")?.graphExtractionIncludeTypes;
     if (!configured || configured.length === 0)
@@ -200,6 +230,20 @@ function validateGraphCacheShape(raw) {
         ...(typeof obj.reason === "string" ? { reason: obj.reason } : {}),
     };
 }
+/**
+ * A `"failed"` extraction (provider error, invalid JSON, context overflow —
+ * see {@link GraphExtractionStatus}) must never be reused as a cache hit or
+ * re-persisted as one. R2: a dead provider upserted ~30,900 rows shaped
+ * `{"entities":[],"relations":[],"status":"failed","reason":"llm_error"}`,
+ * and both hit paths (the `llm_enrichment_cache` lookup and `reuseGraphNode`
+ * over the previous graph) validated the shape without checking `status`, so
+ * 92% of the persisted graph became a permanent hit that never retried. A
+ * failed result becomes a miss naturally and is overwritten on the next
+ * successful extraction; existing failed rows are left on disk untouched.
+ */
+function isFailedExtractionStatus(status) {
+    return status === "failed";
+}
 function loadGraphFile(stashRoot, db) {
     if (!db)
         return { files: [] };
@@ -252,6 +296,8 @@ function reuseGraphNode(previousNodes, candidate, bodyHash) {
         return undefined;
     if (node.bodyHash !== bodyHash)
         return undefined;
+    if (isFailedExtractionStatus(node.status))
+        return undefined;
     const validated = validateGraphCacheShape({ entities: node.entities, relations: node.relations });
     if (!validated)
         return undefined;
@@ -276,8 +322,9 @@ function planEligibleGraphExtractions(args) {
             if (entry?.bodyHash === bodyHash) {
                 try {
                     const cached = validateGraphCacheShape(JSON.parse(entry.resultJson));
-                    if (cached)
+                    if (cached && !isFailedExtractionStatus(cached.status)) {
                         return { kind: "cache-hit", candidate, bodyHash, cached, persistCache: false };
+                    }
                 }
                 catch {
                     // Corrupt cache rows are immutable model plans for this pass.
@@ -305,7 +352,7 @@ function graphRecordFromCachePlan(plan) {
     };
 }
 async function extractGraphBatches(args) {
-    const { plans, batchSize, signal, db, cacheVariant, telemetry, llmRunner, lease, featureConfig, onFallback, batchState, runtimeTelemetry, onNotices, reportProgress, } = args;
+    const { plans, batchSize, signal, db, cacheVariant, telemetry, llmRunner, lease, featureConfig, onFallback, abortState, batchState, runtimeTelemetry, onNotices, reportProgress, maxChunksPerAsset, } = args;
     const results = new Array(plans.length).fill(undefined);
     const chunkStarts = [];
     for (let start = 0; start < plans.length; start += batchSize)
@@ -328,12 +375,12 @@ async function extractGraphBatches(args) {
                 continue;
             telemetry.cacheHits += 1;
             results[start + index] = graphRecordFromCachePlan(plan);
-            if (db && plan.persistCache) {
+            if (db && plan.persistCache && !isFailedExtractionStatus(plan.cached.status)) {
                 upsertLlmCacheEntry(db, plan.candidate.absPath, plan.bodyHash, JSON.stringify(plan.cached), cacheVariant);
             }
         }
         const modelPlans = chunk.filter((plan) => plan.kind === "model");
-        if (modelPlans.length === 0) {
+        if (modelPlans.length === 0 || abortState.aborted) {
             reportChunkProgress();
             return;
         }
@@ -345,6 +392,7 @@ async function extractGraphBatches(args) {
                 telemetry: runtimeTelemetry,
                 onNotices,
                 ...(lease ? { lease } : {}),
+                ...(maxChunksPerAsset != null ? { maxChunksPerAsset } : {}),
             });
         }
         catch (error) {
@@ -355,6 +403,8 @@ async function extractGraphBatches(args) {
             throw error;
         }
         let llmIndex = 0;
+        let dispatchHadResult = false;
+        let dispatchAllFailed = true;
         for (let index = 0; index < chunk.length; index++) {
             const plan = chunk[index];
             if (!plan || plan.kind !== "model")
@@ -369,7 +419,10 @@ async function extractGraphBatches(args) {
                 ...(extraction.status ? { status: extraction.status } : {}),
                 ...(extraction.reason ? { reason: extraction.reason } : {}),
             };
-            if (db) {
+            dispatchHadResult = true;
+            if (!isFailedExtractionStatus(cacheShape.status))
+                dispatchAllFailed = false;
+            if (db && !isFailedExtractionStatus(cacheShape.status)) {
                 upsertLlmCacheEntry(db, plan.candidate.absPath, plan.bodyHash, JSON.stringify(cacheShape), cacheVariant);
             }
             results[start + index] = {
@@ -379,6 +432,14 @@ async function extractGraphBatches(args) {
                 ...cacheShape,
             };
         }
+        // One attempt per `extractGraphFromBodies` dispatch (this chunk's batch
+        // call), not one per file it covers — mirrors consolidate.ts's
+        // totalChunksProcessed++/totalChunksFailed, which count once per chunk
+        // regardless of how many memories are in it. Counting per file let a
+        // single batched provider_error satisfy GRAPH_EXTRACTION_ABORT_MIN_ATTEMPTS
+        // after one HTTP failure whenever graphExtractionBatchSize >= 4.
+        if (dispatchHadResult)
+            recordGraphExtractionAttempt(abortState, dispatchAllFailed);
         reportChunkProgress();
     }, llmRunner.connection.concurrency ?? 1);
     return { results, ...(configFailure ? { configFailure } : {}) };
@@ -670,6 +731,7 @@ export async function runGraphExtractionPass(ctx) {
             batchingDisabled: false,
             nonArrayBatchFailures: 0,
         };
+        const abortState = { attempts: 0, failures: 0, aborted: false };
         warnVerbose(`graph extraction: starting for ${considered} eligible file(s) under ${primary.path}; ` +
             `includeTypes=${includeTypes.join(",")}, batchSize=${batchSize}, concurrency=${llmRunner.connection.concurrency ?? 1}, ` +
             `reEnrich=${reEnrich === true}, candidateScoped=${options.candidatePaths ? "true" : "false"}.`);
@@ -690,11 +752,15 @@ export async function runGraphExtractionPass(ctx) {
                 if (plan.kind === "cache-hit") {
                     telemetry.cacheHits += 1;
                     cached = plan.cached;
-                    if (db && plan.persistCache) {
+                    if (db && plan.persistCache && !isFailedExtractionStatus(cached.status)) {
                         upsertLlmCacheEntry(db, candidate.absPath, bodyHash, JSON.stringify(cached), cacheVariant);
                     }
                 }
                 else {
+                    if (abortState.aborted) {
+                        reportProgress(candidate.absPath, undefined);
+                        return undefined;
+                    }
                     telemetry.cacheMisses += 1;
                     let extraction;
                     try {
@@ -703,6 +769,7 @@ export async function runGraphExtractionPass(ctx) {
                             telemetry: runtimeTelemetry,
                             onNotices,
                             ...(dispatchLease ? { lease: dispatchLease } : {}),
+                            ...(options.maxChunksPerAsset != null ? { maxChunksPerAsset: options.maxChunksPerAsset } : {}),
                         });
                     }
                     catch (err) {
@@ -719,7 +786,8 @@ export async function runGraphExtractionPass(ctx) {
                         ...(extraction.status ? { status: extraction.status } : {}),
                         ...(extraction.reason ? { reason: extraction.reason } : {}),
                     };
-                    if (db) {
+                    recordGraphExtractionAttempt(abortState, isFailedExtractionStatus(cached.status));
+                    if (db && !isFailedExtractionStatus(cached.status)) {
                         upsertLlmCacheEntry(db, candidate.absPath, bodyHash, JSON.stringify(cached), cacheVariant);
                     }
                 }
@@ -754,8 +822,10 @@ export async function runGraphExtractionPass(ctx) {
                 onFallback,
                 batchState,
                 runtimeTelemetry,
+                abortState,
                 onNotices,
                 reportProgress,
+                ...(options.maxChunksPerAsset != null ? { maxChunksPerAsset: options.maxChunksPerAsset } : {}),
             });
             extractionResults = batch.results;
             configFailure ??= batch.configFailure;
@@ -797,14 +867,18 @@ export async function runGraphExtractionPass(ctx) {
         const assetRefs = mergedNodes.map((node) => node.path);
         const deduped = deduplicateGraph(mergedNodes.map((node) => ({ entities: node.entities, relations: node.relations })), assetRefs);
         telemetry.truncationCount = runtimeTelemetry.truncationCount ?? 0;
+        telemetry.truncatedChunks = runtimeTelemetry.truncatedChunks ?? 0;
         telemetry.failureCount = runtimeTelemetry.failureCount ?? 0;
         telemetry.htmlErrorCount = runtimeTelemetry.htmlErrorCount ?? 0;
         telemetry.retryAttempts = runtimeTelemetry.retryAttempts ?? 0;
         telemetry.nonArrayBatchFailures = runtimeTelemetry.nonArrayBatchFailures ?? 0;
+        telemetry.aborted = abortState.aborted;
         const qualityConsidered = mergedNodes.length;
         const qualityExtracted = mergedNodes.filter((node) => node.status === "extracted" && node.entities.length > 0).length;
         const quality = computeGraphQualityTelemetry(qualityConsidered, qualityExtracted, deduped.entities.length, deduped.relations.length);
         const warnings = buildLowQualityWarnings(quality, telemetry);
+        if (abortState.message)
+            warnings.push(abortState.message);
         for (const warning of warnings)
             warnVerbose(`graph extraction quality: ${warning}`);
         const graph = {

package/dist/indexer/passes/memory-inference.js

@@ -46,7 +46,7 @@ import { todayIso } from "../../core/common.js";
 import { concurrentMap } from "../../core/concurrent.js";
 import { ConfigError } from "../../core/errors.js";
 import { warn } from "../../core/warn.js";
-import { recordWrittenPath } from "../../core/write-provenance.js";
+import { beginWriteProvenance, recordWrittenPath } from "../../core/write-provenance.js";
 import { writeAssetToSource } from "../../core/write-source.js";
 import { disposeLoweredExecutionDispatchLease, } from "../../integrations/agent/execution-lowering.js";
 import { isProcessEnabled } from "../../llm/feature-gate.js";
@@ -188,6 +188,18 @@ async function inferPendingMemoryRecord(plan, ctx) {
  * short-circuits to a no-op result.
  */
 export async function runMemoryInferencePass(ctx) {
+    // R78 (tier1-0917): owns the write-provenance journal end-to-end so it closes on every
+    // exit path, including a throw out of the body below — an unclosed journal
+    // would keep reporting every later write in this process as this call's own.
+    const provenance = beginWriteProvenance();
+    try {
+        return await runMemoryInferencePassBody(ctx, provenance);
+    }
+    finally {
+        provenance.end();
+    }
+}
+async function runMemoryInferencePassBody(ctx, provenance) {
     const { config, sources, signal, db, reEnrich, onProgress, options = {} } = ctx;
     const invocationOwnsRunner = Object.hasOwn(ctx, "llmRunner");
     const compressMemoryToDerivedMemory = options.compressMemoryToDerivedMemory ?? memoryInfer.compressMemoryToDerivedMemory;
@@ -202,6 +214,7 @@ export async function runMemoryInferencePass(ctx) {
         skippedAborted: 0,
         unaccounted: 0,
         htmlErrorCount: 0,
+        writtenPaths: [],
     };
     // Mutable sink threaded into compressMemoryToDerivedMemory so the per-call
     // HTML-error categorization (which is otherwise swallowed inside the feature
@@ -215,6 +228,8 @@ export async function runMemoryInferencePass(ctx) {
     const completeResult = () => {
         if (noticesByKey.size > 0)
             result.notices = Object.freeze([...noticesByKey.values()]);
+        // Non-destructive read — the wrapper's `finally` owns closing the journal.
+        result.writtenPaths = provenance.writtenPaths();
         return result;
     };
     // Gate 1 — feature gate via isProcessEnabled, which reads the 0.8.0 path

package/dist/llm/client.js

@@ -117,12 +117,24 @@ export function isContextSizeError(message) {
         /exceeded|over.*limit|too.*long/.test(lower);
     return evidence;
 }
+/**
+ * Codes describing a failure to reach or get a usable response from the
+ * provider transport itself, as opposed to a malformed-but-received response
+ * (`parse_error`) or a request-shape rejection (`rate_limited`). Shared by
+ * {@link isRetryable} (which additionally requires evidence the failure is
+ * transient) and the batch graph-extraction storm guard in `graph-extract.ts`
+ * (which treats any of these as "the provider is down, stop retrying
+ * per-asset") so the two classifications cannot drift apart.
+ */
+export function isTransportFailure(err) {
+    return err.code === "provider_error" || err.code === "network_error" || err.code === "provider_html_error";
+}
 /**
  * Decide whether a first-attempt {@link LlmCallError} is eligible for a single
  * retry. Retryable: HTTP 5xx (`provider_error` with statusCode >= 500) and
  * `network_error` whose message looks like a transient connection drop.
- * NOT retryable: 4xx, `rate_limited` (429), `timeout`, `parse_error`, and
- * context-overflow-classified errors.
+ * NOT retryable: 4xx, `rate_limited` (429), `timeout`, `parse_error`,
+ * `provider_html_error`, and context-overflow-classified errors.
  *
  * The connection-drop heuristic covers the substrings emitted across runtimes
  * for a mid-flight socket close:
@@ -141,6 +153,8 @@ export function isContextSizeError(message) {
 function isRetryable(err) {
     if (isContextSizeError(err.message))
         return false;
+    if (!isTransportFailure(err))
+        return false;
     if (err.code === "provider_error") {
         return typeof err.statusCode === "number" && err.statusCode >= 500;
     }