akm-cli 0.7.4 → 0.8.0-rc.10

package/dist/indexer/graph-extraction.js

@@ -1,133 +1,589 @@
-/**
- * Graph-extraction pass for `akm index` (#207).
- *
- * Walks the primary stash for `memory:` and `knowledge:` assets, asks the
- * configured LLM to extract entities and relations from each one, and
- * persists the result to a single stash-local artifact at
- * `<stashRoot>/.akm/graph.json`. The artifact is consumed by the search
- * pipeline (see `src/indexer/graph-boost.ts`) as a single boost component
- * inside the existing FTS5+boosts loop — there is NO second SearchHit
- * scorer and no parallel ranking track.
- *
- * Disabling — three preconditions must ALL hold for the pass to run:
- *   1. `akm.llm` must be configured (no provider = no extraction). When
- *      absent, `resolveIndexPassLLM("graph", config)` returns `undefined`
- *      and the pass short-circuits.
- *   2. `llm.features.graph_extraction !== false` — the locked v1 spec §14
- *      feature-flag layer. Set to `false` to block the pass at the
- *      feature-gate layer (no network call may ever issue).
- *   3. `index.graph.llm !== false` — the per-pass opt-out layer (#208).
- *      Set to `false` to skip just this pass while leaving other passes
- *      that share the same `llm` block enabled.
- *   Toggling any one off does NOT delete the existing `graph.json` — the
- *   user keeps the boost component they already have, it just stops
- *   refreshing.
- *
- * Locked v1 contract:
- *   - LLM access is exclusively via `resolveIndexPassLLM("graph", config)`.
- *   - The `graph.json` file is an indexer artifact, NOT a user-visible
- *     asset. It does not have an asset ref, does not appear in search
- *     hits, and is not addressable via `akm show`. Direct `fs.writeFile`
- *     is therefore the correct primitive — `writeAssetToSource` is
- *     reserved for asset writes (CLAUDE.md / spec §10 step 5).
- */
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import fs from "node:fs";
 import path from "node:path";
+import { TYPE_DIRS } from "../core/asset-spec";
+import { concurrentMap } from "../core/concurrent";
+import { getIndexPassConfig, resolveBatchSize } from "../core/config";
 import { parseFrontmatter } from "../core/frontmatter";
-import { warn } from "../core/warn";
-import { extractGraphFromBody } from "../llm/graph-extract";
+import { warn, warnVerbose } from "../core/warn";
+import { isProcessEnabled } from "../llm/feature-gate";
+import * as graphExtract from "../llm/graph-extract";
 import { resolveIndexPassLLM } from "../llm/index-passes";
+import { computeBodyHash, GRAPH_SCHEMA_VERSION, getLlmCacheEntriesByRefs, getLlmCacheEntry, upsertLlmCacheEntry, } from "./db";
+import { loadStoredGraphSnapshot, replaceStoredGraph } from "./graph-db";
+import { deduplicateGraph } from "./graph-dedup";
+import { walkMarkdownFiles } from "./walker";
 /** Schema version for the persisted artifact — bumps trigger a full rebuild. */
-export const GRAPH_FILE_SCHEMA_VERSION = 1;
-/** Path scheme — kept stable so consumers (search-time boost) can find it. */
-export const GRAPH_FILE_RELATIVE_PATH = path.join(".akm", "graph.json");
-/** Public path resolver — exported so the search-side reader and tests share the rule. */
-export function getGraphFilePath(stashRoot) {
-    return path.join(stashRoot, GRAPH_FILE_RELATIVE_PATH);
-}
+export const GRAPH_FILE_SCHEMA_VERSION = GRAPH_SCHEMA_VERSION;
+const EMPTY_QUALITY = {
+    consideredFiles: 0,
+    extractedFiles: 0,
+    entityCount: 0,
+    relationCount: 0,
+    extractionCoverage: 0,
+    density: 0,
+};
 const EMPTY_RESULT = {
     considered: 0,
     extracted: 0,
     totalEntities: 0,
     totalRelations: 0,
     written: false,
+    quality: { ...EMPTY_QUALITY },
+    telemetry: {
+        cacheHits: 0,
+        cacheMisses: 0,
+        truncationCount: 0,
+        failureCount: 0,
+    },
+    warnings: [],
 };
+function roundMetric(value) {
+    return Number(value.toFixed(4));
+}
+function computeGraphQualityTelemetry(consideredFiles, extractedFiles, entityCount, relationCount) {
+    const extractionCoverage = consideredFiles > 0 ? extractedFiles / consideredFiles : 0;
+    const maxEdges = entityCount > 1 ? (entityCount * (entityCount - 1)) / 2 : 0;
+    const density = maxEdges > 0 ? relationCount / maxEdges : 0;
+    return {
+        consideredFiles,
+        extractedFiles,
+        entityCount,
+        relationCount,
+        extractionCoverage: roundMetric(extractionCoverage),
+        density: roundMetric(density),
+    };
+}
+export const DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES = ["memory", "knowledge"];
+const SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES = new Set([
+    "memory",
+    "knowledge",
+    "skill",
+    "command",
+    "agent",
+    "workflow",
+    "lesson",
+    "task",
+    "wiki",
+]);
+const GRAPH_CACHE_VARIANT_PREFIX = "graph-extraction";
+function normalizeConfidence(raw) {
+    if (typeof raw !== "number" || !Number.isFinite(raw))
+        return undefined;
+    return Math.max(0, Math.min(1, raw));
+}
+function getGraphExtractorId(config) {
+    const fingerprint = computeBodyHash(JSON.stringify({
+        promptVersion: graphExtract.GRAPH_EXTRACT_PROMPT_VERSION,
+        model: config.model,
+        batchSize: config.batchSize,
+        includeTypes: config.includeTypes,
+        maxChunkBodyChars: 1600,
+        maxBatchBodyChars: 1600,
+    })).slice(0, 16);
+    return `${GRAPH_CACHE_VARIANT_PREFIX}:${graphExtract.GRAPH_EXTRACT_PROMPT_VERSION}:${config.model}:${fingerprint}`;
+}
+function buildLowQualityWarnings(quality, telemetry) {
+    const warnings = [];
+    if (quality.consideredFiles >= 5 && quality.extractionCoverage < 0.3) {
+        warnings.push(`Low graph extraction coverage (${quality.extractedFiles}/${quality.consideredFiles}, ${quality.extractionCoverage}).`);
+    }
+    if (quality.entityCount >= 8 && quality.relationCount === 0) {
+        warnings.push("Graph extraction produced many entities but no relations.");
+    }
+    if (telemetry.failureCount > 0) {
+        warnings.push(`Graph extraction encountered ${telemetry.failureCount} failed file extraction(s).`);
+    }
+    return warnings;
+}
+export function getGraphExtractionIncludeTypes(config) {
+    const configured = getIndexPassConfig(config.index, "graph")?.graphExtractionIncludeTypes;
+    if (!configured || configured.length === 0)
+        return [...DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES];
+    const out = [];
+    const seen = new Set();
+    for (const rawType of configured) {
+        const type = rawType.trim().toLowerCase();
+        if (!type || seen.has(type))
+            continue;
+        if (!SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES.has(type))
+            continue;
+        seen.add(type);
+        out.push(type);
+    }
+    return out.length > 0 ? out : [...DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES];
+}
+function validateGraphCacheShape(raw) {
+    if (!raw || typeof raw !== "object")
+        return undefined;
+    const obj = raw;
+    if (!Array.isArray(obj.entities) || !obj.entities.every((e) => typeof e === "string"))
+        return undefined;
+    if (obj.relations !== undefined &&
+        (!Array.isArray(obj.relations) ||
+            !obj.relations.every((r) => {
+                if (!r || typeof r !== "object")
+                    return false;
+                const rel = r;
+                if (typeof rel.from !== "string" || typeof rel.to !== "string")
+                    return false;
+                if (rel.type !== undefined && typeof rel.type !== "string")
+                    return false;
+                if (rel.confidence !== undefined && (typeof rel.confidence !== "number" || !Number.isFinite(rel.confidence))) {
+                    return false;
+                }
+                return true;
+            }))) {
+        return undefined;
+    }
+    return {
+        entities: obj.entities,
+        relations: Array.isArray(obj.relations) ? obj.relations : [],
+        confidence: normalizeConfidence(obj.confidence),
+        ...(typeof obj.status === "string" ? { status: obj.status } : {}),
+        ...(typeof obj.reason === "string" ? { reason: obj.reason } : {}),
+    };
+}
+function loadGraphFile(stashRoot, db) {
+    if (!db)
+        return { files: [] };
+    const graph = loadStoredGraphSnapshot(stashRoot, db);
+    if (!graph)
+        return { files: [] };
+    const out = [];
+    for (const node of graph.files) {
+        const cacheShape = validateGraphCacheShape({ entities: node.entities, relations: node.relations });
+        if (!cacheShape)
+            continue;
+        out.push({
+            path: node.path,
+            type: node.type,
+            bodyHash: node.bodyHash,
+            entities: cacheShape.entities,
+            relations: cacheShape.relations,
+            confidence: normalizeConfidence(node.confidence),
+            ...(node.status ? { status: node.status } : {}),
+            ...(node.reason ? { reason: node.reason } : {}),
+            ...(node.extractionRunId ? { extractionRunId: node.extractionRunId } : {}),
+        });
+    }
+    return {
+        files: out,
+        ...(graph.telemetry ? { telemetry: graph.telemetry } : {}),
+    };
+}
+function mergeGraphNodes(previousNodes, refreshedNodes, candidatePaths) {
+    if (!candidatePaths)
+        return refreshedNodes;
+    const refreshedByPath = new Map(refreshedNodes.map((node) => [node.path, node]));
+    const merged = [];
+    for (const node of previousNodes) {
+        if (candidatePaths.has(node.path))
+            continue;
+        merged.push(node);
+    }
+    for (const node of refreshedNodes)
+        merged.push(refreshedByPath.get(node.path) ?? node);
+    return merged;
+}
+function reuseGraphNode(previousNodes, candidate, bodyHash) {
+    const node = previousNodes.get(candidate.absPath);
+    if (!node)
+        return undefined;
+    if (node.type !== candidate.type)
+        return undefined;
+    if (typeof node.bodyHash !== "string" || node.bodyHash.length === 0)
+        return undefined;
+    if (node.bodyHash !== bodyHash)
+        return undefined;
+    const validated = validateGraphCacheShape({ entities: node.entities, relations: node.relations });
+    if (!validated)
+        return undefined;
+    return {
+        entities: validated.entities,
+        relations: validated.relations,
+        confidence: normalizeConfidence(node.confidence),
+        ...(node.status ? { status: node.status } : {}),
+        ...(node.reason ? { reason: node.reason } : {}),
+    };
+}
 /**
  * Top-level entry point. Returns a no-op result when the pass is disabled.
  *
  * Three preconditions — ALL must hold for the pass to run:
  *
- *   1. **Provider configured** — `akm.llm` must be present. Without a
+ *   1. **Provider configured** — an LLM profile must be selectable. Without a
  *      configured provider, `resolveIndexPassLLM("graph", config)` returns
  *      `undefined` (the pass cannot run because there is no model to call).
- *   2. **Feature gate** — `llm.features.graph_extraction` (defaults to
- *      `true`). When `false`, no network call may issue regardless of
- *      per-pass settings. This is the locked spec-§14 gate.
+ *   2. **Feature gate** — `profiles.improve.default.processes.graphExtraction.enabled`
+ *      (defaults to `true`). When `false`, no network call may issue regardless
+ *      of per-pass settings.
  *   3. **Per-pass gate** — `index.graph.llm` (defaults to `true`). When
  *      `false`, the indexer simply skips this pass for the current run.
  *
  * If any of the three is missing or `false`, this function short-circuits
- * to an empty no-op result, leaving any existing `graph.json` untouched on
- * disk.
+ * to an empty no-op result, leaving any existing persisted graph untouched.
+ *
+ * When `config.index.graph.graphExtractionBatchSize > 1`, eligible files are
+ * chunked into batches and each chunk is processed with a single LLM call via
+ * `extractGraphFromBodies`. Default batch size is 1 (one call per asset —
+ * preserves existing behaviour, fully opt-in).
  */
-export async function runGraphExtractionPass(config, sources, signal) {
-    // Gate 1 — locked feature flag (§14). Defaults to enabled; only an
-    // explicit `false` disables the pass entirely.
-    if (config.llm?.features?.graph_extraction === false)
+export async function runGraphExtractionPass(config, sources, signal, db, reEnrich, onProgress, options = {}) {
+    // Gate 1 — feature gate via isProcessEnabled, which reads the 0.8.0 path
+    // (profiles.improve.default.processes.graphExtraction.enabled). Defaults to
+    // enabled when the key is absent.
+    if (!isProcessEnabled("index", "graph_extraction", config))
         return { ...EMPTY_RESULT };
     // Gate 2 — per-pass opt-out (#208). Returns the resolved llm config or
     // `undefined` when the pass should not run.
     const llmConfig = resolveIndexPassLLM("graph", config);
-    if (!llmConfig)
+    if (!llmConfig) {
+        const reason = getIndexPassConfig(config.index, "graph")?.llm === false
+            ? "index.graph.llm is false"
+            : "no default LLM profile is configured";
+        warnVerbose(`graph extraction: skipped because ${reason}.`);
         return { ...EMPTY_RESULT };
+    }
     // The pass only writes to the primary (working) stash. Read-only caches
     // (git, npm, website) are deliberately untouched — the graph artifact for
     // those sources would be clobbered by the next sync().
     const primary = sources[0];
-    if (!primary)
+    if (!primary) {
+        warnVerbose("graph extraction: skipped because no primary stash source is available.");
         return { ...EMPTY_RESULT };
-    const eligible = collectEligibleFiles(primary.path);
+    }
+    const includeTypes = getGraphExtractionIncludeTypes(config);
+    const eligible = collectEligibleFiles(primary.path, includeTypes).filter((candidate) => !options.candidatePaths || options.candidatePaths.has(candidate.absPath));
     const considered = eligible.length;
-    if (considered === 0)
+    if (considered === 0) {
+        const scoped = options.candidatePaths ? ` matching ${options.candidatePaths.size} candidate path(s)` : "";
+        warnVerbose(`graph extraction: skipped because no eligible files${scoped} were found under ${primary.path}. ` +
+            `includeTypes=${includeTypes.join(",")}`);
         return { ...EMPTY_RESULT };
+    }
+    const previousGraph = loadGraphFile(primary.path, db);
+    const previousNodes = new Map(previousGraph.files.map((node) => [node.path, node]));
     const nodes = [];
     let totalEntities = 0;
     let totalRelations = 0;
-    for (const candidate of eligible) {
-        if (signal?.aborted)
-            break;
-        const extraction = await extractGraphFromBody(llmConfig, candidate.body, signal);
-        if (extraction.entities.length === 0)
+    let processed = 0;
+    let extracted = 0;
+    onProgress?.({ processed, total: considered, extracted, totalEntities, totalRelations });
+    const reportProgress = (currentPath, result) => {
+        processed += 1;
+        if (result) {
+            if (result.entities.length > 0)
+                extracted += 1;
+            totalEntities += result.entities.length;
+            totalRelations += result.relations.length;
+        }
+        onProgress?.({
+            processed,
+            total: considered,
+            extracted,
+            totalEntities,
+            totalRelations,
+            currentPath,
+        });
+    };
+    // Resolve the effective batch size. Falls back to
+    // DEFAULT_GRAPH_EXTRACTION_BATCH_SIZE (4) when unset, and clamps against
+    // `llm.contextLength` if the model's context window is configured.
+    const batchSize = resolveBatchSize(getIndexPassConfig(config.index, "graph")?.graphExtractionBatchSize, llmConfig.contextLength);
+    const extractionRunId = crypto.randomUUID();
+    const extractorId = getGraphExtractorId({ model: llmConfig.model, batchSize, includeTypes });
+    const cacheVariant = extractorId;
+    const telemetry = {
+        extractorId,
+        extractionRunId,
+        model: llmConfig.model,
+        promptVersion: graphExtract.GRAPH_EXTRACT_PROMPT_VERSION,
+        batchSize,
+        cacheHits: 0,
+        cacheMisses: 0,
+        truncationCount: 0,
+        failureCount: 0,
+    };
+    const canReusePreviousGraph = previousGraph.telemetry?.extractorId === extractorId;
+    const runtimeTelemetry = {
+        truncationCount: 0,
+        failureCount: 0,
+        filteredGenericEntities: 0,
+        filteredInvalidRelations: 0,
+        filteredLowConfidenceRelations: 0,
+        contextBatchRetries: 0,
+        nonArrayBatchFailures: 0,
+    };
+    const batchState = {
+        batchingDisabled: false,
+        nonArrayBatchFailures: 0,
+    };
+    warnVerbose(`graph extraction: starting for ${considered} eligible file(s) under ${primary.path}; ` +
+        `includeTypes=${includeTypes.join(",")}, batchSize=${batchSize}, concurrency=${llmConfig.concurrency ?? 1}, ` +
+        `reEnrich=${reEnrich === true}, candidateScoped=${options.candidatePaths ? "true" : "false"}.`);
+    const onFallback = (evt) => {
+        warn(`[akm] LLM fallback for ${evt.feature}: ${evt.reason}`);
+    };
+    let extractionResults;
+    if (batchSize <= 1) {
+        // ── Original per-asset path (with incremental cache) ─────────────────
+        extractionResults = await concurrentMap(eligible, async (candidate) => {
+            if (signal?.aborted) {
+                reportProgress(candidate.absPath, undefined);
+                return undefined;
+            }
+            const bodyHash = computeBodyHash(candidate.body);
+            let cached;
+            if (db) {
+                if (!(reEnrich ?? false)) {
+                    const cacheEntry = getLlmCacheEntry(db, candidate.absPath, bodyHash, cacheVariant);
+                    if (cacheEntry) {
+                        try {
+                            cached = validateGraphCacheShape(JSON.parse(cacheEntry.resultJson));
+                            if (cached)
+                                telemetry.cacheHits += 1;
+                        }
+                        catch {
+                            cached = undefined;
+                        }
+                    }
+                }
+            }
+            else if (!(reEnrich ?? false)) {
+                // No DB — best-effort reuse from the previous in-memory graph.
+                cached = reuseGraphNode(previousNodes, candidate, bodyHash);
+            }
+            if (!cached && !(reEnrich ?? false) && canReusePreviousGraph) {
+                const reused = reuseGraphNode(previousNodes, candidate, bodyHash);
+                if (reused) {
+                    cached = reused;
+                    if (db) {
+                        upsertLlmCacheEntry(db, candidate.absPath, bodyHash, JSON.stringify(reused), cacheVariant);
+                    }
+                    telemetry.cacheHits += 1;
+                }
+            }
+            if (!cached) {
+                telemetry.cacheMisses += 1;
+                const extraction = await graphExtract.extractGraphFromBody(llmConfig, candidate.body, signal, config, onFallback, { batchState, telemetry: runtimeTelemetry });
+                cached = {
+                    entities: extraction.entities,
+                    relations: extraction.relations,
+                    ...(extraction.confidence !== undefined ? { confidence: extraction.confidence } : {}),
+                    ...(extraction.status ? { status: extraction.status } : {}),
+                    ...(extraction.reason ? { reason: extraction.reason } : {}),
+                };
+                if (db) {
+                    upsertLlmCacheEntry(db, candidate.absPath, bodyHash, JSON.stringify(cached), cacheVariant);
+                }
+            }
+            const result = {
+                absPath: candidate.absPath,
+                type: candidate.type,
+                bodyHash,
+                entities: cached.entities,
+                relations: cached.relations,
+                ...(cached.confidence !== undefined ? { confidence: cached.confidence } : {}),
+                ...(cached.status ? { status: cached.status } : {}),
+                ...(cached.reason ? { reason: cached.reason } : {}),
+            };
+            reportProgress(candidate.absPath, result);
+            return result;
+        }, 
+        // Default concurrency of 4 for cloud APIs. Set `llm.concurrency: 1`
+        // in config.json for local model servers (LM Studio, Ollama).
+        llmConfig.concurrency ?? 1);
+    }
+    else {
+        // ── Batched path (with incremental cache) ────────────────────────────
+        // Chunk eligible files into groups of `batchSize` and call
+        // `extractGraphFromBodies` once per chunk. Cache hits are resolved
+        // before chunking so they don't consume LLM tokens in the batch call.
+        const rawResults = new Array(eligible.length).fill(undefined);
+        const chunkStarts = [];
+        for (let start = 0; start < eligible.length; start += batchSize)
+            chunkStarts.push(start);
+        await concurrentMap(chunkStarts, async (start) => {
+            if (signal?.aborted)
+                return;
+            const chunk = eligible.slice(start, start + batchSize);
+            const reportChunkProgress = () => {
+                for (let j = 0; j < chunk.length; j++) {
+                    const candidate = chunk[j];
+                    if (!candidate)
+                        continue;
+                    reportProgress(candidate.absPath, rawResults[start + j]);
+                }
+            };
+            // Pre-resolve cache hits for this chunk; track which positions need LLM.
+            const bodyHashes = chunk.map((c) => computeBodyHash(c.body));
+            // Batch the cache lookup: one IN(...) query for the whole chunk instead
+            // of N individual SELECTs. The map covers every ref in this chunk that
+            // has any cached row; the per-position hash check happens below.
+            const chunkCache = db && !reEnrich
+                ? getLlmCacheEntriesByRefs(db, chunk.map((c) => c.absPath), cacheVariant)
+                : new Map();
+            const needsLlm = chunk.map((c, j) => {
+                if (!db || reEnrich)
+                    return true;
+                const cached = chunkCache.get(c.absPath);
+                // Hash mismatch → body changed, treat as cache miss.
+                if (!cached || cached.bodyHash !== (bodyHashes[j] ?? ""))
+                    return true;
+                try {
+                    const parsed = validateGraphCacheShape(JSON.parse(cached.resultJson));
+                    if (!parsed)
+                        return true;
+                    telemetry.cacheHits += 1;
+                    rawResults[start + j] = {
+                        absPath: c.absPath,
+                        type: c.type,
+                        bodyHash: bodyHashes[j] ?? "",
+                        entities: parsed.entities,
+                        relations: parsed.relations,
+                        ...(parsed.confidence !== undefined ? { confidence: parsed.confidence } : {}),
+                        ...(parsed.status ? { status: parsed.status } : {}),
+                        ...(parsed.reason ? { reason: parsed.reason } : {}),
+                    };
+                    return false;
+                }
+                catch {
+                    return true;
+                }
+            });
+            // Secondary incremental path: reuse previous graph nodes when the body hash
+            // still matches and DB cache is missing/stale/unavailable.
+            if (!(reEnrich ?? false) && canReusePreviousGraph) {
+                for (let j = 0; j < chunk.length; j++) {
+                    if (!needsLlm[j])
+                        continue;
+                    const candidate = chunk[j];
+                    if (!candidate)
+                        continue;
+                    const reused = reuseGraphNode(previousNodes, candidate, bodyHashes[j] ?? "");
+                    if (!reused)
+                        continue;
+                    telemetry.cacheHits += 1;
+                    rawResults[start + j] = {
+                        absPath: candidate.absPath,
+                        type: candidate.type,
+                        bodyHash: bodyHashes[j] ?? "",
+                        entities: reused.entities,
+                        relations: reused.relations,
+                        ...(reused.confidence !== undefined ? { confidence: reused.confidence } : {}),
+                        ...(reused.status ? { status: reused.status } : {}),
+                        ...(reused.reason ? { reason: reused.reason } : {}),
+                    };
+                    if (db) {
+                        upsertLlmCacheEntry(db, candidate.absPath, bodyHashes[j] ?? "", JSON.stringify(reused), cacheVariant);
+                    }
+                    needsLlm[j] = false;
+                }
+            }
+            const uncachedChunk = chunk.filter((_, j) => needsLlm[j]);
+            if (uncachedChunk.length === 0) {
+                reportChunkProgress();
+                return;
+            }
+            const bodies = uncachedChunk.map((c) => c.body);
+            telemetry.cacheMisses += uncachedChunk.length;
+            // extractGraphFromBodies always returns an array of the same length
+            // as bodies (it falls back per-asset for any missing indices).
+            const batchExtractions = await graphExtract.extractGraphFromBodies(llmConfig, bodies, signal, config, onFallback, { batchState, telemetry: runtimeTelemetry });
+            // Map LLM results back to original positions and write cache entries.
+            let llmIdx = 0;
+            for (let j = 0; j < chunk.length; j++) {
+                if (!needsLlm[j])
+                    continue;
+                const candidate = chunk[j];
+                const extraction = batchExtractions[llmIdx++];
+                if (!candidate || !extraction)
+                    continue;
+                if (db) {
+                    upsertLlmCacheEntry(db, candidate.absPath, bodyHashes[j] ?? "", JSON.stringify({
+                        entities: extraction.entities,
+                        relations: extraction.relations,
+                        ...(extraction.confidence !== undefined ? { confidence: extraction.confidence } : {}),
+                        ...(extraction.status ? { status: extraction.status } : {}),
+                        ...(extraction.reason ? { reason: extraction.reason } : {}),
+                    }), cacheVariant);
+                }
+                rawResults[start + j] = {
+                    absPath: candidate.absPath,
+                    type: candidate.type,
+                    bodyHash: bodyHashes[j] ?? "",
+                    entities: extraction.entities,
+                    relations: extraction.relations,
+                    ...(extraction.confidence !== undefined ? { confidence: extraction.confidence } : {}),
+                    ...(extraction.status ? { status: extraction.status } : {}),
+                    ...(extraction.reason ? { reason: extraction.reason } : {}),
+                };
+            }
+            reportChunkProgress();
+        }, llmConfig.concurrency ?? 1);
+        extractionResults = rawResults;
+    }
+    for (const result of extractionResults) {
+        if (!result)
             continue;
         nodes.push({
-            path: candidate.absPath,
-            type: candidate.type,
-            // Lower-case once at write time so the search-time boost can do a
-            // single case-folded comparison without re-canonicalising on every
-            // query.
-            entities: extraction.entities.map((e) => e.toLowerCase()),
-            relations: extraction.relations.map((r) => ({
-                from: r.from.toLowerCase(),
-                to: r.to.toLowerCase(),
-                ...(r.type ? { type: r.type.toLowerCase() } : {}),
-            })),
+            path: result.absPath,
+            type: result.type,
+            bodyHash: result.bodyHash,
+            entities: [...new Set(result.entities.map((entity) => entity.trim()).filter(Boolean))],
+            relations: result.relations
+                .map((r) => ({
+                from: r.from.trim(),
+                to: r.to.trim(),
+                ...(r.type ? { type: r.type.trim() } : {}),
+                ...(normalizeConfidence(r.confidence) !== undefined ? { confidence: normalizeConfidence(r.confidence) } : {}),
+            }))
+                .filter((relation) => relation.from && relation.to),
+            ...(normalizeConfidence(result.confidence) !== undefined
+                ? { confidence: normalizeConfidence(result.confidence) }
+                : {}),
+            status: result.status ?? (result.entities.length > 0 ? "extracted" : "empty"),
+            reason: result.reason ?? (result.entities.length > 0 ? "none" : "no_graph_content"),
+            extractionRunId,
         });
-        totalEntities += extraction.entities.length;
-        totalRelations += extraction.relations.length;
     }
+    const mergedNodes = mergeGraphNodes(previousGraph.files, nodes, options.candidatePaths);
+    const assetRefs = mergedNodes.map((node) => node.path);
+    const deduped = deduplicateGraph(mergedNodes.map((node) => ({ entities: node.entities, relations: node.relations })), assetRefs);
+    telemetry.truncationCount = runtimeTelemetry.truncationCount ?? 0;
+    telemetry.failureCount = runtimeTelemetry.failureCount ?? 0;
+    const qualityConsidered = mergedNodes.length;
+    const qualityExtracted = mergedNodes.filter((node) => node.status === "extracted" && node.entities.length > 0).length;
+    const quality = computeGraphQualityTelemetry(qualityConsidered, qualityExtracted, deduped.entities.length, deduped.relations.length);
+    const warnings = buildLowQualityWarnings(quality, telemetry);
+    for (const warning of warnings)
+        warnVerbose(`graph extraction quality: ${warning}`);
     const graph = {
         schemaVersion: GRAPH_FILE_SCHEMA_VERSION,
         generatedAt: new Date().toISOString(),
         stashRoot: primary.path,
-        files: nodes,
+        files: mergedNodes,
+        entities: deduped.entities,
+        relations: deduped.relations,
+        quality,
+        telemetry,
     };
-    const written = writeGraphFile(primary.path, graph);
+    const written = writeGraphFile(primary.path, graph, db);
+    warnVerbose(`graph extraction: ${written ? "persisted" : "did not persist"} graph for ${primary.path}; ` +
+        `considered=${considered}, extractedThisRun=${extracted}, storedFiles=${mergedNodes.length}, ` +
+        `entities=${deduped.entities.length}, relations=${deduped.relations.length}, coverage=${quality.extractionCoverage}.`);
     return {
         considered,
-        extracted: nodes.length,
+        extracted,
         totalEntities,
         totalRelations,
         written,
+        quality,
+        telemetry,
+        warnings,
     };
 }
 /**
@@ -141,10 +597,16 @@ export async function runGraphExtractionPass(config, sources, signal) {
  *
  * Exported for direct unit testing.
  */
-export function collectEligibleFiles(stashRoot) {
+export function collectEligibleFiles(stashRoot, includeTypes = [...DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES]) {
     const out = [];
-    for (const type of ["memory", "knowledge"]) {
-        const dir = path.join(stashRoot, `${type === "memory" ? "memories" : "knowledge"}`);
+    for (const rawType of includeTypes) {
+        const type = rawType.trim().toLowerCase();
+        if (!SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES.has(type))
+            continue;
+        const stashDir = TYPE_DIRS[type];
+        if (!stashDir)
+            continue;
+        const dir = path.join(stashRoot, stashDir);
         if (!fs.existsSync(dir))
             continue;
         for (const filePath of walkMarkdownFiles(dir)) {
@@ -168,47 +630,21 @@ export function collectEligibleFiles(stashRoot) {
     }
     return out;
 }
-function* walkMarkdownFiles(root) {
-    let entries;
-    try {
-        entries = fs.readdirSync(root, { withFileTypes: true });
-    }
-    catch {
-        return;
-    }
-    for (const entry of entries) {
-        const full = path.join(root, entry.name);
-        if (entry.isDirectory()) {
-            yield* walkMarkdownFiles(full);
-        }
-        else if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) {
-            yield full;
-        }
-    }
-}
 // ── Persistence ─────────────────────────────────────────────────────────────
 /**
- * Write `graph.json` atomically to `<stashRoot>/.akm/graph.json`.
- *
- * Direct `fs.writeFile` is intentional. The graph artifact is an indexer
- * cache — not a user-visible asset — so it does not have an asset ref and
- * `writeAssetToSource` (which routes through the asset-spec rendering
- * layer) is the wrong primitive here. See CLAUDE.md / spec §10 step 5 for
- * the carve-out: kind-branching writes for asset content live in
- * `src/core/write-source.ts`; opaque indexer artifacts may write directly.
+ * Persist graph rows into the SQLite index DB.
  */
-function writeGraphFile(stashRoot, graph) {
-    const target = getGraphFilePath(stashRoot);
-    const dir = path.dirname(target);
+function writeGraphFile(stashRoot, graph, db) {
+    if (!db) {
+        warn("graph extraction: no database handle available; skipping graph persistence.");
+        return false;
+    }
     try {
-        fs.mkdirSync(dir, { recursive: true });
-        const tmp = `${target}.tmp.${process.pid}.${Math.random().toString(36).slice(2)}`;
-        fs.writeFileSync(tmp, `${JSON.stringify(graph, null, 2)}\n`, "utf8");
-        fs.renameSync(tmp, target);
+        replaceStoredGraph(db, graph);
         return true;
     }
     catch (err) {
-        warn(`graph extraction: failed to write ${target}: ${err instanceof Error ? err.message : String(err)}`);
+        warn(`graph extraction: failed to persist graph for ${stashRoot}: ${err instanceof Error ? err.message : String(err)}`);
         return false;
     }
 }