akm-cli 0.9.17-alpha.4 → 0.9.17-alpha.6

package/dist/indexer/graph/graph-extraction.js

@@ -42,19 +42,16 @@ import path from "node:path";
 import { stashDirFor } from "../../core/asset/asset-placement.js";
 import { parseFrontmatter } from "../../core/asset/frontmatter.js";
 import { concurrentMap } from "../../core/concurrent.js";
-import { getIndexPassConfig, loadConfig, resolveBatchSize } from "../../core/config/config.js";
-import { ConfigError, rethrowIfTestIsolationError } from "../../core/errors.js";
+import { getIndexPassConfig, resolveBatchSize } from "../../core/config/config.js";
+import { ConfigError } from "../../core/errors.js";
 import { warn, warnVerbose } from "../../core/warn.js";
 import { assertRunnerCredentials } from "../../integrations/agent/runner-dispatch.js";
 import { isProcessEnabled } from "../../llm/feature-gate.js";
 import * as graphExtract from "../../llm/graph-extract.js";
 import { resolveIndexPassExecution } from "../../llm/index-passes.js";
 import { computeBodyHash, getLlmCacheEntriesByRefs, upsertLlmCacheEntry, } from "../../storage/repositories/index-llm-cache-repository.js";
-import { GRAPH_SCHEMA_VERSION } from "../../storage/repositories/index-schema.js";
-import { acknowledgeExtractionQueueEntry, enqueueGraphExtraction, loadStoredGraphMeta, loadStoredGraphSnapshot, peekExtractionQueue, replaceStoredGraph, } from "../db/graph-db.js";
+import { loadStoredGraphMeta, loadStoredGraphSnapshot, replaceStoredGraph } from "../db/graph-db.js";
 import { walkMarkdownFiles } from "../walk/walker.js";
-/** Schema version for the persisted artifact — bumps trigger a full rebuild. */
-export const GRAPH_FILE_SCHEMA_VERSION = GRAPH_SCHEMA_VERSION;
 /**
  * The frozen execution a graph call runs under: the invocation's own runner
  * when the caller passed one (it already passed its own gates), otherwise the
@@ -94,13 +91,7 @@ const EMPTY_RESULT = {
     },
     warnings: [],
 };
-export const DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES = ["memory", "knowledge"];
-/**
- * Max number of lazy-extraction queue rows drained per pass (#624-P3). Bounds
- * per-run work so a large backlog is spread across runs rather than processed
- * all at once. Generous default — the queue is normally near-empty.
- */
-const GRAPH_EXTRACTION_QUEUE_DRAIN_LIMIT = 100;
+const DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES = ["memory", "knowledge"];
 const SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES = new Set([
     "memory",
     "knowledge",
@@ -128,6 +119,31 @@ export function getGraphExtractorId(config) {
     })).slice(0, 16);
     return `${GRAPH_CACHE_VARIANT_PREFIX}:${graphExtract.GRAPH_EXTRACT_PROMPT_VERSION}:${config.model}:${fingerprint}`;
 }
+/**
+ * GR-D16: one notice when this run's extractor differs from the one that last
+ * wrote the graph. Cached extractions are keyed by extractor, so a config
+ * change that alters it (model, batch size, included types, prompt version)
+ * re-extracts every cached file; the notice says which change and how many.
+ */
+function extractorChangeNotice(args) {
+    const { previous, current, files, db } = args;
+    if (!previous?.extractorId || previous.extractorId === current.extractorId)
+        return undefined;
+    const cachedUnder = (cacheVariant) => new Set(planEligibleGraphExtractions({ eligible: files, db, reEnrich: false, cacheVariant })
+        .filter((plan) => plan.kind === "cache-hit")
+        .map((plan) => plan.candidate.absPath));
+    const stillCached = cachedUnder(current.extractorId);
+    const reextracted = [...cachedUnder(previous.extractorId)].filter((file) => !stillCached.has(file)).length;
+    const changes = [
+        previous.model !== current.model ? `model ${previous.model} -> ${current.model}` : undefined,
+        previous.batchSize !== current.batchSize ? `batch size ${previous.batchSize} -> ${current.batchSize}` : undefined,
+        previous.promptVersion !== current.promptVersion
+            ? `prompt ${previous.promptVersion} -> ${current.promptVersion}`
+            : undefined,
+    ].filter((change) => change !== undefined);
+    return (`graph extraction: the extractor changed (${changes.join(", ") || "included asset types"}), ` +
+        `so ${reextracted} file(s) with a cached extraction will be extracted again.`);
+}
 function buildLowQualityWarnings(quality, telemetry) {
     const warnings = [];
     if (quality.consideredFiles >= 5 && quality.extractionCoverage < 0.3) {
@@ -282,21 +298,6 @@ function mergeGraphNodes(previousNodes, refreshedNodes, keptPaths) {
     merged.push(...refreshedByPath.values());
     return merged;
 }
-/** A previous node (validated by {@link loadGraphFile}) for this exact body, unless it failed. */
-function reuseGraphNode(previousNodes, candidate, bodyHash) {
-    const node = previousNodes.get(candidate.absPath);
-    if (!node || node.type !== candidate.type || node.bodyHash !== bodyHash)
-        return undefined;
-    if (isFailedExtractionStatus(node.status))
-        return undefined;
-    return {
-        entities: node.entities,
-        relations: node.relations,
-        confidence: node.confidence,
-        ...(node.status ? { status: node.status } : {}),
-        ...(node.reason ? { reason: node.reason } : {}),
-    };
-}
 /**
  * A file is a cache hit only through `llm_enrichment_cache`, whose variant is
  * the extractor id. A stored graph node is never reused here: the graph keeps
@@ -421,69 +422,6 @@ async function extractGraphBatches(args) {
     }, llmRunner.connection.concurrency ?? 1);
     return { results, ...(configFailure ? { configFailure } : {}) };
 }
-function readCurrentGraphBodyHash(filePath) {
-    try {
-        const body = parseFrontmatter(fs.readFileSync(filePath, "utf8")).content.trim();
-        return body ? computeBodyHash(body) : undefined;
-    }
-    catch {
-        return undefined;
-    }
-}
-function planQueuedGraphExtractions(args) {
-    const { db, stashRoot, previousNodes, signal, reEnrich } = args;
-    return peekExtractionQueue(db, stashRoot, GRAPH_EXTRACTION_QUEUE_DRAIN_LIMIT).map((queued) => {
-        const base = { filePath: queued.filePath, queuedBodyHash: queued.bodyHash, priority: queued.priority };
-        if (signal?.aborted)
-            return { kind: "deferred", ...base };
-        const currentBodyHash = readCurrentGraphBodyHash(queued.filePath);
-        if (!currentBodyHash)
-            return { kind: "discard", ...base };
-        const type = inferGraphTypeForPath(stashRoot, queued.filePath) ?? "memory";
-        const hit = !reEnrich && reuseGraphNode(previousNodes, { absPath: queued.filePath, type }, currentBodyHash);
-        return { kind: hit ? "hit" : "model", ...base, currentBodyHash };
-    });
-}
-async function executeQueuedGraphPlans(args) {
-    const { plans, db, stashRoot, featureConfig, signal, llmRunner, onNotices } = args;
-    let graphChanged = false;
-    const acknowledgements = [];
-    for (const plan of plans) {
-        if (signal?.aborted || plan.kind === "deferred")
-            break;
-        // The body this plan settled: none for a discard (the file was gone or empty).
-        let settledBodyHash;
-        if (plan.kind === "model") {
-            const outcome = await extractGraphForSingleFileRevision(db, stashRoot, plan.filePath, {
-                config: featureConfig,
-                signal,
-                llmRunner,
-                onNotices,
-            });
-            if (!outcome.written)
-                continue;
-            graphChanged = true;
-            settledBodyHash = outcome.bodyHash;
-        }
-        else if (plan.kind === "hit") {
-            settledBodyHash = plan.currentBodyHash;
-        }
-        // A body that changed since it was planned goes back on the queue.
-        const currentBodyHash = readCurrentGraphBodyHash(plan.filePath);
-        if (currentBodyHash !== settledBodyHash) {
-            if (currentBodyHash)
-                enqueueGraphExtraction(db, stashRoot, plan.filePath, currentBodyHash, plan.priority);
-            continue;
-        }
-        acknowledgements.push({ filePath: plan.filePath, queuedBodyHash: plan.queuedBodyHash });
-    }
-    return { graphChanged, acknowledgements };
-}
-function acknowledgeQueuedGraphPlans(db, stashRoot, execution) {
-    for (const intent of execution.acknowledgements) {
-        acknowledgeExtractionQueueEntry(db, stashRoot, intent.filePath, intent.queuedBodyHash);
-    }
-}
 /**
  * Top-level entry point. Returns a no-op result when the pass is disabled.
  *
@@ -546,31 +484,16 @@ export async function runGraphExtractionPass(ctx) {
         return emptyResult();
     }
     const includeTypes = options.includeTypes ?? getGraphExtractionIncludeTypes(config);
-    let previousGraph = loadGraphFile(primary.path, db);
-    const previousNodes = new Map(previousGraph.files.map((node) => [node.path, node]));
+    const previousGraph = loadGraphFile(primary.path, db);
     const batchSize = resolveBatchSize(options.batchSize ?? getIndexPassConfig(config.index, "graph")?.graphExtractionBatchSize, llmRunner.connection.contextLength);
     const extractorId = getGraphExtractorId({ model: llmRunner.connection.model, batchSize, includeTypes });
-    const queuePlans = planQueuedGraphExtractions({
-        db,
-        stashRoot: primary.path,
-        previousNodes,
-        signal,
-        reEnrich,
-    });
-    const queuedPaths = new Set(queuePlans.map((plan) => plan.filePath));
     const scan = collectEligibleFiles(primary.path, includeTypes);
     // The stored nodes this run keeps without touching them: every eligible file
-    // (outside candidatePaths or topN, or never reached before an abort) and every
-    // file the queue handled. Only a node whose file left the eligible set — gone,
-    // emptied, inferred, or of a type no longer included — is dropped, and an
-    // incomplete scan drops nothing.
-    const keptPaths = scan.complete
-        ? new Set([
-            ...scan.files.map((file) => file.absPath),
-            ...queuePlans.filter((plan) => plan.kind !== "discard").map((plan) => plan.filePath),
-        ])
-        : undefined;
-    let eligible = scan.files.filter((candidate) => (!options.candidatePaths || options.candidatePaths.has(candidate.absPath)) && !queuedPaths.has(candidate.absPath));
+    // (outside candidatePaths or topN, or never reached before an abort). Only a
+    // node whose file left the eligible set — gone, emptied, inferred, or of a
+    // type no longer included — is dropped, and an incomplete scan drops nothing.
+    const keptPaths = scan.complete ? new Set(scan.files.map((file) => file.absPath)) : undefined;
+    let eligible = scan.files.filter((candidate) => !options.candidatePaths || options.candidatePaths.has(candidate.absPath));
     // P2 (#624): when topN is set, rank the (already candidate-filtered)
     // eligible set by utility_scores DESC and keep only the top-N. Unset issues
     // no ranking query. Ranking composes WITH the candidatePaths filter:
@@ -582,25 +505,12 @@ export async function runGraphExtractionPass(ctx) {
     const eligiblePlans = planEligibleGraphExtractions({ eligible, db, reEnrich, cacheVariant: extractorId });
     if (signal?.aborted)
         return emptyResult();
-    // Validate exactly once iff classification found real model work. Queue
-    // acknowledgements, cache writes, and graph replacement all happen after
-    // this boundary, so a missing credential cannot partially mutate a batch.
-    if ([...queuePlans, ...eligiblePlans].some((plan) => plan.kind === "model"))
+    // Validate exactly once iff classification found real model work. Cache
+    // writes and graph replacement happen after this boundary, so a missing
+    // credential cannot partially mutate a batch.
+    if (eligiblePlans.some((plan) => plan.kind === "model"))
         assertRunnerCredentials(llmRunner);
-    const queueExecution = await executeQueuedGraphPlans({
-        plans: queuePlans,
-        db,
-        stashRoot: primary.path,
-        featureConfig,
-        signal,
-        llmRunner,
-        onNotices,
-    });
-    if (queueExecution.graphChanged) {
-        previousGraph = loadGraphFile(primary.path, db);
-    }
     if (considered === 0) {
-        acknowledgeQueuedGraphPlans(db, primary.path, queueExecution);
         const scoped = options.candidatePaths ? ` matching ${options.candidatePaths.size} candidate path(s)` : "";
         warnVerbose(`graph extraction: skipped because no eligible files${scoped} were found under ${primary.path}. ` +
             `includeTypes=${includeTypes.join(",")}`);
@@ -655,6 +565,19 @@ export async function runGraphExtractionPass(ctx) {
         nonArrayBatchFailures: 0,
     };
     const abortState = { attempts: 0, failures: 0, aborted: false };
+    const extractorNotice = extractorChangeNotice({
+        previous: previousGraph.telemetry,
+        current: {
+            extractorId,
+            model: llmRunner.connection.model,
+            batchSize,
+            promptVersion: graphExtract.GRAPH_EXTRACT_PROMPT_VERSION,
+        },
+        files: scan.files,
+        db,
+    });
+    if (extractorNotice)
+        warn(extractorNotice);
     warnVerbose(`graph extraction: starting for ${considered} eligible file(s) under ${primary.path}; ` +
         `includeTypes=${includeTypes.join(",")}, batchSize=${batchSize}, concurrency=${llmRunner.connection.concurrency ?? 1}, ` +
         `reEnrich=${reEnrich === true}, candidateScoped=${options.candidatePaths ? "true" : "false"}.`);
@@ -677,7 +600,6 @@ export async function runGraphExtractionPass(ctx) {
     });
     if (configFailure)
         throw configFailure;
-    acknowledgeQueuedGraphPlans(db, primary.path, queueExecution);
     // A failed attempt says nothing about the file, so a stored node for it stays
     // as it was; only a file with no stored node records the failure.
     const storedPaths = new Set(previousGraph.files.map((node) => node.path));
@@ -690,11 +612,17 @@ export async function runGraphExtractionPass(ctx) {
     telemetry.htmlErrorCount = runtimeTelemetry.htmlErrorCount ?? 0;
     telemetry.retryAttempts = runtimeTelemetry.retryAttempts ?? 0;
     telemetry.nonArrayBatchFailures = runtimeTelemetry.nonArrayBatchFailures ?? 0;
+    telemetry.filteredGenericEntities = runtimeTelemetry.filteredGenericEntities ?? 0;
+    telemetry.filteredInvalidRelations = runtimeTelemetry.filteredInvalidRelations ?? 0;
+    telemetry.filteredLowConfidenceRelations = runtimeTelemetry.filteredLowConfidenceRelations ?? 0;
+    telemetry.contextBatchRetries = runtimeTelemetry.contextBatchRetries ?? 0;
     telemetry.aborted = abortState.aborted;
     const graph = buildGraphFile(primary.path, mergeGraphNodes(previousGraph.files, nodes, keptPaths), telemetry);
     const written = writeGraphFile(db, graph);
     const quality = loadStoredGraphMeta(primary.path, db)?.quality ?? EMPTY_RESULT.quality;
     const warnings = buildLowQualityWarnings(quality, telemetry);
+    if (extractorNotice)
+        warnings.push(extractorNotice);
     if (abortState.message)
         warnings.push(abortState.message);
     for (const warning of warnings)
@@ -714,14 +642,28 @@ export async function runGraphExtractionPass(ctx) {
         ...(noticesByKey.size > 0 ? { notices: Object.freeze([...noticesByKey.values()]) } : {}),
     };
 }
-/** The persisted node for one extraction outcome (entities and relations trimmed, deduplicated). */
+/**
+ * The persisted node for one extraction outcome: entities trimmed and kept
+ * once per {@link graphExtract.normalizeEntityKey} (the first form wins),
+ * relations trimmed.
+ */
 function toGraphNode(record, extractionRunId) {
     const confidence = normalizeConfidence(record.confidence);
+    const entityKeys = new Set();
+    const entities = record.entities
+        .map((entity) => entity.trim())
+        .filter((entity) => {
+        const key = graphExtract.normalizeEntityKey(entity);
+        if (!key || entityKeys.has(key))
+            return false;
+        entityKeys.add(key);
+        return true;
+    });
     return {
         path: record.absPath,
         type: record.type,
         bodyHash: record.bodyHash,
-        entities: [...new Set(record.entities.map((entity) => entity.trim()).filter(Boolean))],
+        entities,
         relations: record.relations
             .map((r) => ({
             from: r.from.trim(),
@@ -739,107 +681,12 @@ function toGraphNode(record, extractionRunId) {
 /** The graph snapshot to store for `files`; its counts are derived from the stored rows on write. */
 function buildGraphFile(stashRoot, files, telemetry) {
     return {
-        schemaVersion: GRAPH_FILE_SCHEMA_VERSION,
         generatedAt: new Date().toISOString(),
         stashRoot,
         files,
         ...(telemetry ? { telemetry } : {}),
     };
 }
-/**
- * Infer the asset type (`memory`, `knowledge`, …) for a path from the stash
- * directory segment it lives under. Returns the matching include-type, or
- * `undefined` when the path is not under a known graph-eligible type dir.
- */
-function inferGraphTypeForPath(stashRoot, absPath) {
-    const rel = path.relative(stashRoot, absPath);
-    const firstSeg = rel.split(path.sep)[0];
-    if (!firstSeg)
-        return undefined;
-    for (const type of SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES) {
-        if (stashDirFor(type) === firstSeg)
-            return type;
-    }
-    return undefined;
-}
-/**
- * #624-P3 — extract graph data for a SINGLE file and merge it into the stored
- * graph WITHOUT clobbering other files' rows.
- *
- * Re-reads the body from disk at call time (the queued body_hash is NOT trusted
- * blindly — the file may have been deleted or changed since enqueue) and skips
- * silently (returns `false`) when the file is gone or empty. Resolves a frozen
- * LLM execution via {@link resolveIndexPassExecution} (model-available guard:
- * returns `false` when no provider is configured) UNLESS a caller supplies
- * `opts.llmRunner` or `opts.llmOverride`. Those seams let a command reuse an
- * invocation-owned selection or provide a test extractor without re-resolving.
- *
- * Returns `true` when a graph row was written for the file, `false` on any
- * skip (missing file, empty body, unknown type, no model, or extraction error).
- */
-async function extractGraphForSingleFileRevision(db, stashRoot, filePath, opts) {
-    try {
-        // Re-read from disk — never trust a stale queued body.
-        let raw;
-        try {
-            raw = fs.readFileSync(filePath, "utf8");
-        }
-        catch {
-            return { written: false }; // file gone / unreadable → silent skip
-        }
-        const parsed = parseFrontmatter(raw);
-        const body = parsed.content.trim();
-        if (!body)
-            return { written: false };
-        const type = inferGraphTypeForPath(stashRoot, filePath) ?? "memory";
-        const effectiveHash = computeBodyHash(body);
-        // Extract — via the injected seam, or the real per-asset path.
-        let extraction;
-        if (opts?.llmOverride) {
-            extraction = await opts.llmOverride(body);
-        }
-        else {
-            const selection = selectGraphExecution(opts ?? {}, opts?.config ?? loadConfig());
-            if (!selection)
-                return { written: false };
-            opts?.onNotices?.(selection.execution.notices);
-            const llmRunner = selection.execution.runner;
-            if (!llmRunner)
-                return { written: false }; // model-available guard
-            extraction = await graphExtract.extractGraphFromBody(llmRunner, body, opts?.signal, selection.featureConfig, undefined, { ...(opts?.onNotices ? { onNotices: opts.onNotices } : {}) });
-        }
-        // A single-file refresh records only entities, relations and confidence;
-        // its status follows from the entities that survive trimming.
-        const entities = [...new Set(extraction.entities.map((e) => e.trim()).filter(Boolean))];
-        const node = toGraphNode({
-            absPath: filePath,
-            type,
-            bodyHash: effectiveHash,
-            entities,
-            relations: extraction.relations,
-            ...(extraction.confidence !== undefined ? { confidence: extraction.confidence } : {}),
-        }, crypto.randomUUID());
-        // Merge with the previously-stored nodes, replacing JUST this path so other
-        // files' rows are preserved (and graph_meta counts refresh).
-        const previousGraph = loadGraphFile(stashRoot, db);
-        const graph = buildGraphFile(stashRoot, mergeGraphNodes(previousGraph.files, [node]), previousGraph.telemetry);
-        return writeGraphFile(db, graph) ? { written: true, bodyHash: effectiveHash } : { written: false };
-    }
-    catch (err) {
-        if (err instanceof ConfigError)
-            throw err;
-        rethrowIfTestIsolationError(err);
-        // A genuine extraction/write failure, distinct from the deliberate
-        // "nothing to do" skips above (missing file, empty body, no model). Warn
-        // so it is visible instead of looking identical to a no-op skip; the
-        // entry stays queued and is retried on the next pass.
-        warn(`graph extraction: failed to extract graph for ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
-        return { written: false };
-    }
-}
-export async function extractGraphForSingleFile(db, stashRoot, filePath, opts) {
-    return (await extractGraphForSingleFileRevision(db, stashRoot, filePath, opts)).written;
-}
 // ── Eligible-file detection ─────────────────────────────────────────────────
 /**
  * Rank eligible graph-extraction candidates by their entry `utility_scores`,

package/dist/indexer/graph/graph-related.js

@@ -31,13 +31,14 @@ export function listRelatedPathsForFile(stashRoot, filePath, limit = 5, db) {
     if (row === undefined)
         return [];
     const effectiveLimit = Math.max(1, limit);
-    // Shared-entity count per candidate file_path. The target's entities are the
-    // rows for `filePath`; candidates are any OTHER file_path in the stash that
-    // shares a normalized entity.
+    // Distinct shared entities per candidate file_path. The target's entities
+    // are the rows for `filePath`; candidates are any OTHER file_path in the
+    // stash that shares a normalized entity. Counting distinct keys keeps two
+    // stored forms of one entity (rows older extractors wrote) from counting twice.
     const candidateRows = db
         .prepare(`SELECT gf.file_path   AS file_path,
               gf.file_type   AS file_type,
-              COUNT(*)       AS shared
+              COUNT(DISTINCT e.entity_norm) AS shared
           FROM graph_file_entities target
           JOIN graph_file_entities e
             ON e.stash_root = target.stash_root

package/dist/indexer/search/db-search.js

@@ -239,11 +239,14 @@ async function searchDatabase(db, input) {
 }
 /**
  * Walk the fused list in order, loading entries a batch at a time, and keep
- * the first `limit` that survive path deduplication and the filters.
+ * the first `limit` that survive path deduplication and the filters. An entry
+ * whose indexed content is identical to a kept one's (the same body saved
+ * under another name or in another bundle) is a duplicate and is skipped.
  */
 function selectFusedEntries(db, fused, limit, filterOptions) {
     const selected = [];
     const seenPaths = new Set();
+    const seenContent = new Set();
     const batchSize = Math.max(limit * 2, 20);
     for (let offset = 0; offset < fused.length && selected.length < limit; offset += batchSize) {
         const batch = [];
@@ -256,6 +259,12 @@ function selectFusedEntries(db, fused, limit, filterOptions) {
         }
         for (const kept of applyEntryFilters(batch, filterOptions)) {
             const { candidate, ...row } = kept;
+            const content = row.entry.content?.replace(/\s+/g, " ").trim();
+            if (content) {
+                if (seenContent.has(content))
+                    continue;
+                seenContent.add(content);
+            }
             selected.push({ candidate, row });
         }
     }

package/dist/llm/feature-gate.js

@@ -79,9 +79,6 @@ export function isProcessEnabled(section, processName, config) {
         return false;
     // Index passes are first-class 0.9 entries.
     if (section === "index") {
-        if (processName === "metadata_enhance" || processName === "metadataEnhance") {
-            return config.index?.metadataEnhance?.enabled ?? true;
-        }
         if (processName === "memory_inference" || processName === "memoryInference") {
             return isLlmFeatureEnabled(config, "memory_inference");
         }