akm-cli 0.9.16-alpha.1 → 0.9.16

package/dist/indexer/passes/dir-staleness.js

@@ -0,0 +1,161 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Incremental dir-staleness engine.
+ *
+ * Decides, per stash directory, whether the directory's indexed rows are still
+ * fresh relative to what is on disk — so an incremental `akm index` run can
+ * skip unchanged directories instead of regenerating their metadata.
+ *
+ * Two persisted signals back the decision:
+ *   1. The `entries` rows already indexed for the directory (`getEntriesByDir`).
+ *   2. The `index_dir_state` row (`getIndexDirState`): the fingerprint of the
+ *      directory's walked file set (basename set + max mtime, `computeDirFingerprint`)
+ *      as of its last drain, plus the row count that drain persisted.
+ *
+ * `getCachedDirState` is the pre-drain gate (#900): a directory whose walked
+ * fingerprint still matches its row is skipped before any file is read.
+ */
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { compareCodePoints } from "../../core/common.js";
+import { getEntriesByDir } from "../../storage/repositories/index-entries-repository.js";
+import { getIndexDirState } from "../../storage/repositories/index-meta-repository.js";
+/**
+ * Post-drain freshness verdict. `files` is the recognized file set the drain
+ * produced (compared against the persisted entries); `fingerprint` is the
+ * walked-set fingerprint compared against the persisted row and defaults to
+ * one computed over `files`.
+ */
+export function getDirIndexState(db, dirPath, files, builtAtMs, indexVariant = "", fingerprint = computeDirFingerprint(dirPath, files, indexVariant)) {
+    const prevEntries = getEntriesByDir(db, dirPath);
+    if (prevEntries.length > 0) {
+        const staleReason = getDirStaleReason(dirPath, files, prevEntries, builtAtMs);
+        if (staleReason)
+            return { stale: true, reason: staleReason, persistedRowCount: prevEntries.length };
+        const cachedState = getIndexDirState(db, dirPath);
+        if (!cachedState || cachedState.fileSetHash !== fingerprint.fileSetHash) {
+            return {
+                stale: true,
+                reason: { kind: "index-context-changed", detail: indexVariant },
+                persistedRowCount: prevEntries.length,
+            };
+        }
+        return { stale: false, reason: { kind: "unchanged" }, persistedRowCount: prevEntries.length };
+    }
+    const cachedState = getIndexDirState(db, dirPath);
+    if (cachedState && cachedState.fileSetHash === fingerprint.fileSetHash) {
+        return {
+            stale: false,
+            reason: { kind: "cached-zero-row-state", detail: cachedState.reason },
+            persistedRowCount: 0,
+        };
+    }
+    return {
+        stale: true,
+        reason: { kind: "no-previous-rows", detail: cachedState ? `cached=${cachedState.reason}` : undefined },
+        persistedRowCount: 0,
+    };
+}
+/**
+ * Pre-drain gate (#900). A directory whose walked-set fingerprint matches its
+ * persisted row cannot recognize differently than last time, so it is skipped
+ * before `drainDirDocuments` reads a single file. A row that recorded a real
+ * generation (`rowCount > 0`) is skipped outright; a zero-row or pre-#900 row
+ * goes through the entries-aware check so the dedup-order guard still applies.
+ */
+export function getCachedDirState(db, dirPath, files, builtAtMs, priorDirsChanged, indexVariant, fingerprint) {
+    const cached = getIndexDirState(db, dirPath);
+    if (!cached || cached.fileSetHash !== fingerprint.fileSetHash)
+        return undefined;
+    if (cached.rowCount !== undefined && cached.rowCount > 0) {
+        return { stale: false, reason: { kind: "unchanged-precheck" }, persistedRowCount: cached.rowCount };
+    }
+    const state = getDirIndexState(db, dirPath, files, builtAtMs, indexVariant, fingerprint);
+    if (state.stale || state.reason.kind !== "cached-zero-row-state")
+        return undefined;
+    if (!canUseIncrementalSkip(state, priorDirsChanged))
+        return undefined;
+    return state;
+}
+export function canUseIncrementalSkip(state, priorDirsChanged) {
+    return !(priorDirsChanged &&
+        state.reason.kind === "cached-zero-row-state" &&
+        state.reason.detail === "deduped-zero-row");
+}
+export function computeDirFingerprint(_dirPath, files, indexVariant = "") {
+    // One `statSync` per file — the same call this function has always made — but
+    // every field it returns that can witness a change is kept, per file, instead
+    // of being collapsed into a single max.
+    //
+    // `Math.max` over mtimes discarded everything except the newest file, so an
+    // edit to any other file landed below the max and was invisible; and mtime
+    // alone is writable by ordinary tooling (`touch -r`, `rsync --times`,
+    // `cp -p`, archive extraction), so a restored timestamp hid an edit outright.
+    // Size catches any length-changing edit; ctime catches the rest, because
+    // utimes(2) cannot hold the inode's change time back.
+    //
+    // This is still a heuristic: ctime also moves on metadata-only changes
+    // (chmod/chown) and after copying a tree, which costs an unnecessary rescan.
+    // That direction is safe — extra work, never stale content.
+    const entries = [];
+    let fileMtimeMaxMs = 0;
+    for (const file of [...new Set(files)].sort(compareCodePoints)) {
+        const name = path.basename(file);
+        try {
+            // `bigint: true` is the same syscall but reports nanoseconds. Millisecond
+            // floats would let an edit made inside the same millisecond as the last
+            // run's stat land on an identical digest.
+            const stat = fs.statSync(file, { bigint: true });
+            fileMtimeMaxMs = Math.max(fileMtimeMaxMs, Number(stat.mtimeMs));
+            entries.push(`${name}\0${stat.size}\0${stat.mtimeNs}\0${stat.ctimeNs}`);
+        }
+        catch {
+            // Unreadable or vanished: record it as such so the digest differs from
+            // any run where the file could be read, forcing a rescan.
+            entries.push(`${name}\0unreadable`);
+        }
+    }
+    const digest = createHash("sha256")
+        .update([indexVariant, ...entries].join("\n"), "utf8")
+        .digest("hex");
+    return { fileSetHash: digest, fileMtimeMaxMs };
+}
+function getDirStaleReason(_dirPath, currentFiles, previousEntries, builtAtMs) {
+    const prevFileNames = new Set(previousEntries
+        .map((ie) => {
+        const fromPath = path.basename(ie.filePath);
+        return fromPath || ie.entry.filename;
+    })
+        .filter((e) => !!e));
+    const currFileNames = new Set(currentFiles.map((f) => path.basename(f)));
+    if (prevFileNames.size !== currFileNames.size) {
+        return { kind: "file-set-changed", detail: `${prevFileNames.size} -> ${currFileNames.size} files` };
+    }
+    for (const name of currFileNames) {
+        if (!prevFileNames.has(name))
+            return { kind: "file-set-changed", detail: name };
+    }
+    for (const file of currentFiles) {
+        try {
+            if (fs.statSync(file).mtimeMs > builtAtMs)
+                return { kind: "mtime-changed", detail: path.basename(file) };
+        }
+        catch {
+            return { kind: "missing-file", detail: path.basename(file) };
+        }
+    }
+    return undefined;
+}
+export function inferZeroRowReason(stash, priorReason, warnings, dirPath, dedupedRows) {
+    if (dedupedRows > 0)
+        return "deduped-zero-row";
+    const workflowNoise = warnings.some((warning) => warning.startsWith("Skipped workflow ") && warning.includes(dirPath));
+    if (workflowNoise)
+        return "workflow-noise";
+    if (!stash || stash.entries.length === 0)
+        return "empty-generated-set";
+    return `zero-row:${priorReason?.kind ?? "unknown"}`;
+}

package/dist/indexer/passes/metadata.js

@@ -1349,24 +1349,7 @@ export function applyPostContributorFields(entry, file, canonicalName, dirPath)
     // filename word. `normalizeTerms` below dedupes author-restated tokens.
     entry.tags = [...(entry.tags ?? []), ...extractDirTagsFromName(canonicalName)];
     entry.tags = normalizeTerms(entry.tags ?? []);
-    // fix-ranking-derived-outranks-primary: `.derived` is a structural marker
-    // on a memory's OWN filename (memory-inference.ts's `derivedChildPath`/
-    // `isDerivedByName` convention: `<parent>.derived.md`), not a topical
-    // word — `extractTagsFromPath` above tokenizes on `.` alongside `-`/`_`,
-    // so every derived twin's filename contributes a "derived" tag purely as
-    // an artifact of that convention (kept in `entry.tags` itself; only the
-    // ALIAS input below is filtered, so anything else keyed on the raw tag
-    // set is unaffected). Left in, `buildAliases`' tags.join(" ") mints a
-    // SYNTHETIC "<base> derived" alias purely because tags.length > 1, which
-    // then wins `alias-ranking` credit any time the base name is a query
-    // token — see `exactNameRankingContributor`, which already strips this
-    // exact suffix before treating a derived twin's name as content, for the
-    // identical reason. Scoped to memory so a coincidentally
-    // ".derived"-suffixed asset of another type is untouched.
-    const aliasTagInput = entry.type === "memory" && canonicalName.toLowerCase().endsWith(".derived")
-        ? entry.tags.filter((tag) => tag !== "derived")
-        : entry.tags;
-    entry.aliases = mergeAliases(entry.aliases, buildAliases(canonicalName, aliasTagInput));
+    entry.aliases = mergeAliases(entry.aliases, buildAliases(canonicalName, entry.tags));
     // Search hints are only generated when LLM is configured (via enhanceStashWithLlm)
     // Heuristic search hints are too noisy to be useful for search quality
     entry.filename = path.basename(file);

package/dist/indexer/scan/drain-dir.js

@@ -5,18 +5,19 @@
  * Per-directory document drain — akm 0.9.0 Chunk 5, milestone F4a M-core-2 (the
  * engine swap). Replaces the live indexer's per-dir flat-walk matcher-pass
  * `IndexDocument` stream with the `akm` adapter's `recognize` `IndexDocument`
- * stream, reconstructing the durable `IndexDocument` via {@link parseFileDocument}
- * (proven lossless by the shadow-parity gate).
+ * stream, reconstructing the durable `IndexDocument` via {@link
+ * indexDocumentToStashEntry} (proven lossless by the shadow-parity gate).
  *
- * One directory-scoped behavior the per-file parse does NOT carry, restored
- * here at the drain layer (spec §14.2 "drain the full document stream"):
+ * Two behaviors the adapter fold does NOT carry, restored here at the drain
+ * layer (spec §14.2 "drain the full document stream"):
  *
- *  - **Peer-workflow-format ownership arbitration.** A full directory drain
- *    may contain both peer workflow formats for one canonical ref; resolving
- *    which one owns the ref requires seeing every file in the directory at
- *    once, so it stays here rather than in the single-file
- *    {@link parseFileDocument} (index-redesign B1 also calls that function to
- *    parse one already-known-changed file, with no peer files in view).
+ *  - **Broken-workflow drop (item 3).** The live path dropped a broken workflow
+ *    via the renderer contributor's throw → metadata-pass skip-with-warning; the
+ *    `akm` adapter's synchronous `foldRecognizedMetadata` SWALLOWS the parse
+ *    error, so a broken workflow would otherwise silently index. We run the
+ *    shared source-IR compiler on drained workflow docs and DROP
+ *    the entry with the same `Skipped workflow …` warning
+ *    ({@link buildMetadataSkipWarning}), so the workflow-skip summary counts it.
  *
  * `doc.hash` (= sha256 of the file content) is surfaced per recognized file so
  * the persist layer can populate the `content_hash` column (item 2). It is keyed
@@ -30,8 +31,12 @@ import { akmAdapter } from "../../core/adapter/adapters/akm-adapter.js";
 import { compareCodePoints } from "../../core/common.js";
 import { canonicalizeWorkflowName } from "../../core/recognition-util.js";
 import { resolveWorkflowSourceDomains, workflowNameForSourcePath } from "../../workflows/source-files.js";
+import { compileWorkflowSource } from "../../workflows/source-ir/compile.js";
+import { buildMetadataSkipWarning } from "../passes/metadata.js";
 import { buildFileContext } from "../walk/file-context.js";
-import { parseFileDocument } from "./parse-file.js";
+import { indexDocumentToStashEntry } from "./doc-to-entry.js";
+/** The markdown-workflow renderer name the `akm` adapter carries on `documentJson.renderer`. */
+const WORKFLOW_MD_RENDERER = "workflow-md";
 /**
  * Drain one directory's recognized documents into durable entries.
  *
@@ -83,24 +88,26 @@ export function drainDirDocuments(adapter, component, fileContexts) {
                 continue;
             }
         }
-        // The recognize → conceptId check → workflow-validity fold is the SAME
-        // per-file parse the reconcile engine uses for a single changed file
-        // (index-redesign B1) — extracted to `parse-file.ts` so the two never
-        // drift apart.
-        const outcome = parseFileDocument(adapter, component, file);
-        if (outcome.parsed === null) {
-            if (outcome.warning !== null) {
-                warnings.push(outcome.warning);
-                if (outcome.isWorkflowDrop && workflowName !== undefined) {
-                    invalidWorkflowOwnerNames.add(canonicalizeWorkflowName(workflowName));
-                }
-            }
+        const doc = adapter.recognize(component, file);
+        if (doc === null)
+            continue;
+        if (!doc.conceptId) {
+            warnings.push(`Skipped ${file.absPath}: adapter "${adapter.id}" returned no conceptId.`);
+            continue;
+        }
+        const entry = indexDocumentToStashEntry(doc);
+        // Workflow docs: drop-with-warning if broken; otherwise cache a lossless
+        // runtime projection when the current executor can represent the source.
+        const dropWarning = handleWorkflowDoc(doc, file, component.root);
+        if (dropWarning !== null) {
+            warnings.push(dropWarning);
+            if (workflowName !== undefined)
+                invalidWorkflowOwnerNames.add(canonicalizeWorkflowName(workflowName));
             continue;
         }
-        const { entry, hash, conceptId } = outcome.parsed;
-        if (hash !== undefined)
-            hashByFile.set(file.absPath, hash);
-        conceptIdByFile.set(file.absPath, conceptId);
+        if (doc.hash !== undefined)
+            hashByFile.set(file.absPath, doc.hash);
+        conceptIdByFile.set(file.absPath, doc.conceptId);
         entries.push(entry);
     }
     return { entries, warnings, hashByFile, conceptIdByFile, rejectedPaths, rejectedConceptIds };
@@ -126,3 +133,39 @@ export function recognizeStashEntries(stashRoot, files) {
         ? { entries: drained.entries, warnings: drained.warnings }
         : { entries: drained.entries };
 }
+/**
+ * If `doc` is a workflow, compile it through source IR: return a
+ * `Skipped workflow …` drop warning when it is broken, or return `null` when
+ * it compiles. Non-workflow docs return `null` immediately.
+ */
+function handleWorkflowDoc(doc, file, workspaceRoot) {
+    if (doc.type !== "workflow" ||
+        (doc.adapterId !== "akm" && doc.adapterId !== "akm-workflow") ||
+        (docRenderer(doc) !== WORKFLOW_MD_RENDERER && doc.adapterId !== "akm-workflow")) {
+        return null;
+    }
+    const result = compileWorkflowSource(file.content(), { path: file.relPath, workspaceRoot });
+    if (!result.ok)
+        return workflowDropWarning(file, result.errors);
+    return null;
+}
+/** The winning renderer name the `akm` adapter carries on `documentJson.renderer`, or `undefined`. */
+function docRenderer(doc) {
+    const dj = doc.documentJson;
+    if (dj !== null && typeof dj === "object" && "renderer" in dj) {
+        const renderer = dj.renderer;
+        return typeof renderer === "string" ? renderer : undefined;
+    }
+    return undefined;
+}
+/**
+ * Build the `Skipped workflow <path>:\n…` warning byte-for-byte the way the live
+ * pipeline did: the workflow parser's `path:line — message` summary wrapped in
+ * the `Workflow has errors:` prefix (the string `loadDocument`/`loadProgram`
+ * threw), then {@link buildMetadataSkipWarning}'s workflow branch. `startsWith
+ * "Skipped workflow "` so `isWorkflowSkipWarning` counts it for the summary.
+ */
+function workflowDropWarning(file, errors) {
+    const summary = errors.map((e) => `${file.relPath}:${e.line} — ${e.message}`).join("\n");
+    return buildMetadataSkipWarning(file.absPath, "workflow", `Workflow has errors:\n${summary}`);
+}