akm-cli 0.9.0-rc.0 → 0.9.0-rc.13

package/dist/indexer/scan/doc-to-entry.js

@@ -0,0 +1,160 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * `indexDocumentToStashEntry` — the inverse of the `akm` adapter's
+ * `indexDocumentFromEntry` (`adapters/akm-adapter.ts`). akm 0.9.0 Chunk 5,
+ * milestone F4a (M2 groundwork).
+ *
+ * The engine swap drains an `IndexDocument` stream (`drainDirDocuments` × the
+ * dispatched adapter's `recognize`) in place of the per-dir flat-walk matcher-pass
+ * `IndexDocument` stream, then persists it. But the durable `entries.entry_json`
+ * column stays a faithful `IndexDocument` — every reader (`rowToIndexedEntry` →
+ * `DbIndexedEntry.entry`) consumes it as one, and the byte-for-byte goldens pin
+ * it. So the persist path must reconstruct the exact `IndexDocument` the old
+ * pipeline stored FROM the `IndexDocument` the new pipeline produces.
+ *
+ * That reconstruction is lossless by construction: the akm adapter's `recognize`
+ * assembles a full `IndexDocument` (P1/P2 → fold → P4) and then maps it onto the
+ * `IndexDocument` via `indexDocumentFromEntry` — first-class search/signal fields
+ * onto named members, every other search-surface/provenance field onto
+ * `documentJson` (the `DOCUMENT_JSON_CARRIED_FIELDS` set). This function reverses
+ * that mapping exactly, so `indexDocumentToStashEntry(recognize(file))` deep-
+ * equals the entry the legacy flat-walk pass stored for the same file (the
+ * `tests/integration/shadow-scan-parity.test.ts` gate pins the persisted index).
+ *
+ * Two deliberate non-round-trip fields:
+ *   - `filename` — dropped by `indexDocumentFromEntry` (no IndexDocument home),
+ *     recovered here as `basename(doc.path)`. The old pipeline set it to
+ *     `basename(file)` in `applyPostContributorFields`, and `doc.path` IS that
+ *     file, so the value is identical.
+ *   - `fileSize` — never set by `recognize` NOR by the legacy flat-walk pass;
+ *     both pipelines attach it at PERSIST time (`attachFileSize`). It is therefore
+ *     intentionally absent here and added by the persist layer, exactly as
+ *     before.
+ *
+ * Pure, type-only imports (no cycle participation).
+ */
+import path from "node:path";
+/**
+ * Reconstruct the `IndexDocument` an `IndexDocument` was mapped from. First-class
+ * IndexDocument members and the `documentJson`-carried extras are both restored;
+ * every field is set only when present, matching the old pipeline's
+ * "assign-only-when-defined" assembly so the reconstruction deep-equals it.
+ */
+export function indexDocumentToStashEntry(doc) {
+    const dj = (doc.documentJson ?? {});
+    const entry = {
+        name: doc.name,
+        // `type` is a required member of the merged IndexDocument (M-core-1), so no
+        // `?? ""` fallback is needed — `recognize` always sets it.
+        type: doc.type,
+        // Dropped by indexDocumentFromEntry — recovered from the read path (= the
+        // basename the old pipeline stored). Always present on a recognized doc.
+        filename: path.basename(doc.path ?? ""),
+    };
+    // ── First-class IndexDocument members (spec §3) ──
+    if (doc.description !== undefined)
+        entry.description = doc.description;
+    if (doc.tags !== undefined)
+        entry.tags = doc.tags;
+    if (doc.content !== undefined)
+        entry.content = doc.content;
+    if (doc.ownsPresentation !== undefined)
+        entry.ownsPresentation = doc.ownsPresentation;
+    if (doc.updated !== undefined)
+        entry.updated = doc.updated;
+    if (doc.links !== undefined)
+        entry.links = doc.links;
+    if (doc.adapterId === "okf") {
+        if (doc.documentJson !== undefined)
+            entry.documentJson = doc.documentJson;
+        // OKF v0.2 trust/provenance/lifecycle family (okf-support.md v0.2 note).
+        // These are first-class IndexDocument members the okf adapter sets
+        // directly (never routed through documentJson), so — unlike the akm
+        // adapter's DOCUMENT_JSON_CARRIED_FIELDS reconstruction below — they need
+        // an explicit carry here too, or persistence silently drops them even
+        // though `recognize()` itself returns them correctly.
+        if (doc.provenance !== undefined)
+            entry.provenance = doc.provenance;
+        if (doc.lifecycleStatus !== undefined)
+            entry.lifecycleStatus = doc.lifecycleStatus;
+        if (doc.staleAfter !== undefined)
+            entry.staleAfter = doc.staleAfter;
+        if (doc.okfVersion !== undefined)
+            entry.okfVersion = doc.okfVersion;
+        return entry;
+    }
+    if (doc.aliases !== undefined)
+        entry.aliases = doc.aliases;
+    if (doc.searchHints !== undefined)
+        entry.searchHints = doc.searchHints;
+    if (doc.hints !== undefined)
+        entry.hints = doc.hints;
+    if (doc.quality !== undefined)
+        entry.quality = doc.quality;
+    if (doc.confidence !== undefined)
+        entry.confidence = doc.confidence;
+    if (doc.beliefState !== undefined)
+        entry.beliefState = doc.beliefState;
+    if (doc.currentBeliefRefs !== undefined)
+        entry.currentBeliefRefs = doc.currentBeliefRefs;
+    if (doc.scope !== undefined)
+        entry.scope = doc.scope;
+    if (doc.captureMode !== undefined)
+        entry.captureMode = doc.captureMode;
+    if (doc.lessonStrength !== undefined)
+        entry.lessonStrength = doc.lessonStrength;
+    if (doc.derivedFrom !== undefined)
+        entry.derivedFrom = doc.derivedFrom;
+    if (doc.pinned !== undefined)
+        entry.pinned = doc.pinned;
+    // ── documentJson-carried extras (DOCUMENT_JSON_CARRIED_FIELDS) ──
+    // The `renderer` key on documentJson is adapter-internal (WI-C presentation),
+    // NOT a IndexDocument field — deliberately not restored.
+    assignStringList(entry, "examples", dj.examples);
+    assignStringList(entry, "usage", dj.usage);
+    if (isIntent(dj.intent))
+        entry.intent = dj.intent;
+    assignStringList(entry, "xrefs", dj.xrefs);
+    assignString(entry, "pageKind", dj.pageKind);
+    assignString(entry, "whenToUse", dj.whenToUse);
+    if (dj.toc !== undefined)
+        entry.toc = dj.toc;
+    if (dj.parameters !== undefined)
+        entry.parameters = dj.parameters;
+    assignString(entry, "bodyOpening", dj.bodyOpening);
+    if (dj.source !== undefined)
+        entry.source = dj.source;
+    assignString(entry, "category", dj.category);
+    assignStringList(entry, "supersededBy", dj.supersededBy);
+    assignStringList(entry, "contradictedBy", dj.contradictedBy);
+    assignString(entry, "run", dj.run);
+    assignString(entry, "setup", dj.setup);
+    assignString(entry, "cwd", dj.cwd);
+    if (dj.wikiRole !== undefined)
+        entry.wikiRole = dj.wikiRole;
+    assignStringList(entry, "sources", dj.sources);
+    if (typeof dj.generation === "number")
+        entry.generation = dj.generation;
+    assignStringList(entry, "evidenceSources", dj.evidenceSources);
+    // D2 (#730): OKF v0.2 provenance promoteProposal stamps onto AKM-native
+    // writes, carried via DOCUMENT_JSON_CARRIED_FIELDS (akm-adapter.ts) —
+    // unpacked back to a first-class member here exactly like `sources`/
+    // `generation`/`evidenceSources` above, so it round-trips to the top level
+    // of the persisted entry rather than staying nested under `documentJson`.
+    if (dj.provenance !== undefined)
+        entry.provenance = dj.provenance;
+    return entry;
+}
+function assignStringList(entry, key, value) {
+    if (Array.isArray(value))
+        entry[key] = value;
+}
+function assignString(entry, key, value) {
+    if (typeof value === "string")
+        entry[key] = value;
+}
+function isIntent(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/dist/indexer/scan/drain-dir.js

@@ -0,0 +1,134 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Per-directory document drain — akm 0.9.0 Chunk 5, milestone F4a M-core-2 (the
+ * engine swap). Replaces the live indexer's per-dir flat-walk matcher-pass
+ * `IndexDocument` stream with the `akm` adapter's `recognize` `IndexDocument`
+ * stream, reconstructing the durable `IndexDocument` via {@link
+ * indexDocumentToStashEntry} (proven lossless by the shadow-parity gate).
+ *
+ * Two behaviors the adapter fold does NOT carry, restored here at the drain
+ * layer (spec §14.2 "drain the full document stream"):
+ *
+ *  - **Broken-workflow drop (item 3).** The live path dropped a broken workflow
+ *    via the renderer contributor's throw → metadata-pass skip-with-warning; the
+ *    `akm` adapter's synchronous `foldRecognizedMetadata` SWALLOWS the parse
+ *    error, so a broken workflow would otherwise silently index. We re-run
+ *    `parseWorkflow` on drained workflow docs and DROP
+ *    the entry with the same `Skipped workflow …` warning
+ *    ({@link buildMetadataSkipWarning}), so the workflow-skip summary counts it.
+ *  - **Workflow-document side-table (workflow-md only).** The valid parsed
+ *    `WorkflowDocument` is handed to the persist layer through the same
+ *    `document-cache` side channel the live renderer contributor used, keyed by
+ *    the reconstructed entry — so `takeWorkflowDocument(entry)` in the persist
+ *    loop writes the `workflow_documents` row exactly as before.
+ *
+ * `doc.hash` (= sha256 of the file content) is surfaced per recognized file so
+ * the persist layer can populate the `content_hash` column (item 2). It is keyed
+ * by the file's absolute path rather than by the entry object.
+ *
+ * Pure of DB/global state beyond the workflow-document side channel; a new leaf
+ * (nothing imports it back), so it joins no import cycle.
+ */
+import { akmAdapter } from "../../core/adapter/adapters/akm-adapter.js";
+import { parseWorkflow } from "../../workflows/parser.js";
+import { cacheWorkflowDocument } from "../../workflows/runtime/document-cache.js";
+import { buildMetadataSkipWarning } from "../passes/metadata.js";
+import { buildFileContext } from "../walk/file-context.js";
+import { indexDocumentToStashEntry } from "./doc-to-entry.js";
+/** The markdown-workflow renderer name the `akm` adapter carries on `documentJson.renderer`. */
+const WORKFLOW_MD_RENDERER = "workflow-md";
+/**
+ * Drain one directory's recognized documents into durable entries.
+ *
+ * `fileContexts` are the dir's walked files (the drain no longer pre-filters —
+ * adapter-owned filtering, owner ruling 2026-07-21). `adapter.recognize` returns
+ * `null` for a file it abstains on (no matcher claims it, an OKF reserved file,
+ * or an AKM sensitive/infra file) — silently skipped, the same contract the
+ * legacy flat-walk pass's "no matcher claims the file" case had.
+ */
+export function drainDirDocuments(adapter, component, fileContexts) {
+    const entries = [];
+    const warnings = [];
+    const hashByFile = new Map();
+    const conceptIdByFile = new Map();
+    for (const file of fileContexts) {
+        const doc = adapter.recognize(component, file);
+        if (doc === null)
+            continue;
+        if (!doc.conceptId) {
+            warnings.push(`Skipped ${file.absPath}: adapter "${adapter.id}" returned no conceptId.`);
+            continue;
+        }
+        const entry = indexDocumentToStashEntry(doc);
+        // Workflow docs: drop-with-warning if broken; otherwise cache the parsed
+        // markdown document for the persist-time `workflow_documents` write.
+        const dropWarning = handleWorkflowDoc(doc, entry, file);
+        if (dropWarning !== null) {
+            warnings.push(dropWarning);
+            continue;
+        }
+        if (doc.hash !== undefined)
+            hashByFile.set(file.absPath, doc.hash);
+        conceptIdByFile.set(file.absPath, doc.conceptId);
+        entries.push(entry);
+    }
+    return { entries, warnings, hashByFile, conceptIdByFile };
+}
+/**
+ * `(stashRoot, files) → StashFile` drop-in for the deleted flat-walk matcher
+ * pass (F4a M-core-3): builds a FileContext per file and drains them through the
+ * `akm` adapter's `recognize`. The recognize engine is the proven-equal
+ * replacement for the old matcher-pass metadata assembly (shadow-parity gate), so
+ * callers that only need the recognized entries (`manifest`'s no-index fallback,
+ * the `registry` static-index builder, and the metadata unit tests) get identical
+ * entries — plus the D-R6 reserved-file exclusion and the AKM sensitive/infra
+ * abstention the adapter now enforces itself. Provenance is not persisted by these
+ * callers, so the synthetic component id is immaterial.
+ */
+export function recognizeStashEntries(stashRoot, files) {
+    const component = { id: stashRoot, adapter: "akm", root: stashRoot, writable: false };
+    // No pre-filter: the `akm` adapter's `recognize` claims/abstains per file
+    // (owner ruling 2026-07-21 — adapter-owned filtering).
+    const contexts = files.map((file) => buildFileContext(stashRoot, file));
+    const drained = drainDirDocuments(akmAdapter, component, contexts);
+    return drained.warnings.length > 0
+        ? { entries: drained.entries, warnings: drained.warnings }
+        : { entries: drained.entries };
+}
+/**
+ * If `doc` is a workflow, re-parse it: return a `Skipped workflow …` drop
+ * warning when it is broken, or cache the parsed markdown `WorkflowDocument`
+ * (workflow-md only) and return `null` when valid. Non-workflow docs return
+ * `null` immediately.
+ */
+function handleWorkflowDoc(doc, entry, file) {
+    if (docRenderer(doc) !== WORKFLOW_MD_RENDERER)
+        return null;
+    const result = parseWorkflow(file.content(), { path: file.relPath });
+    if (!result.ok)
+        return workflowDropWarning(file, result.errors);
+    cacheWorkflowDocument(entry, result.document);
+    return null;
+}
+/** The winning renderer name the `akm` adapter carries on `documentJson.renderer`, or `undefined`. */
+function docRenderer(doc) {
+    const dj = doc.documentJson;
+    if (dj !== null && typeof dj === "object" && "renderer" in dj) {
+        const renderer = dj.renderer;
+        return typeof renderer === "string" ? renderer : undefined;
+    }
+    return undefined;
+}
+/**
+ * Build the `Skipped workflow <path>:\n…` warning byte-for-byte the way the live
+ * pipeline did: the workflow parser's `path:line — message` summary wrapped in
+ * the `Workflow has errors:` prefix (the string `loadDocument`/`loadProgram`
+ * threw), then {@link buildMetadataSkipWarning}'s workflow branch. `startsWith
+ * "Skipped workflow "` so `isWorkflowSkipWarning` counts it for the summary.
+ */
+function workflowDropWarning(file, errors) {
+    const summary = errors.map((e) => `${file.relPath}:${e.line} — ${e.message}`).join("\n");
+    return buildMetadataSkipWarning(file.absPath, "workflow", `Workflow has errors:\n${summary}`);
+}