akm-cli 0.9.0-rc.0 → 0.9.0-rc.13

package/dist/indexer/passes/metadata.js

@@ -3,15 +3,12 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import fs from "node:fs";
 import path from "node:path";
-import { deriveCanonicalAssetName, deriveCanonicalAssetNameFromStashRoot, isRelevantAssetFile, } from "../../core/asset/asset-spec.js";
+import { parseBundleRef } from "../../core/asset/asset-ref.js";
 import { parseFrontmatter } from "../../core/asset/frontmatter.js";
-import { asNonEmptyString, isAssetType, writeFileAtomic } from "../../core/common.js";
+import { asNonEmptyString } from "../../core/common.js";
+import { loadUserConfig } from "../../core/config/config.js";
 import { isVerbose, warn } from "../../core/warn.js";
-import { buildFileContext, buildRenderContext, getRenderer, runMatchers } from "../walk/file-context.js";
-import { applyMetadataContributors } from "./metadata-contributors.js";
 export const SCOPE_KEYS = ["user", "agent", "run", "channel"];
-// ── Load / Write ────────────────────────────────────────────────────────────
-const STASH_FILENAME = ".stash.json";
 // ── Quality semantics (v1 spec §4.2) ────────────────────────────────────────
 /**
  * Well-known quality values. `generated`, `curated`, and `enriched` are included in
@@ -50,49 +47,12 @@ export function _resetUnknownQualityWarnings() {
 export function isProposedQuality(quality) {
     return quality === "proposed";
 }
-export function stashFilePath(dirPath) {
-    return path.join(dirPath, STASH_FILENAME);
-}
-export function loadStashFile(dirPath, options) {
-    const filePath = stashFilePath(dirPath);
-    if (!fs.existsSync(filePath))
-        return null;
-    try {
-        const raw = JSON.parse(fs.readFileSync(filePath, "utf8"));
-        if (!raw || !Array.isArray(raw.entries))
-            return null;
-        const entries = [];
-        for (const e of raw.entries) {
-            const validated = validateStashEntry(e);
-            if (validated) {
-                if (options?.requireFilename && !validated.filename)
-                    continue;
-                entries.push(validated);
-            }
-            else {
-                const name = typeof e === "object" && e !== null && typeof e.name === "string"
-                    ? e.name
-                    : "(unknown)";
-                warn(`Warning: Skipping invalid entry "${name}" in ${filePath}`);
-            }
-        }
-        return entries.length > 0 ? { entries } : null;
-    }
-    catch {
-        return null;
-    }
-}
-export function writeStashFile(dirPath, stash) {
-    const filePath = stashFilePath(dirPath);
-    writeFileAtomic(filePath, `${JSON.stringify(stash, null, 2)}\n`);
-}
 /**
- * Validate and normalize a raw object into a `StashEntry`.
+ * Validate and normalize a raw object into a `IndexDocument`.
  *
- * **Ordering dependency:** Uses `isAssetType()` to check `entry.type`, which
- * only recognizes custom types registered via `registerAssetType()`. If this
- * function is called before custom types are registered, those entries will be
- * rejected as invalid.
+ * Open type token: `entry.type` accepts any non-empty string. Type ownership
+ * and capability decisions belong to the adapter; this format-neutral
+ * projection must not reject a value merely because AKM does not own it.
  */
 export function validateStashEntry(entry) {
     if (typeof entry !== "object" || entry === null)
@@ -100,7 +60,7 @@ export function validateStashEntry(entry) {
     const e = entry;
     if (typeof e.name !== "string" || !e.name)
         return null;
-    if (typeof e.type !== "string" || !isAssetType(e.type))
+    if (typeof e.type !== "string" || !e.type)
         return null;
     const result = {
         name: e.name,
@@ -182,6 +142,11 @@ export function validateStashEntry(entry) {
     const sources = normalizeNonEmptyStringList(e.sources);
     if (sources)
         result.sources = sources;
+    // SPEC-6: `category` must survive the projection. Non-string values are
+    // dropped, not coerced.
+    if (typeof e.category === "string" && e.category.trim().length > 0) {
+        result.category = e.category.trim();
+    }
     if (typeof e.beliefState === "string" && e.beliefState.trim().length > 0) {
         result.beliefState = e.beliefState.trim();
     }
@@ -197,9 +162,6 @@ export function validateStashEntry(entry) {
     if (typeof e.generation === "number" && Number.isFinite(e.generation) && e.generation > 0) {
         result.generation = Math.floor(e.generation);
     }
-    const sourceRefs = normalizeNonEmptyStringList(e.sourceRefs);
-    if (sourceRefs)
-        result.sourceRefs = sourceRefs;
     const currentBeliefRefs = normalizeNonEmptyStringList(e.currentBeliefRefs);
     if (currentBeliefRefs)
         result.currentBeliefRefs = currentBeliefRefs;
@@ -218,6 +180,11 @@ export function validateStashEntry(entry) {
     if (typeof e.derivedFrom === "string" && e.derivedFrom.trim().length > 0) {
         result.derivedFrom = e.derivedFrom.trim();
     }
+    // SPEC-8: `bodyOpening` must survive the projection. The extractor already
+    // trimmed and capped it at capture time.
+    if (typeof e.bodyOpening === "string" && e.bodyOpening.trim().length > 0) {
+        result.bodyOpening = e.bodyOpening;
+    }
     if (typeof e.scope === "object" && e.scope !== null && !Array.isArray(e.scope)) {
         const scope = normalizeScopeObject(e.scope);
         if (scope)
@@ -308,6 +275,45 @@ function normalizeIntent(value) {
 function normalizeStringListOrUndefined(value) {
     return normalizeNonEmptyStringList(value);
 }
+/**
+ * Normalize a current derived-memory parent ref to its `memories/<name>`
+ * conceptId for the `derived_from` column. Returns `undefined` for a non-memory
+ * or bare value so the caller can inspect the current `derivedFrom` key.
+ */
+function normalizeMemoryBackref(value) {
+    if (!value)
+        return undefined;
+    try {
+        const parsed = parseBundleRef(value);
+        return parsed.fragment === undefined && parsed.conceptId.startsWith("memories/") ? parsed.conceptId : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Resolve a derived memory's parent conceptId from its current `source:` backref
+ * or, failing that, its bare `derivedFrom: <name>` frontmatter key.
+ */
+function derivedFromConceptId(source, derivedFrom) {
+    const fromSource = normalizeMemoryBackref(source);
+    if (fromSource)
+        return fromSource;
+    const rawDerivedFrom = derivedFrom?.trim();
+    if (!rawDerivedFrom)
+        return undefined;
+    const qualified = normalizeMemoryBackref(rawDerivedFrom);
+    if (qualified)
+        return qualified;
+    if (rawDerivedFrom.includes(":") || rawDerivedFrom.includes("//"))
+        return undefined;
+    try {
+        return parseBundleRef(`memories/${rawDerivedFrom}`).conceptId;
+    }
+    catch {
+        return undefined;
+    }
+}
 export function applyCuratedFrontmatter(entry, fmData) {
     const description = asNonEmptyString(fmData.description);
     if (description) {
@@ -342,6 +348,12 @@ export function applyCuratedFrontmatter(entry, fmData) {
     const quality = asNonEmptyString(fmData.quality);
     if (quality)
         entry.quality = normalizeQuality(quality);
+    // SPEC-6 capture step: the `category:` frontmatter key (e.g. `convention`,
+    // `meta` on facts) must land on the indexed entry so category-keyed
+    // policies can see it. Trimmed; blank/non-string values are ignored.
+    const category = asNonEmptyString(fmData.category);
+    if (category)
+        entry.category = category;
     const beliefState = asNonEmptyString(fmData.beliefState);
     if (beliefState)
         entry.beliefState = beliefState;
@@ -351,17 +363,12 @@ export function applyCuratedFrontmatter(entry, fmData) {
     const contradictedBy = normalizeStringListOrUndefined(fmData.contradictedBy);
     if (contradictedBy)
         entry.contradictedBy = contradictedBy;
-    // R5 — consolidation provenance. `generation` (merge depth counter) and
-    // `source_refs` (merge/distill provenance pointers) are written by the
-    // improve pipeline; captured into the index so the collapse detector can
-    // count over-generation assets and follow merges without filesystem reads.
+    // R5 — consolidation generation depth is captured so the collapse detector
+    // can count over-generation assets without filesystem reads.
     const generation = fmData.generation;
     if (typeof generation === "number" && Number.isFinite(generation) && generation > 0) {
         entry.generation = Math.floor(generation);
     }
-    const sourceRefs = normalizeStringListOrUndefined(fmData.source_refs);
-    if (sourceRefs)
-        entry.sourceRefs = sourceRefs;
     const currentBeliefRefs = normalizeStringListOrUndefined(fmData.currentBeliefRefs);
     if (currentBeliefRefs)
         entry.currentBeliefRefs = currentBeliefRefs;
@@ -384,27 +391,20 @@ export function applyCuratedFrontmatter(entry, fmData) {
     if (evidenceSources)
         entry.evidenceSources = evidenceSources;
     // Phase 5A / Advantage D5: capture parent ref for derived memories.
-    // Memory-inference writes `source: "memory:<parent>"` and `inferred: true`
-    // (and a derived child name suffix `.derived`). We mirror that source ref
-    // into `entry.derivedFrom` so the indexer can populate the dedicated
-    // `derived_from` column. Non-derived entries leave this field unset.
+    // Memory-inference writes `source: "memories/<parent>"` and `inferred: true`
+    // (and a derived child name suffix `.derived`). We mirror that source ref into
+    // `entry.derivedFrom` so the indexer can populate the dedicated `derived_from`
+    // column. Group-C item 2: the column is stored in the 0.9.0 `memories/<name>`
+    // conceptId grammar — moving in lockstep with the `getDerivedForParent` lookup
+    // key (search-hit-enrichers) so producer + consumer speak one grammar.
+    // Non-derived entries leave this field unset.
     if (entry.type === "memory") {
         const isDerivedByName = entry.name.toLowerCase().endsWith(".derived");
         const isDerivedByFm = fmData.inferred === true;
         if (isDerivedByName || isDerivedByFm) {
-            const sourceStr = asNonEmptyString(fmData.source);
-            if (sourceStr?.includes(":")) {
-                entry.derivedFrom = sourceStr;
-            }
-            else {
-                // Fallback: some legacy renderings store only `derivedFrom: <name>`
-                // (a bare parent name). Promote it to a `memory:` ref so the lookup
-                // column stays consistent.
-                const derivedFromName = asNonEmptyString(fmData.derivedFrom);
-                if (derivedFromName) {
-                    entry.derivedFrom = derivedFromName.includes(":") ? derivedFromName : `memory:${derivedFromName}`;
-                }
-            }
+            const parent = derivedFromConceptId(asNonEmptyString(fmData.source), asNonEmptyString(fmData.derivedFrom));
+            if (parent)
+                entry.derivedFrom = parent;
         }
     }
     const intent = normalizeIntent(fmData.intent);
@@ -484,53 +484,84 @@ export function applyWikiFrontmatter(entry, fmData) {
             entry.sources = filtered;
     }
 }
-const WIKI_INFRA_FILES = new Set(["schema.md", "index.md", "log.md"]);
 /**
- * Apply wiki-specific index exclusions while leaving all other stash files
- * untouched.
+ * Parse one `verified:` family value into the `IndexDocument.provenance` shape.
+ * Accepts OKF v0.2's list form and its documented single-mapping shorthand
+ * ("consumers MUST treat a bare mapping as a one-element list", SPEC §5.2).
+ * Tolerant: an entry without a usable `by` is dropped individually.
+ */
+function parseVerifiedFamily(value) {
+    const entries = Array.isArray(value) ? value : [value];
+    return entries
+        .filter((v) => v !== null && typeof v === "object" && !Array.isArray(v))
+        .map((v) => {
+        const by = asNonEmptyString(v.by);
+        if (!by)
+            return undefined;
+        const at = asNonEmptyString(v.at);
+        return at ? { by, at } : { by };
+    })
+        .filter((v) => v !== undefined);
+}
+/**
+ * Extract the OKF v0.2 provenance families that `promoteProposal` stamps onto
+ * accepted AKM-native proposals (D2, #730), and apply them to the entry.
+ *
+ * The on-disk shape is deliberately **hybrid** (owner decision, #730 review):
+ * `generated:` and `verified:` are stamped BARE at the top level, exactly as
+ * OKF v0.2 spells them, because neither has any pre-existing AKM consumer — so
+ * a third-party OKF v0.2 reader treating an AKM stash as an OKF bundle sees
+ * spec-conformant trust metadata, which is what `okf-support.md`'s
+ * "AKM Markdown is an OKF-compatible superset" positioning promises. Only
+ * `sources` stays namespaced under `provenance:`, because a bare top-level
+ * `sources:` genuinely collides with {@link applyWikiFrontmatter}'s
+ * pre-existing citation-**string** convention (which silently drops
+ * non-strings).
  *
- * - In a normal stash, excludes wiki-root `schema.md`, `index.md`, `log.md`.
- * - In a wiki-root stash source (`wikiName`), excludes those same root-level
- *   infrastructure files.
+ * The nested `provenance.generatedBy` / `.generatedAt` / `.verified` spellings
+ * are still read as a fallback so assets stamped by an earlier build of this
+ * branch keep resolving. Tolerant throughout: any malformed sub-field is
+ * dropped individually rather than rejecting the whole block.
  */
-export function shouldIndexStashFile(stashRoot, file, options) {
-    const relPath = path.relative(stashRoot, file);
-    if (!relPath || relPath.startsWith("..") || path.isAbsolute(relPath))
-        return true;
-    const segments = relPath.split(/[\\/]+/).filter(Boolean);
-    if (segments.length === 0)
-        return true;
-    // Skip env .env files that have a sibling .sensitive marker file.
-    if (segments[0] === "env" && (file.endsWith(".env") || path.basename(file) === ".env")) {
-        const markerPath = file.replace(/\.env$/, ".sensitive");
-        if (fs.existsSync(markerPath))
-            return false;
-    }
-    // The legacy `vaults/` directory (frozen copy left by the 0.8 migration) is
-    // never indexed — the `vault` asset type was removed in 0.9.0.
-    if (segments[0] === "vaults") {
-        return false;
-    }
-    // Skip secret files that are themselves a `.sensitive` marker, or that have a
-    // sibling `<name>.sensitive` marker. Secrets are otherwise indexed by name
-    // only (their bytes are never read — see buildEntryFromFile guards).
-    if (segments[0] === "secrets") {
-        if (file.endsWith(".sensitive") || file.endsWith(".lock"))
-            return false;
-        if (fs.existsSync(`${file}.sensitive`))
-            return false;
-    }
-    if (options?.treatStashRootAsWikiRoot) {
-        return !(segments.length === 1 && WIKI_INFRA_FILES.has(segments[0]));
+export function applyProvenanceFrontmatter(entry, fmData) {
+    const provenance = fmData.provenance;
+    const nested = provenance !== null && typeof provenance === "object" && !Array.isArray(provenance)
+        ? provenance
+        : {};
+    const result = {};
+    // Bare `generated: {by, at}` (OKF v0.2 §5.3) wins; nested is the fallback.
+    const generatedMapping = fmData.generated !== null && typeof fmData.generated === "object" && !Array.isArray(fmData.generated)
+        ? fmData.generated
+        : undefined;
+    const generatedBy = asNonEmptyString(generatedMapping?.by) ?? asNonEmptyString(nested.generatedBy);
+    if (generatedBy)
+        result.generatedBy = generatedBy;
+    const generatedAt = asNonEmptyString(generatedMapping?.at) ?? asNonEmptyString(nested.generatedAt);
+    if (generatedAt)
+        result.generatedAt = generatedAt;
+    // Bare `verified:` (list or single mapping) wins; nested is the fallback.
+    const verified = fmData.verified !== undefined ? parseVerifiedFamily(fmData.verified) : parseVerifiedFamily(nested.verified);
+    if (verified.length > 0)
+        result.verified = verified;
+    // `sources` is namespaced-only — see the collision note above.
+    if (Array.isArray(nested.sources)) {
+        const sources = nested.sources
+            .filter((s) => s !== null && typeof s === "object" && !Array.isArray(s))
+            .map((s) => {
+            const resource = asNonEmptyString(s.resource);
+            return resource ? { resource } : undefined;
+        })
+            .filter((s) => s !== undefined);
+        if (sources.length > 0)
+            result.sources = sources;
     }
-    const wikisIdx = segments.indexOf("wikis");
-    if (wikisIdx < 0 || wikisIdx + 1 >= segments.length)
-        return true;
-    const wikiRelativeSegments = segments.slice(wikisIdx + 2);
-    if (wikiRelativeSegments.length === 0)
-        return true;
-    return !(wikiRelativeSegments.length === 1 && WIKI_INFRA_FILES.has(wikiRelativeSegments[0]));
+    if (Object.keys(result).length > 0)
+        entry.provenance = result;
 }
+// AKM-stash indexing policy (env/vaults/secrets sensitive-marker + wiki-infra
+// exclusions) moved to the `akm` adapter's `recognize` as path/stat-based
+// abstention (owner ruling 2026-07-21 — adapter-owned filtering). See
+// `akmStashAbstains` in `src/core/adapter/adapters/akm-adapter.ts`.
 /**
  * Extract `@param` JSDoc tags from a script file's leading comment block.
  *
@@ -797,7 +828,6 @@ function mergeAliases(existing, generated) {
  *
  * This predicate is used by `enhanceDirsWithLlm` to skip the LLM call for
  * entries that were previously enriched and already carry all three fields.
- * Pass `reEnrich = true` in the caller to bypass this check.
  */
 export function isEnrichmentComplete(entry) {
     const hasDescription = typeof entry.description === "string" && entry.description.trim().length > 0;
@@ -805,39 +835,206 @@ export function isEnrichmentComplete(entry) {
     const hasSearchHints = Array.isArray(entry.searchHints) && entry.searchHints.length > 0;
     return hasDescription && hasTags && hasSearchHints;
 }
-// ── Metadata Generation ─────────────────────────────────────────────────────
+// ── Body-opening extraction (stash-conventions SPEC-8) ──────────────────────
 /**
- * Shared pipeline (steps 2-6) for building a single StashEntry from a file.
- *
- * Both `generateMetadata` and `generateMetadataFlat` perform identical work
- * once the initial `entry` object has been seeded with type and canonical name.
- * This helper encapsulates that shared pipeline so the two callers only differ
- * in how they determine the asset type and canonical name (step 1):
- *
- *  - `generateMetadata`     — explicit `assetType` arg + `deriveCanonicalAssetName`
- *  - `generateMetadataFlat` — type from `runMatchers()` + `deriveCanonicalAssetNameFromStashRoot`
+ * Maximum length of a captured self-situating body opening. Bounds index-size
+ * growth and keeps a single verbose opening from dominating the low-weight
+ * `content` FTS column.
+ */
+export const BODY_OPENING_MAX_CHARS = 280;
+/**
+ * Minimum characters retained when the cap truncates at a word boundary. A
+ * boundary cut that would retain less than this falls back to a hard cut, so
+ * one pathological long token cannot gut the capture.
+ */
+const BODY_OPENING_MIN_RETAINED_CHARS = 250;
+/**
+ * True when `index.indexBodyOpening` is enabled in the user config.
  *
- * @param file        Absolute path to the file being processed.
- * @param assetType   Resolved asset type string (already validated by caller).
- * @param canonicalName Resolved canonical name (already computed by caller).
- * @param dirPath     Directory containing the file (used for tag fallback).
- * @param pkgMeta     Pre-loaded package.json metadata for this directory (may be null/undefined).
- * @param stashRoot   Stash root used for renderer search hints context.
- * @param ctx         FileContext for the file (may be pre-built by the caller).
- * @param match       Pre-resolved MatchResult when available (from `generateMetadataFlat`).
- * @returns The populated entry, or `{ skip: true, warning: string }` when the
- *          renderer throws and the file should be dropped.
+ * The gate is the GLOBAL user config, read directly by the metadata pass so
+ * every indexing entry point (stash walk, flat walk, write-path indexing)
+ * honors the flag without parameter plumbing. Fail-open: an unreadable or
+ * invalid config must never break indexing (CLI entry points surface config
+ * errors loudly on their own), so any load failure reads as "off".
+ */
+function isBodyOpeningIndexingEnabled() {
+    try {
+        return loadUserConfig().index?.indexBodyOpening === true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Locate a leading nested frontmatter block in a body: up to three blank
+ * lines, then a `---` line, closed by a later `---` line. Mirrors the
+ * base-linter's `parseInnerFrontmatterBlock` recognition — when `akm
+ * remember` wraps a session-capture hook's file in its own frontmatter, the
+ * hook's `---\nakm_memory_kind: …\n---` block survives at the top of the
+ * body. Returns the open/close line indexes, or `null` when no block opens.
+ * Location only — callers apply their own interior checks (marker scan in
+ * {@link hasSessionMemoryMarker}, shape test in {@link isFrontmatterShaped}).
+ */
+function findInnerFrontmatterBlock(lines) {
+    let i = 0;
+    while (i < lines.length && i < 3 && lines[i].trim() === "")
+        i += 1;
+    if (lines[i] !== "---")
+        return null;
+    for (let j = i + 1; j < lines.length; j += 1) {
+        if (lines[j] === "---")
+            return { open: i, close: j };
+    }
+    return null;
+}
+/**
+ * True when the document carries the session-capture `akm_memory_kind`
+ * marker — in the outer frontmatter data OR in a nested inner block at the
+ * top of the body (both producer layouts exist; see base-linter's
+ * `extractFrontmatterRefs`). Session bodies are raw transcripts, never a
+ * self-situating opening.
  */
-async function buildEntryFromFile(file, assetType, canonicalName, dirPath, pkgMeta, stashRoot, ctx, match) {
+function hasSessionMemoryMarker(fmData, body) {
+    if (typeof fmData.akm_memory_kind === "string")
+        return true;
+    const lines = body.split(/\r?\n/);
+    const block = findInnerFrontmatterBlock(lines);
+    if (!block)
+        return false;
+    for (let i = block.open + 1; i < block.close; i += 1) {
+        if (/^akm_memory_kind:\s*\S/.test(lines[i]))
+            return true;
+    }
+    return false;
+}
+/**
+ * True when the interior of a candidate inner block (located by
+ * {@link findInnerFrontmatterBlock}) actually reads as YAML frontmatter:
+ * every line is blank, indented (a continuation or nested value), or shaped
+ * like a top-level `key:` mapping entry. Ordinary prose bracketed by two
+ * thematic-break `---` lines fails this test, so a decorative opening
+ * callout is treated as the paragraph it is instead of being discarded
+ * (review finding on SPEC-8 — the block finder alone accepts ANY content up
+ * to an arbitrarily distant closing `---`).
+ */
+function isFrontmatterShaped(lines, block) {
+    for (let i = block.open + 1; i < block.close; i += 1) {
+        const line = lines[i];
+        if (line.trim() === "")
+            continue;
+        if (/^\s/.test(line))
+            continue; // indented continuation / nested value
+        if (/^[A-Za-z0-9_.-]+:(\s|$)/.test(line))
+            continue; // top-level key
+        return false;
+    }
+    return true;
+}
+/**
+ * Extract the first prose paragraph of a markdown body (frontmatter already
+ * stripped by the caller): skip blank lines, ATX headings, setext `=`
+ * underlines (discarding the heading text above them), thematic breaks,
+ * fenced code blocks (``` or ~~~, including their contents), and a leading
+ * nested frontmatter block — skipped only when its interior is actually
+ * frontmatter-shaped, so prose wrapped in decorative `---` lines is still
+ * captured; then collect consecutive non-blank lines until the paragraph
+ * ends. Deliberate asymmetry: a `---` row after captured prose ENDS the
+ * paragraph and keeps it (favoring the callout/thematic-break reading over
+ * CommonMark's setext-H2), while a `=+` row can only be a setext underline
+ * and so discards the pending lines as heading text. The result is capped at
+ * {@link BODY_OPENING_MAX_CHARS} chars — truncated at the last word boundary
+ * that still retains a substantial prefix, with a trailing ellipsis. Returns
+ * `undefined` when the body has no prose (frontmatter-only files,
+ * headings/fences-only bodies).
+ */
+export function extractBodyOpening(body) {
+    const lines = body.split(/\r?\n/);
+    const innerBlock = findInnerFrontmatterBlock(lines);
+    const start = innerBlock && isFrontmatterShaped(lines, innerBlock) ? innerBlock.close + 1 : 0;
+    const paragraph = [];
+    let inFence = false;
+    let fenceChar = "";
+    let inHtmlComment = false;
+    for (let i = start; i < lines.length; i += 1) {
+        const trimmed = lines[i].trim();
+        const fenceMatch = trimmed.match(/^(`{3,}|~{3,})/);
+        if (inFence) {
+            // Fence interiors are never prose (and may be secrets-adjacent command
+            // text); skip until the matching closing marker.
+            if (fenceMatch && fenceMatch[1].charAt(0) === fenceChar)
+                inFence = false;
+            continue;
+        }
+        if (inHtmlComment) {
+            // Comment interiors are machinery (the skeleton convention facts open
+            // with a <!-- SOFT guidance --> block), never orientation prose.
+            if (trimmed.includes("-->"))
+                inHtmlComment = false;
+            continue;
+        }
+        if (fenceMatch) {
+            if (paragraph.length > 0)
+                break; // a fence ends an open paragraph
+            inFence = true;
+            fenceChar = fenceMatch[1].charAt(0);
+            continue;
+        }
+        if (trimmed.startsWith("<!--")) {
+            if (paragraph.length > 0)
+                break; // a comment ends an open paragraph
+            if (!trimmed.includes("-->"))
+                inHtmlComment = true;
+            continue;
+        }
+        if (/^=+$/.test(trimmed)) {
+            // A row of `=` is a setext H1 underline: the pending lines are heading
+            // text, not prose — discard them and keep searching. (A bare `=` row
+            // with nothing pending is skipped like any other non-prose divider.)
+            paragraph.length = 0;
+            continue;
+        }
+        const isBlank = trimmed === "";
+        const isHeading = /^#{1,6}(\s|$)/.test(trimmed);
+        const isThematicBreak = /^(-{3,}|\*{3,}|_{3,})$/.test(trimmed);
+        if (isBlank || isHeading || isThematicBreak) {
+            if (paragraph.length > 0)
+                break; // paragraph complete
+            continue; // still searching for the first prose line
+        }
+        paragraph.push(trimmed);
+    }
+    const text = paragraph.join("\n");
+    if (!text)
+        return undefined;
+    if (text.length <= BODY_OPENING_MAX_CHARS)
+        return text;
+    // Cap: prefer a word-boundary cut (never mid-token), but only when it keeps
+    // a substantial prefix; append a one-char ellipsis inside the budget.
+    const slice = text.slice(0, BODY_OPENING_MAX_CHARS - 1);
+    const lastBoundary = Math.max(slice.lastIndexOf(" "), slice.lastIndexOf("\n"), slice.lastIndexOf("\t"));
+    let cut = lastBoundary >= BODY_OPENING_MIN_RETAINED_CHARS ? slice.slice(0, lastBoundary) : slice;
+    // slice() counts UTF-16 code units, so the no-boundary fallback can end on
+    // the high half of a surrogate pair — a lone surrogate that corrupts the
+    // FTS/embedding text. Drop it rather than emit invalid UTF-16.
+    const lastCode = cut.charCodeAt(cut.length - 1);
+    if (lastCode >= 0xd800 && lastCode <= 0xdbff)
+        cut = cut.slice(0, -1);
+    return `${cut.trimEnd()}…`;
+}
+// ── Metadata Generation ─────────────────────────────────────────────────────
+/**
+ * Priorities 1-2 of the metadata pipeline — package.json (P1), `.md`
+ * frontmatter (P2), and script `@param`/comment metadata (P2b) — everything
+ * that runs BEFORE the renderer-contributor step (P3). Extracted (Chunk 5 M-b)
+ * so the `akm` adapter's synchronous `recognize` shares this exact assembly
+ * with the live index-drain path; the two differ ONLY in how they obtain the
+ * P3 renderer metadata (the adapter uses the synchronous
+ * `foldRecognizedMetadata`), guaranteeing P1/P2/P4 parity by construction.
+ * Behavior-preserving: this is a verbatim lift of the former inline P1/P2/P2b
+ * block. Mutates `entry` in place.
+ */
+export function applyPreContributorFields(entry, file, ctx, pkgMeta) {
     const ext = path.extname(file).toLowerCase();
-    const baseName = path.basename(file, ext);
-    const entry = {
-        name: canonicalName,
-        type: assetType,
-        quality: "generated",
-        confidence: 0.55,
-        source: "filename",
-    };
     // Priority 1: Package.json metadata
     if (pkgMeta) {
         if (pkgMeta.description && !entry.description) {
@@ -851,7 +1048,7 @@ async function buildEntryFromFile(file, assetType, canonicalName, dirPath, pkgMe
     // Priority 2: Frontmatter (for .md files -- overrides package.json description)
     // Secrets are excluded even when the file happens to be `.md`: the whole file
     // is the secret value and must never be read for frontmatter or any metadata.
-    if (ext === ".md" && assetType !== "secret") {
+    if (ext === ".md" && entry.type !== "secret") {
         const content = ctx.content();
         const parsed = parseFrontmatter(content);
         applyCuratedFrontmatter(entry, parsed.data);
@@ -861,6 +1058,19 @@ async function buildEntryFromFile(file, assetType, canonicalName, dirPath, pkgMe
             entry.parameters = fmParams;
         // Pass wiki-pattern frontmatter through onto the entry
         applyWikiFrontmatter(entry, parsed.data);
+        // D2 (#730): reread the namespaced `provenance:` block promoteProposal stamps.
+        applyProvenanceFrontmatter(entry, parsed.data);
+        // Stash-organization conventions (SPEC-8): config-gated capture of the
+        // self-situating body opening. Default off — enabling it changes indexed
+        // text (collapse-detector canary baselines shift, and embeddings for
+        // already-embedded entries are NOT regenerated; see docs/reference/configuration.md).
+        // Session-kind memories are raw transcripts and are never captured;
+        // secrets never reach this branch (guard above) and env files are not .md.
+        if (isBodyOpeningIndexingEnabled() && !hasSessionMemoryMarker(parsed.data, parsed.content)) {
+            const bodyOpening = extractBodyOpening(parsed.content);
+            if (bodyOpening)
+                entry.bodyOpening = bodyOpening;
+        }
         // Extract parameters from template placeholders ($1, $ARGUMENTS, {{named}})
         if (entry.type === "command") {
             const cmdParams = extractCommandParameters(parsed.content);
@@ -873,35 +1083,24 @@ async function buildEntryFromFile(file, assetType, canonicalName, dirPath, pkgMe
     // Env files (.env) and secret files (whole-file secrets) are deliberately
     // excluded — their contents are secrets and must never be parsed for @param
     // or any other metadata that could embed a value into the entry.
-    if (ext !== ".md" && assetType !== "env" && assetType !== "secret") {
+    if (ext !== ".md" && entry.type !== "env" && entry.type !== "secret") {
         const content = ctx.content();
         const scriptParams = extractScriptParameters(file, content);
         if (scriptParams)
             entry.parameters = scriptParams;
         applyCommentMetadata(entry, extractCommentMetadata(file, content));
     }
-    // Priority 3: Renderer metadata extraction
-    // When no pre-resolved match is available (generateMetadata path), run
-    // matchers now so the renderer can extract type-specific metadata.
-    const resolvedMatch = match ?? (await runMatchers(ctx));
-    if (resolvedMatch) {
-        const renderer = await getRenderer(resolvedMatch.renderer);
-        if (renderer) {
-            const renderCtx = buildRenderContext(ctx, resolvedMatch, [stashRoot]);
-            try {
-                await applyMetadataContributors(entry, {
-                    rendererName: renderer.name,
-                    renderContext: renderCtx,
-                });
-            }
-            catch (error) {
-                return {
-                    skip: true,
-                    warning: buildMetadataSkipWarning(file, assetType, error),
-                };
-            }
-        }
-    }
+}
+/**
+ * Priority 4 of the metadata pipeline — filename-heuristic fallbacks (P4) that
+ * run AFTER the renderer-contributor step (P3): a filename description when none
+ * was set, path/dir-derived tags, tag normalization, and alias generation.
+ * Extracted (Chunk 5 M-b) alongside {@link applyPreContributorFields} so both
+ * pipeline paths share it. Behavior-preserving verbatim lift. Mutates `entry`.
+ */
+export function applyPostContributorFields(entry, file, canonicalName, dirPath) {
+    const ext = path.extname(file).toLowerCase();
+    const baseName = path.basename(file, ext);
     // Priority 4: Filename heuristics (fallback)
     if (!entry.description) {
         entry.description = fileNameToDescription(baseName);
@@ -911,84 +1110,31 @@ async function buildEntryFromFile(file, assetType, canonicalName, dirPath, pkgMe
     if (!entry.tags || entry.tags.length === 0) {
         entry.tags = extractTagsFromPath(file, dirPath);
     }
+    // Stash-organization conventions (SPEC-2): directory (scope/domain) tokens
+    // always reach the tags column, even when the author set explicit tags, so
+    // nested assets keep the exact-tag ranking boost for their scope token.
+    // Derived from canonicalName (the ref subpath) rather than the filesystem
+    // path so the stash-walk and flat-walk indexing paths agree (the flat walk
+    // passes `path.dirname(file)` as dirPath, which strips directory segments
+    // from the fallback above). Filename tokens are deliberately NOT merged when
+    // explicit tags exist — they already live in the FTS name column and in
+    // aliases, and merging them would inflate exact-tag matches for every
+    // filename word. `normalizeTerms` below dedupes author-restated tokens.
+    entry.tags = [...(entry.tags ?? []), ...extractDirTagsFromName(canonicalName)];
     entry.tags = normalizeTerms(entry.tags ?? []);
     entry.aliases = mergeAliases(entry.aliases, buildAliases(canonicalName, entry.tags));
     // Search hints are only generated when LLM is configured (via enhanceStashWithLlm)
     // Heuristic search hints are too noisy to be useful for search quality
     entry.filename = path.basename(file);
-    return entry;
-}
-export async function generateMetadata(dirPath, assetType, files, typeRoot = dirPath) {
-    const entries = [];
-    const warnings = [];
-    const pkgMeta = extractPackageMetadata(dirPath);
-    for (const file of files) {
-        const ext = path.extname(file).toLowerCase();
-        const baseName = path.basename(file, ext);
-        const fileName = path.basename(file);
-        // Skip non-relevant files
-        if (!isRelevantAssetFile(assetType, fileName))
-            continue;
-        const canonicalName = deriveCanonicalAssetName(assetType, typeRoot, file) ?? baseName;
-        // Build file context with typeRoot as the stash root so renderer context
-        // and search hints are scoped to the type directory.
-        const fileCtx = buildFileContext(typeRoot, file);
-        // Step 1: type is explicit; delegate steps 2-6 to the shared pipeline.
-        const result = await buildEntryFromFile(file, assetType, canonicalName, dirPath, pkgMeta, typeRoot, fileCtx, null);
-        if ("skip" in result) {
-            warnings.push(result.warning);
-            continue;
-        }
-        entries.push(result);
-    }
-    return warnings.length > 0 ? { entries, warnings } : { entries };
-}
-/**
- * Generate metadata for files using the matcher system instead of a fixed asset type.
- *
- * This is the flat-walk counterpart of `generateMetadata`. It classifies each
- * file via `runMatchers()` and uses the matched type for canonical naming.
- * Files that no matcher claims are silently skipped.
- */
-export async function generateMetadataFlat(stashRoot, files) {
-    const entries = [];
-    const warnings = [];
-    const pkgMetaCache = new Map();
-    for (const file of files) {
-        if (!shouldIndexStashFile(stashRoot, file))
-            continue;
-        // Step 1: determine type and canonical name via the matcher system.
-        const ctx = buildFileContext(stashRoot, file);
-        const match = await runMatchers(ctx);
-        if (!match)
-            continue;
-        const assetType = match.type;
-        if (!isAssetType(assetType))
-            continue;
-        // If the file lives under a known type directory, use that as the root
-        // for canonical naming so names don't include the type prefix.
-        // e.g. scripts/deploy.sh → "deploy.sh" not "scripts/deploy.sh"
-        const ext = path.extname(file).toLowerCase();
-        const baseName = path.basename(file, ext);
-        const canonicalName = deriveCanonicalAssetNameFromStashRoot(assetType, stashRoot, file) ?? baseName;
-        // Resolve package.json metadata with a per-directory cache.
-        const dirPath = path.dirname(file);
-        if (!pkgMetaCache.has(dirPath)) {
-            pkgMetaCache.set(dirPath, extractPackageMetadata(dirPath));
-        }
-        const pkgMeta = pkgMetaCache.get(dirPath);
-        // Steps 2-6: delegate to the shared pipeline; pass the pre-resolved match
-        // so we don't run matchers a second time.
-        const result = await buildEntryFromFile(file, assetType, canonicalName, dirPath, pkgMeta, stashRoot, ctx, match);
-        if ("skip" in result) {
-            warnings.push(result.warning);
-            continue;
-        }
-        entries.push(result);
-    }
-    return warnings.length > 0 ? { entries, warnings } : { entries };
 }
-function buildMetadataSkipWarning(filePath, assetType, error) {
+// The pre-0.9.0 flat-walk matcher-pass metadata source was DELETED in Chunk 5
+// F4a M-core-3. Its role (recognize a stash root's files into durable entries)
+// is now the `akm` adapter's `recognize`, drained by `indexer/scan/drain-dir.ts`
+// (`drainDirDocuments` for the live indexer, `recognizeStashEntries` for the
+// `manifest` fallback / `registry` index builder / metadata unit tests). The
+// flip is the F4 engine swap; the shadow-parity gate proved recognize produced
+// the identical entries before the old pass was removed.
+export function buildMetadataSkipWarning(filePath, assetType, error) {
     const detail = error instanceof Error ? error.message : String(error);
     // Workflow errors are already multi-line `path:line — message` blocks; print
     // them as-is so the author sees a flat list without a redundant prefix.
@@ -1127,3 +1273,26 @@ export function extractTagsFromPath(filePath, rootDir) {
     }
     return Array.from(tags);
 }
+/**
+ * Extract scope/domain tags from the DIRECTORY segments of a canonical asset
+ * name (the ref subpath — e.g. `"projectA/auth-tip"` → `["projecta"]`).
+ *
+ * Unlike {@link extractTagsFromPath} this never tokenizes the filename
+ * segment: it exists so a nested asset's directory (scope/domain) tokens can
+ * be merged into explicit author tags without dragging every filename word
+ * into exact-tag matching (SPEC-2, docs/architecture/specs/stash-conventions-code-spec.md).
+ * Tokenization mirrors `extractTagsFromPath`: each segment splits on `-`/`_`/
+ * `.`, lowercased, single-character tokens dropped. A name with no directory
+ * segments yields no tags.
+ */
+export function extractDirTagsFromName(name) {
+    const tags = new Set();
+    for (const segment of name.split("/").slice(0, -1)) {
+        for (const token of segment.split(/[-_.]+/)) {
+            const clean = token.toLowerCase().trim();
+            if (clean && clean.length > 1)
+                tags.add(clean);
+        }
+    }
+    return Array.from(tags);
+}