akm-cli 0.9.2-alpha.1 → 0.9.2-alpha.2

package/dist/indexer/materialize-embeddings.js

@@ -0,0 +1,155 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { isVerbose, warn, warnVerbose } from "../core/warn.js";
+import { embedBatch } from "../llm/embedder.js";
+import { estimateTokenCount } from "../llm/embedders/remote.js";
+import { getEmbeddableEntryCount } from "../storage/repositories/index-entries-repository.js";
+import { deleteMeta, getMeta, setMeta } from "../storage/repositories/index-meta-repository.js";
+import { getAllEntriesForEmbedding, getEmbeddingCount, isVecAvailable, isVecFastPathComplete, isVecFastPathReady, purgeEmbeddings, setVecFastPathReady, upsertEmbedding, } from "../storage/repositories/index-vec-repository.js";
+import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./search/semantic-status.js";
+function throwIfAborted(signal) {
+    if (signal?.aborted) {
+        throw signal.reason instanceof Error ? signal.reason : new Error("index interrupted");
+    }
+}
+export async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds) {
+    throwIfAborted(signal);
+    if (config.semanticSearchMode === "off") {
+        onProgress({ phase: "embeddings", message: "Semantic search disabled; skipping embeddings." });
+        return { success: false, reason: "index-missing", message: "Semantic search is disabled." };
+    }
+    // A targeted call starts from an already-published generation. Preserve its
+    // trust decision in O(1): successful writes for the changed IDs keep a
+    // healthy fast path healthy, but can never promote a generation already
+    // marked degraded. Global runs can afford to verify the entire derived set.
+    const vecFastPathWasReady = isVecFastPathReady(db);
+    const currentFingerprint = deriveSemanticProviderFingerprint(config.embedding);
+    const storedFingerprint = getMeta(db, "embeddingFingerprint");
+    let targetEntryIds = entryIds;
+    if (storedFingerprint && storedFingerprint !== currentFingerprint) {
+        purgeEmbeddings(db, { dropVecTable: true });
+        deleteMeta(db, "embeddingDim");
+        // A provider/model change invalidates the entire vector generation, even
+        // when a targeted write happened to discover it first.
+        targetEntryIds = undefined;
+    }
+    try {
+        throwIfAborted(signal);
+        const allEntries = getAllEntriesForEmbedding(db, targetEntryIds);
+        if (allEntries.length === 0) {
+            onProgress({ phase: "embeddings", message: "Embeddings already up to date." });
+            setMeta(db, "embeddingFingerprint", currentFingerprint);
+            return { success: true };
+        }
+        onProgress({
+            phase: "embeddings",
+            message: `Generating embeddings for ${allEntries.length} entr${allEntries.length === 1 ? "y" : "ies"}.`,
+        });
+        const texts = allEntries.map((entry) => entry.searchText);
+        if (isVerbose()) {
+            const EMBED_BATCH_SIZE = 100;
+            const totalBatches = Math.ceil(texts.length / EMBED_BATCH_SIZE);
+            for (const [i, entry] of allEntries.entries()) {
+                const batchNum = Math.floor(i / EMBED_BATCH_SIZE) + 1;
+                const chars = entry.searchText.length;
+                const tokens = estimateTokenCount(entry.searchText);
+                const ref = entry.itemRef;
+                warnVerbose(`[embed] ${ref} (${chars} chars, est. ${tokens} tokens) → batch ${batchNum}/${totalBatches}`);
+            }
+        }
+        let heartbeatTimer;
+        try {
+            heartbeatTimer = setInterval(() => {
+                onProgress({
+                    phase: "embeddings",
+                    message: `Still generating embeddings for ${allEntries.length} entr${allEntries.length === 1 ? "y" : "ies"}; waiting on embedding provider.`,
+                });
+            }, 15000);
+            const embeddings = await embedBatch(texts, config.embedding, signal);
+            throwIfAborted(signal);
+            let storedCount = 0;
+            let skippedCount = 0;
+            let vecFailedCount = 0;
+            let vecUnavailableCount = 0;
+            db.transaction(() => {
+                for (const [i, entry] of allEntries.entries()) {
+                    const embedding = embeddings[i];
+                    if (!embedding)
+                        throw new Error(`Embedding provider returned no vector for ${entry.itemRef}.`);
+                    const result = upsertEmbedding(db, entry.id, embedding);
+                    if (result.stored)
+                        storedCount++;
+                    else
+                        skippedCount++;
+                    if (result.vec === "failed")
+                        vecFailedCount++;
+                    if (result.vec === "unavailable")
+                        vecUnavailableCount++;
+                }
+            })();
+            if (skippedCount > 0) {
+                warn(`[embed] ${skippedCount} embedding${skippedCount === 1 ? "" : "s"} skipped (entry deleted between queue and write)`);
+            }
+            const vecGenerationComplete = targetEntryIds === undefined ? isVecFastPathComplete(db) : vecFastPathWasReady;
+            setVecFastPathReady(db, vecFailedCount === 0 && vecUnavailableCount === 0 && vecGenerationComplete);
+            if (vecFailedCount > 0) {
+                warn(`[embed] ${vecFailedCount} sqlite-vec fast-path insert${vecFailedCount === 1 ? "" : "s"} failed — ` +
+                    "semantic search will use the slower JS-cosine fallback over stored embeddings. " +
+                    "Rebuild with 'akm index --full' after resolving the vec table (often a vector-dimension mismatch).");
+            }
+            onProgress({
+                phase: "embeddings",
+                message: `Stored ${storedCount} embedding${storedCount === 1 ? "" : "s"}.`,
+            });
+            setMeta(db, "embeddingFingerprint", currentFingerprint);
+            return { success: true, vecInsertFailures: vecFailedCount };
+        }
+        finally {
+            if (heartbeatTimer)
+                clearInterval(heartbeatTimer);
+        }
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        warn("Embedding generation failed, continuing without:", message);
+        onProgress({ phase: "embeddings", message: `Embedding generation failed: ${message}` });
+        return {
+            success: false,
+            reason: classifySemanticFailure(message),
+            message: `Semantic search verification failed: ${message}`,
+        };
+    }
+}
+/** Publish the canonical semantic health snapshot after a targeted mutation. */
+export function publishTargetedSemanticStatus(db, config, result) {
+    if (config.semanticSearchMode === "off") {
+        clearSemanticStatus();
+        setMeta(db, "hasEmbeddings", "0");
+        return;
+    }
+    const entryCount = getEmbeddableEntryCount(db);
+    const embeddingCount = getEmbeddingCount(db);
+    let status;
+    if (entryCount === 0)
+        status = "pending";
+    else if (embeddingCount >= entryCount)
+        status = isVecAvailable(db) && isVecFastPathReady(db) ? "ready-vec" : "ready-js";
+    else
+        status = "blocked";
+    setMeta(db, "hasEmbeddings", status === "ready-js" || status === "ready-vec" ? "1" : "0");
+    writeSemanticStatus({
+        status,
+        ...(status === "blocked" ? { reason: result.reason ?? "index-failed" } : {}),
+        ...(status === "blocked"
+            ? {
+                message: result.message ??
+                    `Semantic search verification failed (${embeddingCount}/${entryCount} embeddings available).`,
+            }
+            : {}),
+        providerFingerprint: deriveSemanticProviderFingerprint(config.embedding),
+        lastCheckedAt: new Date().toISOString(),
+        entryCount,
+        embeddingCount,
+    });
+}

package/dist/indexer/passes/metadata.js

@@ -6,7 +6,6 @@ import path from "node:path";
 import { parseBundleRef } from "../../core/asset/asset-ref.js";
 import { parseFrontmatter } from "../../core/asset/frontmatter.js";
 import { asNonEmptyString } from "../../core/common.js";
-import { loadUserConfig } from "../../core/config/config.js";
 import { isVerbose, warn } from "../../core/warn.js";
 export const SCOPE_KEYS = ["user", "agent", "run", "channel"];
 // ── Quality semantics (v1 spec §4.2) ────────────────────────────────────────
@@ -180,10 +179,8 @@ export function validateStashEntry(entry) {
     if (typeof e.derivedFrom === "string" && e.derivedFrom.trim().length > 0) {
         result.derivedFrom = e.derivedFrom.trim();
     }
-    // SPEC-8: `bodyOpening` must survive the projection. The extractor already
-    // trimmed and capped it at capture time.
-    if (typeof e.bodyOpening === "string" && e.bodyOpening.trim().length > 0) {
-        result.bodyOpening = e.bodyOpening;
+    if (typeof e.content === "string" && e.content.trim().length > 0) {
+        result.content = e.content;
     }
     if (typeof e.scope === "object" && e.scope !== null && !Array.isArray(e.scope)) {
         const scope = normalizeScopeObject(e.scope);
@@ -835,36 +832,13 @@ export function isEnrichmentComplete(entry) {
     const hasSearchHints = Array.isArray(entry.searchHints) && entry.searchHints.length > 0;
     return hasDescription && hasTags && hasSearchHints;
 }
-// ── Body-opening extraction (stash-conventions SPEC-8) ──────────────────────
+// ── Native Markdown search projection ──────────────────────────────────────
 /**
- * Maximum length of a captured self-situating body opening. Bounds index-size
- * growth and keeps a single verbose opening from dominating the low-weight
- * `content` FTS column.
+ * Maximum native Markdown prose carried by the low-weight `content` field.
+ * Structured fields remain separate and higher-weighted; this bound prevents
+ * large documents from dominating index size or embedding inputs.
  */
-export const BODY_OPENING_MAX_CHARS = 280;
-/**
- * Minimum characters retained when the cap truncates at a word boundary. A
- * boundary cut that would retain less than this falls back to a hard cut, so
- * one pathological long token cannot gut the capture.
- */
-const BODY_OPENING_MIN_RETAINED_CHARS = 250;
-/**
- * True when `index.indexBodyOpening` is enabled in the user config.
- *
- * The gate is the GLOBAL user config, read directly by the metadata pass so
- * every indexing entry point (stash walk, flat walk, write-path indexing)
- * honors the flag without parameter plumbing. Fail-open: an unreadable or
- * invalid config must never break indexing (CLI entry points surface config
- * errors loudly on their own), so any load failure reads as "off".
- */
-function isBodyOpeningIndexingEnabled() {
-    try {
-        return loadUserConfig().index?.indexBodyOpening === true;
-    }
-    catch {
-        return false;
-    }
-}
+export const MARKDOWN_CONTENT_MAX_CHARS = 16_384;
 /**
  * Locate a leading nested frontmatter block in a body: up to three blank
  * lines, then a `---` line, closed by a later `---` line. Mirrors the
@@ -892,7 +866,7 @@ function findInnerFrontmatterBlock(lines) {
  * marker — in the outer frontmatter data OR in a nested inner block at the
  * top of the body (both producer layouts exist; see base-linter's
  * `extractFrontmatterRefs`). Session bodies are raw transcripts, never a
- * self-situating opening.
+ * searchable content projection.
  */
 function hasSessionMemoryMarker(fmData, body) {
     if (typeof fmData.akm_memory_kind === "string")
@@ -930,96 +904,271 @@ function isFrontmatterShaped(lines, block) {
     }
     return true;
 }
+function truncateUnicodeSafe(text, maxChars) {
+    if (text.length <= maxChars)
+        return text;
+    let cut = text.slice(0, maxChars);
+    const lastCode = cut.charCodeAt(cut.length - 1);
+    if (lastCode >= 0xd800 && lastCode <= 0xdbff)
+        cut = cut.slice(0, -1);
+    const boundary = cut.lastIndexOf(" ");
+    if (boundary >= Math.floor(maxChars * 0.9))
+        cut = cut.slice(0, boundary);
+    return cut.trimEnd();
+}
+function parseMarkdownFenceOpening(line) {
+    const match = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line);
+    const run = match?.[1];
+    if (!run)
+        return undefined;
+    const marker = run[0];
+    // CommonMark forbids backticks in the info string of a backtick fence.
+    if (marker === "`" && match?.[2]?.includes("`"))
+        return undefined;
+    return { marker, length: run.length };
+}
+function isMarkdownFenceClosing(line, fence) {
+    let cursor = 0;
+    while (cursor < line.length && cursor < 3 && line[cursor] === " ")
+        cursor += 1;
+    const runStart = cursor;
+    while (cursor < line.length && line[cursor] === fence.marker)
+        cursor += 1;
+    if (cursor - runStart < fence.length)
+        return false;
+    return /^[\t ]*$/.test(line.slice(cursor));
+}
+function stripMarkdownHtmlComments(line, state) {
+    let cursor = 0;
+    let visible = "";
+    while (cursor < line.length) {
+        if (state.inComment) {
+            const close = line.indexOf("-->", cursor);
+            if (close < 0)
+                return visible;
+            state.inComment = false;
+            cursor = close + 3;
+            continue;
+        }
+        const open = line.indexOf("<!--", cursor);
+        if (open < 0)
+            return `${visible}${line.slice(cursor)}`;
+        visible += line.slice(cursor, open);
+        state.inComment = true;
+        cursor = open + 4;
+    }
+    return visible;
+}
+function findBalancedMarkdownClose(text, openAt, open, close) {
+    let depth = 1;
+    for (let cursor = openAt + 1; cursor < text.length; cursor += 1) {
+        const char = text[cursor];
+        if (char === "\\") {
+            cursor += 1;
+            continue;
+        }
+        if (char === open)
+            depth += 1;
+        if (char !== close)
+            continue;
+        depth -= 1;
+        if (depth === 0)
+            return cursor;
+    }
+    return undefined;
+}
 /**
- * Extract the first prose paragraph of a markdown body (frontmatter already
- * stripped by the caller): skip blank lines, ATX headings, setext `=`
- * underlines (discarding the heading text above them), thematic breaks,
- * fenced code blocks (``` or ~~~, including their contents), and a leading
- * nested frontmatter block — skipped only when its interior is actually
- * frontmatter-shaped, so prose wrapped in decorative `---` lines is still
- * captured; then collect consecutive non-blank lines until the paragraph
- * ends. Deliberate asymmetry: a `---` row after captured prose ENDS the
- * paragraph and keeps it (favoring the callout/thematic-break reading over
- * CommonMark's setext-H2), while a `=+` row can only be a setext underline
- * and so discards the pending lines as heading text. The result is capped at
- * {@link BODY_OPENING_MAX_CHARS} chars — truncated at the last word boundary
- * that still retains a substantial prefix, with a trailing ellipsis. Returns
- * `undefined` when the body has no prose (frontmatter-only files,
- * headings/fences-only bodies).
+ * Find the closing parenthesis of an inline link destination. Parentheses in
+ * angle-delimited destinations and quoted titles are data, while unquoted
+ * parentheses remain balanced. Backslash escapes suppress delimiter meaning.
  */
-export function extractBodyOpening(body) {
-    const lines = body.split(/\r?\n/);
-    const innerBlock = findInnerFrontmatterBlock(lines);
-    const start = innerBlock && isFrontmatterShaped(lines, innerBlock) ? innerBlock.close + 1 : 0;
-    const paragraph = [];
-    let inFence = false;
-    let fenceChar = "";
-    let inHtmlComment = false;
-    for (let i = start; i < lines.length; i += 1) {
-        const trimmed = lines[i].trim();
-        const fenceMatch = trimmed.match(/^(`{3,}|~{3,})/);
-        if (inFence) {
-            // Fence interiors are never prose (and may be secrets-adjacent command
-            // text); skip until the matching closing marker.
-            if (fenceMatch && fenceMatch[1].charAt(0) === fenceChar)
-                inFence = false;
+function findMarkdownDestinationClose(text, openAt) {
+    let depth = 1;
+    let phase = "before-destination";
+    let quote;
+    let parenthesizedTitle = false;
+    let titleSeparator = false;
+    for (let cursor = openAt + 1; cursor < text.length; cursor += 1) {
+        const char = text[cursor];
+        if (char === "\\") {
+            cursor += 1;
             continue;
         }
-        if (inHtmlComment) {
-            // Comment interiors are machinery (the skeleton convention facts open
-            // with a <!-- SOFT guidance --> block), never orientation prose.
-            if (trimmed.includes("-->"))
-                inHtmlComment = false;
+        if (phase === "before-destination") {
+            if (/\s/u.test(char ?? ""))
+                continue;
+            if (char === "<") {
+                phase = "angle-destination";
+                continue;
+            }
+            if (char === ")")
+                return cursor;
+            phase = "bare-destination";
+        }
+        if (phase === "angle-destination") {
+            if (char === ">")
+                phase = "after-destination";
             continue;
         }
-        if (fenceMatch) {
-            if (paragraph.length > 0)
-                break; // a fence ends an open paragraph
-            inFence = true;
-            fenceChar = fenceMatch[1].charAt(0);
+        if (phase === "title") {
+            if (quote) {
+                if (char === quote) {
+                    quote = undefined;
+                    phase = "after-destination";
+                    titleSeparator = false;
+                }
+                continue;
+            }
+            if (parenthesizedTitle) {
+                if (char === "(")
+                    depth += 1;
+                if (char !== ")")
+                    continue;
+                depth -= 1;
+                if (depth === 1) {
+                    parenthesizedTitle = false;
+                    phase = "after-destination";
+                    titleSeparator = false;
+                }
+            }
+            continue;
+        }
+        if (phase === "after-destination") {
+            if (/\s/u.test(char ?? "")) {
+                titleSeparator = true;
+                continue;
+            }
+            if (char === ")")
+                return cursor;
+            if (titleSeparator && (char === '"' || char === "'")) {
+                quote = char;
+                phase = "title";
+                continue;
+            }
+            if (titleSeparator && char === "(") {
+                depth += 1;
+                parenthesizedTitle = true;
+                phase = "title";
+                continue;
+            }
+            // Invalid trailing bytes are still consumed conservatively until the
+            // balanced outer close. Quotes here are ordinary bytes, never titles.
+            phase = "bare-destination";
+        }
+        if (phase !== "bare-destination")
+            continue;
+        if (char === "(") {
+            depth += 1;
+            continue;
+        }
+        if (char === ")") {
+            depth -= 1;
+            if (depth === 0)
+                return cursor;
             continue;
         }
-        if (trimmed.startsWith("<!--")) {
-            if (paragraph.length > 0)
-                break; // a comment ends an open paragraph
-            if (!trimmed.includes("-->"))
-                inHtmlComment = true;
+        if (/\s/u.test(char ?? "") && depth === 1) {
+            phase = "after-destination";
+            titleSeparator = true;
+        }
+    }
+    return undefined;
+}
+const MAX_MARKDOWN_LINK_NESTING = 32;
+/** Retain recursively projected link labels while dropping complete destinations. */
+function stripMarkdownLinkDestinations(text, nesting = 0) {
+    let visible = "";
+    let cursor = 0;
+    while (cursor < text.length) {
+        const isImage = text[cursor] === "!" && text[cursor + 1] === "[";
+        const labelOpen = isImage ? cursor + 1 : cursor;
+        if (text[labelOpen] !== "[") {
+            visible += text[cursor];
+            cursor += 1;
             continue;
         }
-        if (/^=+$/.test(trimmed)) {
-            // A row of `=` is a setext H1 underline: the pending lines are heading
-            // text, not prose — discard them and keep searching. (A bare `=` row
-            // with nothing pending is skipped like any other non-prose divider.)
-            paragraph.length = 0;
+        const labelClose = findBalancedMarkdownClose(text, labelOpen, "[", "]");
+        const destinationOpen = labelClose === undefined ? undefined : labelClose + 1;
+        if (destinationOpen === undefined || text[destinationOpen] !== "(") {
+            visible += text[cursor];
+            cursor += 1;
             continue;
         }
-        const isBlank = trimmed === "";
-        const isHeading = /^#{1,6}(\s|$)/.test(trimmed);
-        const isThematicBreak = /^(-{3,}|\*{3,}|_{3,})$/.test(trimmed);
-        if (isBlank || isHeading || isThematicBreak) {
-            if (paragraph.length > 0)
-                break; // paragraph complete
-            continue; // still searching for the first prose line
+        const destinationClose = findMarkdownDestinationClose(text, destinationOpen);
+        if (destinationClose === undefined) {
+            visible += text[cursor];
+            cursor += 1;
+            continue;
         }
-        paragraph.push(trimmed);
+        // Labels may themselves contain images/links. Recursively project them so
+        // an inner destination cannot become visible when its outer link is
+        // removed. At an adversarial nesting depth, omit the label rather than
+        // leaking its unparsed bytes into the search projection.
+        if (nesting < MAX_MARKDOWN_LINK_NESTING) {
+            visible += stripMarkdownLinkDestinations(text.slice(labelOpen + 1, labelClose), nesting + 1);
+        }
+        cursor = destinationClose + 1;
     }
-    const text = paragraph.join("\n");
+    return visible;
+}
+/**
+ * Derive the one low-weight search projection for an AKM-native Markdown body.
+ * Frontmatter is removed by the caller. This projection drops comments,
+ * fenced code, raw link targets, and structural punctuation while retaining
+ * prose, headings, link labels, and inline identifiers. It never receives
+ * secret/env/session bytes; that policy is enforced at the adapter metadata
+ * boundary below.
+ */
+export function projectMarkdownContent(body) {
+    const lines = body.split(/\r?\n/);
+    const innerBlock = findInnerFrontmatterBlock(lines);
+    const start = innerBlock && isFrontmatterShaped(lines, innerBlock) ? innerBlock.close + 1 : 0;
+    const projected = [];
+    let fence;
+    const htmlComment = { inComment: false };
+    for (let i = start; i < lines.length; i += 1) {
+        const rawLine = lines[i];
+        if (fence) {
+            if (isMarkdownFenceClosing(rawLine, fence))
+                fence = undefined;
+            continue;
+        }
+        // A leading fence owns the whole line, including any info string that
+        // resembles HTML. Comment state therefore cannot begin inside a fence.
+        if (!htmlComment.inComment) {
+            const openingFence = parseMarkdownFenceOpening(rawLine);
+            if (openingFence) {
+                fence = openingFence;
+                continue;
+            }
+        }
+        let trimmed = stripMarkdownHtmlComments(rawLine, htmlComment).trim();
+        const openingFence = parseMarkdownFenceOpening(trimmed);
+        if (openingFence) {
+            fence = openingFence;
+            continue;
+        }
+        if (!trimmed || /^(-{3,}|\*{3,}|_{3,}|=+)$/.test(trimmed))
+            continue;
+        if (/^\s*\[[^\]]+\]:\s*\S+/.test(trimmed))
+            continue;
+        if (/^<[^>]+>$/.test(trimmed))
+            continue;
+        // Preserve human-facing labels and inline identifiers, never destinations.
+        trimmed = stripMarkdownLinkDestinations(trimmed);
+        trimmed = trimmed.replace(/`{1,2}([^`]+)`{1,2}/g, "$1");
+        trimmed = trimmed.replace(/^#{1,6}\s+/, "");
+        trimmed = trimmed.replace(/^(?:>|[-+*]|\d+[.)])\s+/, "");
+        trimmed = trimmed.replace(/<[^>]+>/g, " ");
+        trimmed = trimmed.replace(/[|]+/g, " ");
+        trimmed = trimmed.replace(/\s+/g, " ").trim();
+        if (trimmed)
+            projected.push(trimmed);
+    }
+    const text = projected.join(" ").replace(/\s+/g, " ").trim();
     if (!text)
         return undefined;
-    if (text.length <= BODY_OPENING_MAX_CHARS)
-        return text;
-    // Cap: prefer a word-boundary cut (never mid-token), but only when it keeps
-    // a substantial prefix; append a one-char ellipsis inside the budget.
-    const slice = text.slice(0, BODY_OPENING_MAX_CHARS - 1);
-    const lastBoundary = Math.max(slice.lastIndexOf(" "), slice.lastIndexOf("\n"), slice.lastIndexOf("\t"));
-    let cut = lastBoundary >= BODY_OPENING_MIN_RETAINED_CHARS ? slice.slice(0, lastBoundary) : slice;
-    // slice() counts UTF-16 code units, so the no-boundary fallback can end on
-    // the high half of a surrogate pair — a lone surrogate that corrupts the
-    // FTS/embedding text. Drop it rather than emit invalid UTF-16.
-    const lastCode = cut.charCodeAt(cut.length - 1);
-    if (lastCode >= 0xd800 && lastCode <= 0xdbff)
-        cut = cut.slice(0, -1);
-    return `${cut.trimEnd()}…`;
+    return truncateUnicodeSafe(text, MARKDOWN_CONTENT_MAX_CHARS);
 }
 // ── Metadata Generation ─────────────────────────────────────────────────────
 /**
@@ -1060,16 +1209,12 @@ export function applyPreContributorFields(entry, file, ctx, pkgMeta) {
         applyWikiFrontmatter(entry, parsed.data);
         // D2 (#730): reread the namespaced `provenance:` block promoteProposal stamps.
         applyProvenanceFrontmatter(entry, parsed.data);
-        // Stash-organization conventions (SPEC-8): config-gated capture of the
-        // self-situating body opening. Default off — enabling it changes indexed
-        // text (collapse-detector canary baselines shift, and embeddings for
-        // already-embedded entries are NOT regenerated; see docs/reference/configuration.md).
-        // Session-kind memories are raw transcripts and are never captured;
-        // secrets never reach this branch (guard above) and env files are not .md.
-        if (isBodyOpeningIndexingEnabled() && !hasSessionMemoryMarker(parsed.data, parsed.content)) {
-            const bodyOpening = extractBodyOpening(parsed.content);
-            if (bodyOpening)
-                entry.bodyOpening = bodyOpening;
+        // Native Markdown has one bounded low-weight body projection. Sensitive
+        // types and raw session/checkpoint material never cross this boundary.
+        if (entry.type !== "env" && entry.type !== "session" && !hasSessionMemoryMarker(parsed.data, parsed.content)) {
+            const contentProjection = projectMarkdownContent(parsed.content);
+            if (contentProjection)
+                entry.content = contentProjection;
         }
         // Extract parameters from template placeholders ($1, $ARGUMENTS, {{named}})
         if (entry.type === "command") {

package/dist/indexer/scan/doc-to-entry.js

@@ -121,7 +121,6 @@ export function indexDocumentToStashEntry(doc) {
         entry.toc = dj.toc;
     if (dj.parameters !== undefined)
         entry.parameters = dj.parameters;
-    assignString(entry, "bodyOpening", dj.bodyOpening);
     if (dj.source !== undefined)
         entry.source = dj.source;
     assignString(entry, "category", dj.category);