blume 1.5.2 → 1.6.0

package/src/markdown/heading-anchors.ts

@@ -1,24 +1,45 @@
 /**
- * Turn every section heading into its own permalink. A Satteri hast plugin runs
- * after Markdown is turned into hast and wraps each `<h2>`–`<h6>`'s content in an
- * `<a href="#slug">`, so a reader can click the heading to copy, bookmark, or
- * share a link straight to that section. `<h1>` (the page title) is slugged for
- * parity but left unwrapped.
+ * Heading ids, trailing markers, and self-linking anchors. A Satteri hast
+ * plugin runs after Markdown is turned into hast and, for every heading:
+ *
+ * - parses trailing markers (`[#custom-id]`, `{#custom-id}`, `[!toc]`, `[toc]`
+ *   — see `core/heading-markers.ts`) and strips them from the rendered text;
+
+ * - assigns the anchor `id` (the `[#custom-id]` pin, else a `github-slugger`
+ *   slug of the marker-free text);
+ * - wraps `<h2>`–`<h6>` content in an `<a href="#slug">` so a reader can click
+ *   the heading to copy, bookmark, or share a link straight to that section
+ *   (`<h1>` — the page title — is slugged for parity but left unwrapped, and
+ *   `wrap: false` turns the anchor links off without losing the markers).
  *
  * Satteri's own `heading-ids` plugin (which assigns the `id` used by the table
  * of contents) runs *after* every user hast plugin, and it reuses an `id` that
- * is already present rather than re-slugging. So this plugin is the authoritative
- * id setter: it slugs each heading with the same algorithm (a per-document
- * `github-slugger`, the library Satteri and rehype-slug both use) and writes the
- * `id`, which `heading-ids` then adopts — keeping the in-page anchor, the
- * heading's `id`, and the TOC entry in lockstep. To match Satteri's duplicate
- * disambiguation (`setup`, `setup-1`, …) exactly, it advances the slugger over
- * `<h1>`–`<h6>` in document order even though only `<h2>`–`<h6>` get wrapped.
+ * is already present rather than re-slugging. So this plugin is the
+ * authoritative id setter: it slugs each heading with the same algorithm (a
+ * per-document `github-slugger`, the library Satteri and rehype-slug both use)
+ * and writes the `id`, which `heading-ids` then adopts — keeping the in-page
+ * anchor, the heading's `id`, and the TOC entry in lockstep. To match
+ * Satteri's duplicate disambiguation (`setup`, `setup-1`, …) exactly, it
+ * advances the slugger over `<h1>`–`<h6>` in document order even though only
+ * `<h2>`–`<h6>` get wrapped.
+ *
+ * TOC visibility flows out through the render's frontmatter: the slugs of
+ * `[!toc]` headings are pushed onto `frontmatter[TOC_HIDDEN_KEY]`, which Astro
+ * surfaces as `remarkPluginFrontmatter` so the page template can filter them
+ * out of the headings list. `[toc]` headings render with the
+ * `blume-toc-only` class (visually hidden, still a live anchor target) so the
+ * TOC entry has somewhere to scroll to.
  */
 
 import { satteriCollectHastText } from "@astrojs/markdown-satteri";
 import GithubSlugger from "github-slugger";
 
+import {
+  occupySlug,
+  parseHeadingMarkers,
+  TOC_HIDDEN_KEY,
+} from "../core/heading-markers.ts";
+
 /** A hast property value: an attribute primitive or a token list. */
 type HastPropertyValue = string | number | boolean | (string | number)[];
 
@@ -50,10 +71,18 @@ export interface HeadingAnchorPlugin {
   };
 }
 
+export interface HeadingAnchorOptions {
+  /** Wrap `<h2>`–`<h6>` in self-linking anchors (`markdown.headingAnchors`). */
+  wrap?: boolean;
+}
+
 /** Headings slugged for id parity with Satteri; only a subset gets wrapped. */
 const HEADINGS = ["h1", "h2", "h3", "h4", "h5", "h6"];
 const WRAPPED = new Set(["h2", "h3", "h4", "h5", "h6"]);
 
+/** The class that renders a `[toc]`-only heading as an invisible anchor. */
+const TOC_ONLY_CLASS = "blume-toc-only";
+
 /** True if the subtree already contains an `<a>`, so wrapping would nest links. */
 const containsAnchor = (node: HastNode): boolean => {
   for (const child of node.children ?? []) {
@@ -64,61 +93,193 @@ const containsAnchor = (node: HastNode): boolean => {
   return false;
 };
 
-// One slugger per document render. The plugin instance is shared across every
-// page, but slug disambiguation must reset per document; the render-scoped
-// `astro` data object is a stable, unique key for one render (entries are
-// dropped once the render is collected, so this never leaks).
+// One state record per document render. The plugin instance is shared across
+// every page, but slug disambiguation (and the hidden-heading list) must reset
+// per document; the render-scoped `astro` data object is a stable, unique key
+// for one render (entries are dropped once the render is collected, so this
+// never leaks).
+interface RenderState {
+  /** Slugs of `[!toc]` headings, shared by reference with the frontmatter. */
+  hidden: string[];
+  slugger: GithubSlugger;
+}
+
 const FALLBACK_SCOPE = {};
-const sluggers = new WeakMap<object, GithubSlugger>();
+const states = new WeakMap<object, RenderState>();
 
-const sluggerFor = (ctx: HastContext): GithubSlugger => {
+const stateFor = (ctx: HastContext): RenderState => {
   const scope = ctx.data?.astro ?? ctx.data ?? FALLBACK_SCOPE;
-  const existing = sluggers.get(scope);
+  const existing = states.get(scope);
   if (existing) {
     return existing;
   }
-  const slugger = new GithubSlugger();
-  sluggers.set(scope, slugger);
-  return slugger;
+  const state: RenderState = { hidden: [], slugger: new GithubSlugger() };
+  states.set(scope, state);
+  // Surface the hidden list on the render's frontmatter (Astro's
+  // `remarkPluginFrontmatter`) by reference, so slugs pushed later in the
+  // document flow through. Assigning a fresh array on state creation also
+  // clears a stale list left by a previous render of the same entry.
+  const frontmatter = ctx.data?.astro?.frontmatter;
+  if (frontmatter) {
+    frontmatter[TOC_HIDDEN_KEY] = state.hidden;
+  }
+  return state;
 };
 
 /** Whether a heading already carries a usable string `id`. */
 const isStringId = (value: HastPropertyValue | undefined): value is string =>
   typeof value === "string";
 
+/** A parsed heading: marker-free children plus what the markers pinned. */
+interface StrippedHeading {
+  children: HastNode[];
+  id?: string;
+  /** Characters the marker strip removed from the heading's text content. */
+  strippedLength: number;
+  toc?: "hide" | "only";
+}
+
+/**
+ * Strip trailing markers from a heading's last direct text child. Markers only
+ * count at the very end of the heading, so a heading ending in inline code,
+ * emphasis, or an expression has no marker position — mirroring the scan-time
+ * source scanner, which likewise only matches markers that end the raw line.
+ */
+const stripMarkers = (node: HastNode): StrippedHeading => {
+  const children = node.children ?? [];
+  const last = children.at(-1);
+  const none = { children, strippedLength: 0 };
+  if (last?.type !== "text" || last.value === undefined) {
+    return none;
+  }
+  const markers = parseHeadingMarkers(last.value);
+  if (markers.id === undefined && markers.toc === undefined) {
+    return none;
+  }
+  const kept =
+    markers.text === ""
+      ? children.slice(0, -1)
+      : [...children.slice(0, -1), { ...last, value: markers.text }];
+  // A heading that is nothing but markers (`## [toc]`) keeps them as literal
+  // text: with no heading text left there is nothing to annotate, and
+  // stripping would leave an invisible empty element with an empty id and a
+  // blank TOC entry. The scan-time scanner and the search extractor mirror
+  // this rule.
+  if (kept.length === 0) {
+    return none;
+  }
+  return {
+    children: kept,
+    id: markers.id,
+    strippedLength: last.value.length - markers.text.length,
+    toc: markers.toc,
+  };
+};
+
 /** The slug for a heading, mirroring Satteri's `heading-ids` exactly. */
 const slugFor = (
   node: HastNode,
   ctx: HastContext,
-  slugger: GithubSlugger
+  slugger: GithubSlugger,
+  stripped: StrippedHeading
 ): string => {
-  const rawText = ctx.textContent(node);
+  if (stripped.id !== undefined) {
+    // Pinning occupies the id, so a later heading whose auto-slug collides
+    // disambiguates (`setup` → `setup-1`) instead of duplicating the anchor.
+    occupySlug(slugger, stripped.id);
+    return stripped.id;
+  }
+  const existingId = node.properties?.id;
+  if (isStringId(existingId)) {
+    return existingId;
+  }
+  // The marker suffix is a trailing slice of the text content, so the
+  // marker-free text is the content minus exactly what the strip removed.
+  const fullText = ctx.textContent(node);
+  const rawText = stripped.strippedLength
+    ? fullText.slice(0, fullText.length - stripped.strippedLength)
+    : fullText;
   // `frontmatter`-interpolated MDX headings (`## {frontmatter.title}`) need the
   // resolved value; the helper is the same one `heading-ids` defers to.
   // SAFETY: HastNode is a structural subset of the hast element shape the
-  // helper walks (children/type/value), so the visited node always fits.
+  // helper walks (children/type/value), so the node always fits.
   const text = rawText.includes("frontmatter")
     ? satteriCollectHastText(
-        node as Parameters<typeof satteriCollectHastText>[0],
+        {
+          ...node,
+          children: stripped.children,
+        } as Parameters<typeof satteriCollectHastText>[0],
         ctx.data?.astro?.frontmatter ?? {}
       )
     : rawText;
-  const existingId = node.properties?.id;
-  return isStringId(existingId) ? existingId : slugger.slug(text);
+  return slugger.slug(text);
 };
 
-/** Build the plugin. Wraps `<h2>`–`<h6>` in self-linking anchors. */
-export const headingAnchorPlugin = (): HeadingAnchorPlugin => ({
+/** The heading's class list with `blume-toc-only` appended. */
+const withTocOnlyClass = (
+  value: HastPropertyValue | undefined
+): (string | number)[] => {
+  if (Array.isArray(value)) {
+    return [...value, TOC_ONLY_CLASS];
+  }
+  return isStringId(value) && value !== ""
+    ? [value, TOC_ONLY_CLASS]
+    : [TOC_ONLY_CLASS];
+};
+
+/** True if the heading already carries the `[toc]`-only class (a re-visit). */
+const hasTocOnlyClass = (node: HastNode): boolean => {
+  const value = node.properties?.className;
+  return Array.isArray(value)
+    ? value.includes(TOC_ONLY_CLASS)
+    : value === TOC_ONLY_CLASS;
+};
+
+/**
+ * Build the plugin. Always parses markers and assigns ids; `wrap: false` only
+ * disables the self-linking anchor wrap on `<h2>`–`<h6>`.
+ */
+export const headingAnchorPlugin = (
+  options: HeadingAnchorOptions = {}
+): HeadingAnchorPlugin => ({
   element: {
     filter: HEADINGS,
     visit(node, ctx) {
-      const slug = slugFor(node, ctx, sluggerFor(ctx));
-      const wrap = node.tagName
-        ? WRAPPED.has(node.tagName) && slug !== "" && !containsAnchor(node)
-        : false;
+      const state = stateFor(ctx);
+      const stripped = stripMarkers(node);
+      const slug = slugFor(node, ctx, state.slugger, stripped);
+      if (stripped.toc === "hide") {
+        state.hidden.push(slug);
+      }
+      const tocOnly = stripped.toc === "only" || hasTocOnlyClass(node);
+      const wrap =
+        options.wrap !== false &&
+        node.tagName !== undefined &&
+        WRAPPED.has(node.tagName) &&
+        slug !== "" &&
+        !tocOnly &&
+        !containsAnchor(node);
       if (!wrap) {
-        // Unwrapped headings (h1, an empty slug, or one that already links) still
-        // need the id so `heading-ids` adopts it instead of re-slugging.
+        // Marker-free headings mutate in place; a stripped or `[toc]`-only one
+        // needs its children (and class) replaced, so it re-emits as a new
+        // element carrying the original children as refs.
+        if (stripped.strippedLength || (tocOnly && !hasTocOnlyClass(node))) {
+          const properties = tocOnly
+            ? {
+                ...node.properties,
+                className: withTocOnlyClass(node.properties?.className),
+                id: slug,
+              }
+            : { ...node.properties, id: slug };
+          return {
+            children: stripped.children,
+            properties,
+            tagName: node.tagName,
+            type: "element",
+          };
+        }
+        // Unwrapped headings (h1, an empty slug, or one that already links)
+        // still need the id so `heading-ids` adopts it instead of re-slugging.
         if (!isStringId(node.properties?.id)) {
           ctx.setProperty(node, "id", slug);
         }
@@ -129,7 +290,7 @@ export const headingAnchorPlugin = (): HeadingAnchorPlugin => ({
       return {
         children: [
           {
-            children: node.children ?? [],
+            children: stripped.children,
             properties: {
               className: ["blume-heading-anchor"],
               href: `#${slug}`,

package/src/markdown/include.ts

@@ -0,0 +1,247 @@
+import { fileURLToPath } from "node:url";
+
+import { resolve } from "pathe";
+
+import type { IncludeStatement } from "../core/includes.ts";
+import {
+  advanceHtmlCommentState,
+  expandIncludeTarget,
+  hasIncludeStatements,
+  parseIncludeLine,
+} from "../core/includes.ts";
+import type { MdastNode, MdastValue } from "./mdast.ts";
+
+/**
+ * Sätteri MDAST plugin for `<include>` statements — the render half of
+ * content includes (see `core/includes.ts` for the semantics and the
+ * string-level half that powers search, llms.txt, and the `.md` mirrors).
+ * Runs first in the plugin chain: mutations apply before the next plugin
+ * visits, so callouts, mermaid, and math inside a spliced partial transform
+ * exactly like inline content. Replacements use Sätteri's `{ raw }` escape
+ * hatch, so the target is parsed in the including page's format.
+ *
+ * In `.mdx`, a lowercase `<include>` arrives as an `mdxJsxFlowElement`. In
+ * plain `.md`, `<include>` isn't a known block-level HTML tag, so CommonMark
+ * parses the statement line as a *paragraph* holding inline `html` nodes —
+ * the paragraph visitor slices the statement back out of the source by
+ * position. A statement swallowed into a block `html` node (adjacent to real
+ * block HTML) is handled line-wise too. Statements must occupy their own
+ * line — includes are block-level.
+ */
+
+/** The attribute slice of an `mdxJsxAttribute` node the plugin reads. The
+ * index signature keeps the array assignable to `MdastNode`'s `MdastValue`
+ * properties. */
+interface JsxAttributeNode {
+  [key: string]: MdastValue;
+  type?: string;
+  name?: string;
+  value?: MdastValue;
+}
+
+/** The `mdxJsxFlowElement` slice the plugin reads. */
+interface JsxFlowNode extends MdastNode {
+  name?: string | null;
+  attributes?: JsxAttributeNode[];
+  position?: {
+    start?: { line?: number };
+    end?: { line?: number };
+  };
+}
+
+/** The raw-HTML node slice (`.md` pages) the plugin reads. */
+interface HtmlNode extends MdastNode {
+  value: string;
+}
+
+/** The position slice used to recover a paragraph's raw source text. */
+interface PositionedNode extends MdastNode {
+  position?: {
+    start?: { offset?: number };
+    end?: { offset?: number };
+  };
+}
+
+/** The visitor-context slice the plugin uses (see `mdast.ts` for the model). */
+interface IncludeVisitorContext {
+  fileURL: URL | undefined;
+  source: string;
+  replaceNode: (node: MdastNode, replacement: { raw: string }) => void;
+  textContent: (node: MdastNode) => string;
+  report: (report: {
+    message: string;
+    node?: MdastNode;
+    severity?: "error" | "warning" | "info";
+  }) => void;
+}
+
+/** A visible stand-in for a statement that failed to resolve, so a broken
+ * include can't silently render as nothing in dev. */
+const errorBlock = (message: string): string =>
+  `> **Include error:** ${message}`;
+
+export interface IncludePluginOptions {
+  /** The docs content root; bounds include resolution when set. */
+  contentRoot?: string;
+}
+
+/** A plain string attribute value; expression values don't carry a path. */
+const isStringValue = (value: MdastValue): value is string =>
+  typeof value === "string";
+
+const statementFromJsx = (
+  node: JsxFlowNode,
+  ctx: IncludeVisitorContext
+): IncludeStatement => {
+  const attributes: IncludeStatement["attributes"] = {};
+  for (const attr of node.attributes ?? []) {
+    if (attr.type !== "mdxJsxAttribute" || !isStringValue(attr.value)) {
+      continue;
+    }
+    if (attr.name === "lang" && attr.value) {
+      attributes.lang = attr.value;
+    }
+    if (attr.name === "meta" && attr.value) {
+      attributes.meta = attr.value;
+    }
+  }
+  return { attributes, target: ctx.textContent(node).trim() };
+};
+
+export const includePlugin = (options: IncludePluginOptions = {}) => {
+  // An ejected config carries a project-relative content root ("docs"); the
+  // generated `.blume` config an absolute one. Resolve once — `resolve` is a
+  // no-op for absolute paths and anchors relative ones at the process cwd,
+  // which is the project root wherever Astro runs.
+  const contentRoot = options.contentRoot
+    ? resolve(options.contentRoot)
+    : undefined;
+
+  const splice = async (
+    node: MdastNode,
+    statement: IncludeStatement,
+    ctx: IncludeVisitorContext
+  ): Promise<string> => {
+    if (!statement.target) {
+      const message = "<include> needs a file path as its text.";
+      ctx.report({ message, node, severity: "warning" });
+      return errorBlock(message);
+    }
+    if (!ctx.fileURL) {
+      const message = `Include target ${statement.target} can't resolve: the compiler received no file URL.`;
+      ctx.report({ message, node, severity: "warning" });
+      return errorBlock(message);
+    }
+    const expanded = await expandIncludeTarget(statement, {
+      contentRoot,
+      sourcePath: fileURLToPath(ctx.fileURL),
+    });
+    if ("error" in expanded) {
+      ctx.report({
+        message: expanded.error.message,
+        node,
+        severity: "warning",
+      });
+      return errorBlock(expanded.error.message);
+    }
+    for (const nested of expanded.errors) {
+      ctx.report({ message: nested.message, node, severity: "warning" });
+    }
+    return expanded.text;
+  };
+
+  /**
+   * Splice every statement line in a text block; `null` when none matched.
+   * Statement detection mirrors the string-level scanner's `.md` line rules
+   * (`parseIncludeLine`, HTML comment tracking) so the rendered page and the
+   * indexed/mirrored surfaces agree on which lines splice: a statement inside
+   * `<!-- -->` or indented like code stays verbatim on both sides.
+   */
+  const spliceLines = async (
+    node: MdastNode,
+    text: string,
+    ctx: IncludeVisitorContext
+  ): Promise<string | null> => {
+    let matched = false;
+    let inComment = false;
+    const lines = await Promise.all(
+      text.split("\n").map((line) => {
+        const wasInComment = inComment;
+        inComment = advanceHtmlCommentState(line, inComment);
+        const statement = wasInComment ? null : parseIncludeLine(line);
+        if (!statement) {
+          return line;
+        }
+        matched = true;
+        return splice(node, statement, ctx);
+      })
+    );
+    return matched ? lines.join("\n") : null;
+  };
+
+  return {
+    async html(node: HtmlNode, ctx: IncludeVisitorContext) {
+      if (!hasIncludeStatements(node.value)) {
+        return;
+      }
+      // A statement adjacent to real block HTML gets swallowed into that
+      // block's `html` node; splice each statement line, keep the rest.
+      const replaced = await spliceLines(node, node.value, ctx);
+      if (replaced !== null) {
+        ctx.replaceNode(node, { raw: replaced });
+      }
+    },
+    async mdxJsxFlowElement(node: JsxFlowNode, ctx: IncludeVisitorContext) {
+      if (node.name !== "include") {
+        return;
+      }
+      // The string-level scanner (search, mirrors, llms-full.txt, the HMR
+      // graph) only recognizes single-line statements; a wrapped element
+      // would render content those surfaces never see, so reject it loudly
+      // instead of splicing it invisibly.
+      const start = node.position?.start?.line;
+      const end = node.position?.end?.line;
+      if (start !== undefined && end !== undefined && start !== end) {
+        const message =
+          "<include> must be written on a single line: <include>./path.mdx</include>.";
+        ctx.report({ message, node, severity: "warning" });
+        ctx.replaceNode(node, { raw: errorBlock(message) });
+        return;
+      }
+      ctx.replaceNode(node, {
+        raw: await splice(node, statementFromJsx(node, ctx), ctx),
+      });
+    },
+    name: "blume-include",
+    // Positions are opt-in since satteri 0.10 (the parse skips the line index
+    // when no plugin reads them); both the paragraph slice recovery and the
+    // multi-line JSX rejection depend on them.
+    options: { position: true },
+    async paragraph(node: PositionedNode, ctx: IncludeVisitorContext) {
+      // `<include>` isn't a known block-level HTML tag, so in plain `.md` a
+      // statement line parses as a paragraph of inline `html` + text nodes.
+      // Recover the raw text by position and splice the statement lines. The
+      // slice is widened to whole source lines so container markers the
+      // paragraph position excludes (a blockquote's `>`, a list item's `-`)
+      // stay visible — a statement inside those containers is not on a line
+      // of its own, and the string-level scanner never expands it, so
+      // splicing here would render content search and the mirrors never see.
+      const start = node.position?.start?.offset;
+      const end = node.position?.end?.offset;
+      if (start === undefined || end === undefined) {
+        return;
+      }
+      const lineStart = ctx.source.lastIndexOf("\n", start - 1) + 1;
+      const lineEndIndex = ctx.source.indexOf("\n", end);
+      const lineEnd = lineEndIndex === -1 ? ctx.source.length : lineEndIndex;
+      const text = ctx.source.slice(lineStart, lineEnd);
+      if (!hasIncludeStatements(text)) {
+        return;
+      }
+      const replaced = await spliceLines(node, text, ctx);
+      if (replaced !== null) {
+        ctx.replaceNode(node, { raw: replaced });
+      }
+    },
+  };
+};