@dogsbay/format-starlight 0.2.0-beta.114 → 0.2.0-beta.116

package/dist/parse.js

@@ -9,6 +9,7 @@
  *   - Cloudflare custom components (Feature, Plan, Description, etc.) via customization layer
  *   - Standard CommonMark (headings, paragraphs, lists, tables, code, links)
  */
+import { assignHeadingSlugs } from "./github-slug.js";
 import MarkdownIt from "markdown-it";
 import matter from "gray-matter";
 const md = new MarkdownIt({ html: true, linkify: true });
@@ -389,7 +390,8 @@ function segmentsToTree(segments, componentMap, adapter) {
                 continue;
             }
             const props = mapping.propsFromAttrs?.(seg.attrs || {}) || {};
-            const children = seg.inner ? starlightToTree(seg.inner, { adapter }) : [];
+            // Nested: the outermost call assigns ids once, over the whole page.
+            const children = seg.inner ? starlightToTree(seg.inner, { adapter, nested: true }) : [];
             tree.push({ type: mapping.type, props, children });
         }
     }
@@ -694,7 +696,10 @@ export function starlightToTree(source, options) {
     const segments = segmentMdx(cleaned, componentMap, selfClosing);
     const tree = segmentsToTree(segments, componentMap, adapter);
     const unwrapped = unwrapStepsOrderedList(tree);
-    return groupConsecutiveCards(unwrapped);
+    const grouped = groupConsecutiveCards(unwrapped);
+    if (!options?.nested)
+        assignHeadingSlugs(grouped);
+    return grouped;
 }
 /**
  * Starlight's <Steps> wraps a markdown ordered list, producing:

package/dist/template-tokens.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Protecting template directives through the MDX parse (template mode,
+ * plans/page-templates.md).
+ *
+ * The directives the converter emits — `{% with %}{% include %}`,
+ * `{{ x }}`, `{% if %}` — pass through the Starlight MDX parser and the
+ * Dogsbay-MD serializer on their way out. As plain text they were damaged
+ * twice over on the first Cloudflare run: Markdown read `\"` inside a
+ * string literal as an escaped quote, the serializer escaped `_` and `<`
+ * inside a condition, and an unmapped component block swallowed the
+ * `{% endif %}` line after it.
+ *
+ * So each directive travels as an opaque ASCII token (a number between
+ * two marker words) and is restored in the TREE after parsing:
+ *   - a paragraph holding only tokens becomes a `directive-raw` block;
+ *   - a token inside text becomes a `directive-raw-inline` node;
+ *   - a token in any other string (an attribute, an HTML block) is spliced
+ *     back as text.
+ * Both directive-raw forms serialize byte-exact.
+ *
+ * The same pass ESCAPES template openers the converter did not emit:
+ * `headers={{ … }}` from a leaked component, a JSON sample in prose. In a
+ * template page they would be syntax; escaped as `{{ "{{" }}` they print
+ * as before. Code (blocks and inline spans) is never touched: template
+ * pages keep it literal anyway.
+ */
+import type { TreeNode } from "@dogsbay/types";
+export declare class DirectiveTable {
+    private readonly directives;
+    /** Register a directive's source; returns the token that stands for it. */
+    token(source: string): string;
+    get size(): number;
+    /** Replace tokens in a string by their directive source. */
+    splice(s: string): string;
+    /** Split a string into text runs and directive sources, in order. */
+    parts(s: string): Array<{
+        text: string;
+    } | {
+        directive: string;
+    }>;
+}
+/** Escape template openers in a plain string (a title) so a template page prints them literally. */
+export declare function escapeStrayOpeners(s: string): string;
+export interface RestoreStats {
+    /** Stray template openers escaped so they print literally. */
+    escaped: number;
+    /**
+     * Directives that ended up inside code (an indented code block, say):
+     * restored as text, where a template page leaves them literal. Reported,
+     * because the reader sees `{% include %}` instead of the partial.
+     */
+    inCode: number;
+    /**
+     * Fenced code blocks whose value directives are real (`<Code
+     * code={props.x} />` → `{{ x }}`): marked `{minja}` so they render.
+     */
+    fencesMarked?: number;
+}
+/**
+ * Restore directive tokens in a converted tree, and escape stray openers.
+ * Mutates the tree IN PLACE (and returns it).
+ */
+export declare function restoreDirectives(tree: TreeNode[], table: DirectiveTable, stats: RestoreStats): TreeNode[];

package/dist/template-tokens.js

@@ -0,0 +1,206 @@
+// ASCII letters and digits only: the Starlight pipeline strips private-use
+// characters (a first version used U+E030/U+E031 and every directive came
+// out as its bare number), and Markdown neither escapes nor splits a word.
+const OPEN = "DBTPLDIRx";
+const CLOSE = "xDBTPLDIR";
+const TOKEN = /DBTPLDIRx(\d+)xDBTPLDIR/g;
+const BRACES_BEFORE_TOKEN = /\{+(?=DBTPLDIRx\d+xDBTPLDIR)/g;
+/**
+ * A template opener that is not a `{#id}` attribute block. The attribute
+ * rule is the build's (format-dogsbay-md ATTRIBUTE_BLOCK_OPEN): `{#` + a
+ * letter or digit in any script, `_` or `-`, reaching `}` with no `#}`.
+ */
+const STRAY_OPENER = /\{\{|\{%|\{#(?!(?=[\p{L}\p{N}_-][^}\n]*\})(?![^}\n]*#\}))/gu;
+export class DirectiveTable {
+    directives = [];
+    /** Register a directive's source; returns the token that stands for it. */
+    token(source) {
+        this.directives.push(source);
+        return `${OPEN}${this.directives.length - 1}${CLOSE}`;
+    }
+    get size() {
+        return this.directives.length;
+    }
+    /** Replace tokens in a string by their directive source. */
+    splice(s) {
+        return s.replace(TOKEN, (_m, i) => this.directives[Number(i)]);
+    }
+    /** Split a string into text runs and directive sources, in order. */
+    parts(s) {
+        const out = [];
+        let last = 0;
+        for (const m of s.matchAll(TOKEN)) {
+            if (m.index > last)
+                out.push({ text: s.slice(last, m.index) });
+            out.push({ directive: this.directives[Number(m[1])] });
+            last = m.index + m[0].length;
+        }
+        if (last < s.length)
+            out.push({ text: s.slice(last) });
+        return out;
+    }
+}
+/** `{{ "{{" }}` — prints the opener literally in a template page. */
+function literal(opener) {
+    return `{{ "${opener}" }}`;
+}
+/** Escape template openers in a plain string (a title) so a template page prints them literally. */
+export function escapeStrayOpeners(s) {
+    return s.replace(STRAY_OPENER, (m) => literal(m));
+}
+const isCode = (type) => typeof type === "string" && /code/.test(type);
+/**
+ * A directive the converter may evaluate inside a fence: a value or a
+ * conditional. An `{% include %}` or `{% with %}` in code is a tag that
+ * landed there by accident (an indented code block), and rendering a
+ * partial INTO code is no better than showing the tag.
+ */
+const FENCE_SAFE = /^(?:\{\{[\s\S]*\}\}|\{%-?\s*(?:if|elif|else|endif)\b[\s\S]*%\})$/;
+/**
+ * Restore directive tokens in a converted tree, and escape stray openers.
+ * Mutates the tree IN PLACE (and returns it).
+ */
+export function restoreDirectives(tree, table, stats) {
+    /**
+     * Braces right before one of our directives join with it: `{"a": {{ x }}`
+     * reads as `{{{ x }}` and `{{% if %}` as an expression opener, and the
+     * page fails to render. Such a run becomes a literal-printing directive
+     * of its own, so every path below treats it like any other directive.
+     */
+    const guard = (s) => s.replace(BRACES_BEFORE_TOKEN, (m) => table.token(`{{ "${m}" }}`));
+    const escapeString = (raw) => {
+        const s = guard(raw);
+        return table.splice(s.replace(STRAY_OPENER, (m) => {
+            stats.escaped++;
+            return literal(m);
+        }));
+    };
+    /** Inline array → inline array with tokens as directive-raw-inline and stray openers escaped. */
+    const inline = (nodes) => {
+        const out = [];
+        for (const n of nodes) {
+            const node = n;
+            if (isCode(node.type)) {
+                out.push(code(node));
+                continue;
+            }
+            if (node.type === "text" && typeof node.text === "string") {
+                const marks = { ...(node.bold ? { bold: true } : {}), ...(node.italic ? { italic: true } : {}) };
+                for (const part of table.parts(guard(node.text))) {
+                    if ("directive" in part) {
+                        out.push({ type: "directive-raw-inline", source: part.directive, ...marks });
+                        continue;
+                    }
+                    // Stray openers become literal-printing directives too, so the
+                    // serializer cannot escape their braces into something else.
+                    let last = 0;
+                    for (const m of part.text.matchAll(STRAY_OPENER)) {
+                        if (m.index > last)
+                            out.push({ ...node, text: part.text.slice(last, m.index) });
+                        out.push({ type: "directive-raw-inline", source: literal(m[0]), ...marks });
+                        stats.escaped++;
+                        last = m.index + m[0].length;
+                    }
+                    if (last < part.text.length)
+                        out.push({ ...node, text: part.text.slice(last) });
+                }
+                continue;
+            }
+            out.push(walk(node));
+        }
+        return out;
+    };
+    /** Every other string field: splice tokens back, escape stray openers. */
+    /** Code keeps its text; only our tokens are put back (and counted). */
+    const code = (node) => {
+        if (node.type === "code" && markFence(node))
+            return node;
+        for (const [k, v] of Object.entries(node)) {
+            if (typeof v === "string" && table.parts(v).some((p) => "directive" in p)) {
+                stats.inCode += table.parts(v).filter((p) => "directive" in p).length;
+                node[k] = table.splice(v);
+            }
+            else if (v && typeof v === "object" && !Array.isArray(v))
+                node[k] = code(v);
+        }
+        return node;
+    };
+    /**
+     * A fenced block whose directives are all values or conditionals: splice
+     * them back, escape every other opener (it is code, and must stay
+     * literal), and mark the fence `{minja}` so the template page renders it.
+     */
+    const markFence = (node) => {
+        const props = node.props;
+        const fields = [];
+        if (typeof node.html === "string")
+            fields.push([node, "html"]);
+        if (props && typeof props.code === "string")
+            fields.push([props, "code"]);
+        const directives = fields.flatMap(([o, k]) => table.parts(o[k]).filter((p) => "directive" in p));
+        if (!directives.length || !directives.every((d) => FENCE_SAFE.test(d.directive)))
+            return false;
+        for (const [o, k] of fields) {
+            o[k] = table
+                .parts(guard(o[k]))
+                .map((p) => ("directive" in p ? p.directive : p.text.replace(STRAY_OPENER, (m) => literal(m))))
+                .join("");
+        }
+        node.props = { ...(props ?? {}), minja: true };
+        stats.fencesMarked = (stats.fencesMarked ?? 0) + 1;
+        return true;
+    };
+    const walk = (node) => {
+        if (isCode(node.type))
+            return code(node);
+        for (const [k, v] of Object.entries(node)) {
+            if (k === "type")
+                continue;
+            if (typeof v === "string")
+                node[k] = escapeString(v);
+            else if (Array.isArray(v))
+                node[k] = k === "inline" ? inline(v) : blocks(v);
+            else if (v && typeof v === "object")
+                node[k] = walk(v);
+        }
+        return node;
+    };
+    /** A paragraph whose only content is directive tokens (and whitespace). */
+    const onlyTokens = (node) => {
+        if (node.type !== "paragraph")
+            return undefined;
+        const texts = [];
+        const collect = (n) => {
+            if (Array.isArray(n.inline)) {
+                for (const i of n.inline) {
+                    if (i.type !== "text" || typeof i.text !== "string")
+                        return false;
+                    texts.push(i.text);
+                }
+            }
+            if (Array.isArray(n.children))
+                for (const c of n.children)
+                    if (!collect(c))
+                        return false;
+            return true;
+        };
+        if (!collect(node))
+            return undefined;
+        const joined = texts.join("");
+        if (!joined.trim() || joined.replace(TOKEN, "").trim() !== "")
+            return undefined;
+        return table.splice(joined.trim());
+    };
+    const blocks = (nodes) => nodes.map((n) => {
+        if (!n || typeof n !== "object")
+            return n;
+        const raw = onlyTokens(n);
+        if (raw !== undefined)
+            return { type: "directive-raw", props: { source: raw, kind: "template" } };
+        return walk(n);
+    });
+    // In place: callers hold the array (a returned copy was once ignored,
+    // and every top-level directive paragraph stayed a raw token).
+    tree.splice(0, tree.length, ...blocks(tree));
+    return tree;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dogsbay/format-starlight",
-  "version": "0.2.0-beta.114",
+  "version": "0.2.0-beta.116",
   "description": "Starlight MDX importer for Dogsbay — converts Starlight sources (with JSX components, Render partials, and adapter-specific extensions) into TreeNode[]",
   "type": "module",
   "exports": {
@@ -16,8 +16,10 @@
     "gray-matter": "^4.0.3",
     "jsonc-parser": "^3.3.1",
     "markdown-it": "^14.3.0",
-    "@dogsbay/markdown-it-mdx-jsx": "0.2.0-beta.114",
-    "@dogsbay/types": "0.2.0-beta.114"
+    "markdown-it-footnote": "^4.0.0",
+    "@dogsbay/format-dogsbay-md": "0.2.0-beta.116",
+    "@dogsbay/types": "0.2.0-beta.116",
+    "@dogsbay/markdown-it-mdx-jsx": "0.2.0-beta.116"
   },
   "devDependencies": {
     "@types/iarna__toml": "^2.0.5",