@dogsbay/format-starlight 0.2.0-beta.0

package/dist/parse.js

@@ -0,0 +1,729 @@
+/**
+ * Starlight MDX parser — parse Starlight source .mdx files into TreeNode[].
+ *
+ * Handles:
+ *   - YAML frontmatter extraction (title, sidebar.order, sidebar.label)
+ *   - Import line stripping (import { ... } from "~/components")
+ *   - Starlight directive admonitions (:::note, :::caution, :::tip, :::danger)
+ *   - Starlight JSX components (Tabs, TabItem, Steps, Card, etc.)
+ *   - Cloudflare custom components (Feature, Plan, Description, etc.) via customization layer
+ *   - Standard CommonMark (headings, paragraphs, lists, tables, code, links)
+ */
+import MarkdownIt from "markdown-it";
+import matter from "gray-matter";
+const md = new MarkdownIt({ html: true, linkify: true });
+export function extractFrontmatter(source) {
+    const lines = source.split("\n");
+    if (lines[0]?.trim() !== "---") {
+        return { frontmatter: { title: "" }, raw: {}, body: source };
+    }
+    let end = 1;
+    while (end < lines.length && lines[end]?.trim() !== "---") {
+        end++;
+    }
+    const yamlBlock = lines.slice(1, end).join("\n");
+    const body = lines.slice(end + 1).join("\n");
+    // Parse YAML via gray-matter for a full record that preserves custom fields.
+    let raw = {};
+    try {
+        const result = matter(source);
+        raw = result.data ?? {};
+    }
+    catch {
+        // Fall back to regex-based extraction below if YAML is malformed.
+    }
+    // Build the typed subset for backwards compatibility. Prefer gray-matter
+    // values when available; otherwise fall back to the legacy regex extraction.
+    const stripQuotes = (s) => s?.replace(/^["']|["']$/g, "");
+    const get = (key) => {
+        const match = yamlBlock.match(new RegExp(`^${key}:\\s*(.+)$`, "m"));
+        return stripQuotes(match?.[1]?.trim());
+    };
+    const getIndented = (parent, child) => {
+        const re = new RegExp(`^${parent}:\\s*\\n(?:[ \\t]+\\S.*\\n)*?[ \\t]+${child}:\\s*(.+)$`, "m");
+        const match = yamlBlock.match(re);
+        return stripQuotes(match?.[1]?.trim());
+    };
+    const rawSidebar = raw.sidebar;
+    const frontmatter = {
+        title: raw.title ?? get("title") ?? "",
+        description: raw.description ?? get("description"),
+        sidebarLabel: rawSidebar?.label ?? getIndented("sidebar", "label"),
+        sidebarOrder: rawSidebar?.order ??
+            (getIndented("sidebar", "order")
+                ? parseInt(getIndented("sidebar", "order"), 10)
+                : undefined),
+        sidebarHidden: rawSidebar?.hidden ??
+            (getIndented("sidebar", "hidden") === "true"),
+        hideIndex: extractHideIndex(raw) ?? yamlBlock.includes("hideIndex: true"),
+        contentType: raw.pcx_content_type ?? get("pcx_content_type"),
+    };
+    return { frontmatter, raw, body };
+}
+function extractHideIndex(raw) {
+    const sidebar = raw.sidebar;
+    const group = sidebar?.group;
+    return group?.hideIndex;
+}
+// ── Component mapping ───────────────────────────────────
+// Starlight built-in components (always available)
+const STARLIGHT_COMPONENTS = {
+    Tabs: { type: "tabs", container: true },
+    TabItem: {
+        type: "tab",
+        container: true,
+        propsFromAttrs: (attrs) => ({ title: attrs.label || "" }),
+    },
+    Steps: { type: "steps", container: true },
+    Aside: {
+        type: "callout",
+        container: true,
+        propsFromAttrs: (attrs) => ({
+            variant: attrs.type || "note",
+            title: attrs.title,
+        }),
+    },
+    Card: {
+        type: "card",
+        container: true,
+        propsFromAttrs: (attrs) => ({
+            title: attrs.title || "",
+            icon: attrs.icon,
+            href: attrs.href,
+        }),
+    },
+    CardGrid: { type: "cards", container: true },
+    LinkCard: {
+        type: "card",
+        container: false,
+        propsFromAttrs: (attrs) => ({
+            title: attrs.title || "",
+            description: attrs.description,
+            href: attrs.href,
+        }),
+    },
+    Badge: {
+        type: "badge",
+        container: false,
+        propsFromAttrs: (attrs) => ({
+            text: attrs.text || "",
+            variant: attrs.variant,
+        }),
+    },
+    Code: {
+        type: "code",
+        container: false,
+        propsFromAttrs: (attrs) => ({
+            code: attrs.code || "",
+            lang: attrs.lang || "plaintext",
+            title: attrs.title,
+        }),
+    },
+};
+// Core self-closing tags (Starlight built-ins)
+const CORE_SELF_CLOSING = new Set(["LinkCard", "Badge", "Code"]);
+/**
+ * Build the component map and self-closing set from core + adapter.
+ */
+function buildComponentMap(adapter) {
+    const map = { ...STARLIGHT_COMPONENTS };
+    const selfClosing = new Set(CORE_SELF_CLOSING);
+    if (adapter) {
+        Object.assign(map, adapter.components);
+        for (const tag of adapter.selfClosingTags ?? []) {
+            selfClosing.add(tag);
+        }
+    }
+    return { map, selfClosing };
+}
+// ── Import stripping ────────────────────────────────────
+function stripImports(source) {
+    // Strip single-line imports: import { X } from "y";
+    // Strip multi-line imports: import {\n  X,\n  Y,\n} from "y";
+    // Rewrite ~/assets/ Vite alias to relative assets/ path
+    return source
+        .replace(/^import\s+\{[^}]*\}\s+from\s+["'][^"']*["']\s*;?\s*$/gm, "")
+        .replace(/^import\s+\{[\s\S]*?\}\s+from\s+["'][^"']*["']\s*;?\s*$/gm, "")
+        .replace(/~\/assets\//g, "assets/")
+        .replace(/\{\/\*[\s\S]*?\*\/\}/g, "")
+        .replace(/\n{3,}/g, "\n\n");
+}
+/**
+ * Apply adapter's inline component transforms to source text.
+ * These handle components that appear within prose (not on their own line)
+ * and must be transformed before the block-level segmenter runs.
+ */
+function applyInlineTransforms(source, adapter) {
+    if (!adapter?.inlineTransforms)
+        return source;
+    let result = source;
+    for (const [regex, replacement] of adapter.inlineTransforms) {
+        result = result.replace(regex, replacement);
+    }
+    return result;
+}
+// ── Directive parsing ───────────────────────────────────
+/**
+ * Convert Starlight :::note / :::caution / :::tip / :::danger directives
+ * to component-style segments before the main parser runs.
+ *
+ * Input:
+ *   :::note[Custom title]
+ *   Content here
+ *   :::
+ *
+ * Output:
+ *   <Aside type="note" title="Custom title">
+ *   Content here
+ *   </Aside>
+ */
+function convertDirectives(source) {
+    const lines = source.split("\n");
+    const result = [];
+    let i = 0;
+    while (i < lines.length) {
+        const line = lines[i];
+        const directiveMatch = line.match(/^:::(note|tip|caution|danger|warning)(?:\[([^\]]*)\])?\s*$/);
+        if (directiveMatch) {
+            const variant = directiveMatch[1];
+            const title = directiveMatch[2] || "";
+            const titleAttr = title ? ` title="${title}"` : "";
+            result.push(`<Aside type="${variant}"${titleAttr}>`);
+            i++;
+            // Collect until closing :::
+            while (i < lines.length && lines[i]?.trim() !== ":::") {
+                result.push(lines[i]);
+                i++;
+            }
+            result.push("</Aside>");
+            if (i < lines.length)
+                i++; // skip closing :::
+        }
+        else {
+            result.push(line);
+            i++;
+        }
+    }
+    return result.join("\n");
+}
+/**
+ * Split source into alternating markdown and component segments.
+ */
+function segmentMdx(source, componentMap, selfClosing) {
+    const segments = [];
+    const lines = source.split("\n");
+    let markdownBuf = [];
+    let inFence = false;
+    let fenceChar = "";
+    let i = 0;
+    function flushMarkdown() {
+        const text = markdownBuf.join("\n").trim();
+        if (text)
+            segments.push({ kind: "markdown", content: text });
+        markdownBuf = [];
+    }
+    while (i < lines.length) {
+        const line = lines[i];
+        // Track fenced code blocks
+        const fenceMatch = line.match(/^(`{3,}|~{3,})/);
+        if (fenceMatch) {
+            if (!inFence) {
+                inFence = true;
+                fenceChar = fenceMatch[1][0];
+            }
+            else if (line.startsWith(fenceChar.repeat(3)) && line.trim().length <= fenceMatch[1].length + 1) {
+                inFence = false;
+            }
+            markdownBuf.push(line);
+            i++;
+            continue;
+        }
+        if (inFence) {
+            markdownBuf.push(line);
+            i++;
+            continue;
+        }
+        // Detect opening component tag (single-line: <Tag attrs> or <Tag attrs />)
+        // Use trimmed line to handle indented tags (e.g. <Details> inside list items)
+        const trimmed = line.trim();
+        const tagMatch = trimmed.match(/^<(\w+)(\s[^>]*)?\s*(\/?)>/);
+        // Also detect multi-line opening tag: <Tag (no closing > on this line)
+        const multiLineOpen = !tagMatch && trimmed.match(/^<(\w+)(\s.*)?$/);
+        if (tagMatch || multiLineOpen) {
+            const tag = tagMatch ? tagMatch[1] : multiLineOpen[1];
+            let attrsStr = tagMatch ? (tagMatch[2] || "") : (multiLineOpen[2] || "");
+            let selfClose = tagMatch ? tagMatch[3] === "/" : false;
+            let endLine = i;
+            // For multi-line opening tags, collect until we find > or />
+            if (multiLineOpen && !tagMatch) {
+                let j = i + 1;
+                while (j < lines.length) {
+                    attrsStr += " " + lines[j].trim();
+                    if (lines[j].trim().endsWith("/>")) {
+                        selfClose = true;
+                        endLine = j;
+                        break;
+                    }
+                    if (lines[j].trim().endsWith(">")) {
+                        endLine = j;
+                        break;
+                    }
+                    j++;
+                }
+                // Clean up attrs — remove trailing > or />
+                attrsStr = attrsStr.replace(/\s*\/?>$/, "");
+            }
+            const mapping = componentMap[tag];
+            if (mapping) {
+                flushMarkdown();
+                const attrs = parseAttrs(attrsStr);
+                if (selfClose || selfClosing.has(tag)) {
+                    segments.push({ kind: "component", content: line, tag, attrs });
+                }
+                else {
+                    // Collect until closing tag
+                    const inner = collectUntilClose(lines, endLine + 1, tag);
+                    segments.push({
+                        kind: "component",
+                        content: line,
+                        tag,
+                        attrs,
+                        inner: inner.content,
+                    });
+                    i = inner.endIndex;
+                    continue;
+                }
+                i = endLine + 1;
+                continue;
+            }
+        }
+        // Check for single-line component: <Tag ...>content</Tag>
+        const singleLineMatch = trimmed.match(/^<(\w+)(\s[^>]*)?>(.+?)<\/\1>\s*$/);
+        if (singleLineMatch) {
+            const tag = singleLineMatch[1];
+            const mapping = componentMap[tag];
+            if (mapping) {
+                flushMarkdown();
+                const attrs = parseAttrs(singleLineMatch[2] || "");
+                segments.push({
+                    kind: "component",
+                    content: line,
+                    tag,
+                    attrs,
+                    inner: singleLineMatch[3],
+                });
+                i++;
+                continue;
+            }
+        }
+        markdownBuf.push(line);
+        i++;
+    }
+    flushMarkdown();
+    return segments;
+}
+function parseAttrs(str) {
+    const attrs = {};
+    const re = /(\w+)(?:=(?:"([^"]*)"|'([^']*)'|\{([^}]*)\}))?/g;
+    let m;
+    while ((m = re.exec(str))) {
+        attrs[m[1]] = m[2] ?? m[3] ?? m[4] ?? "true";
+    }
+    return attrs;
+}
+function collectUntilClose(lines, start, tag) {
+    let depth = 1;
+    let i = start;
+    const collected = [];
+    while (i < lines.length && depth > 0) {
+        const line = lines[i];
+        const trimmed = line.trim();
+        // Opening tag (same tag name) — match after trimming whitespace
+        if (trimmed.match(new RegExp(`^<${tag}(\\s|>)`)) && !trimmed.endsWith("/>")) {
+            depth++;
+        }
+        // Closing tag
+        if (trimmed.match(new RegExp(`^</${tag}\\s*>`))) {
+            depth--;
+            if (depth === 0) {
+                i++;
+                break;
+            }
+        }
+        collected.push(line);
+        i++;
+    }
+    // Dedent: strip common leading whitespace
+    const nonEmpty = collected.filter((l) => l.trim().length > 0);
+    if (nonEmpty.length > 0) {
+        const minIndent = Math.min(...nonEmpty.map((l) => l.match(/^(\s*)/)?.[1].length ?? 0));
+        if (minIndent > 0) {
+            return {
+                content: collected.map((l) => l.slice(minIndent)).join("\n"),
+                endIndex: i,
+            };
+        }
+    }
+    return { content: collected.join("\n"), endIndex: i };
+}
+// ── Segments → TreeNode[] ───────────────────────────────
+function segmentsToTree(segments, componentMap, adapter) {
+    const tree = [];
+    for (const seg of segments) {
+        if (seg.kind === "markdown") {
+            const nodes = markdownToTree(seg.content, adapter);
+            tree.push(...nodes);
+        }
+        else if (seg.kind === "component" && seg.tag) {
+            const mapping = componentMap[seg.tag];
+            if (!mapping)
+                continue;
+            if (mapping.strip) {
+                // For GlossaryTooltip etc — parse inner content as markdown and include it
+                if (seg.inner && mapping.container) {
+                    const innerNodes = markdownToTree(seg.inner, adapter);
+                    tree.push(...innerNodes);
+                }
+                continue;
+            }
+            const props = mapping.propsFromAttrs?.(seg.attrs || {}) || {};
+            const children = seg.inner ? starlightToTree(seg.inner, { adapter }) : [];
+            tree.push({ type: mapping.type, props, children });
+        }
+    }
+    return tree;
+}
+// ── Markdown → TreeNode[] (via markdown-it) ─────────────
+function markdownToTree(source, adapter) {
+    // Apply inline transforms to markdown content too (catches components inside collected inner content)
+    let cleaned = applyInlineTransforms(source, adapter);
+    // Strip any remaining unrecognized self-closing components
+    cleaned = cleaned.replace(/<[A-Z]\w+\s[^>]*\/>\s*\n?/g, "");
+    const tokens = md.parse(cleaned, {});
+    return tokensToTree(tokens);
+}
+function tokensToTree(tokens) {
+    const tree = [];
+    let i = 0;
+    while (i < tokens.length) {
+        const token = tokens[i];
+        if (token.type === "heading_open") {
+            const level = parseInt(token.tag.slice(1), 10);
+            const inline = tokens[i + 1];
+            const text = inline?.content || "";
+            const slug = text
+                .toLowerCase()
+                .replace(/[^a-z0-9]+/g, "-")
+                .replace(/^-|-$/g, "");
+            tree.push({
+                type: "heading",
+                props: { level, text, slug },
+                inline: inline ? inlineTokensToInline(inline.children || []) : [],
+            });
+            i += 3; // heading_open, inline, heading_close
+            continue;
+        }
+        if (token.type === "paragraph_open") {
+            const inline = tokens[i + 1];
+            if (inline?.children?.length) {
+                tree.push({
+                    type: "paragraph",
+                    children: [
+                        {
+                            type: "prose",
+                            inline: inlineTokensToInline(inline.children),
+                        },
+                    ],
+                });
+            }
+            i += 3; // paragraph_open, inline, paragraph_close
+            continue;
+        }
+        if (token.type === "fence") {
+            tree.push({
+                type: "code",
+                props: {
+                    code: token.content.replace(/\n$/, ""),
+                    lang: token.info?.split(/\s/)[0] || "plaintext",
+                    title: token.info?.match(/title="([^"]*)"/)?.[1],
+                },
+            });
+            i++;
+            continue;
+        }
+        if (token.type === "code_block") {
+            tree.push({
+                type: "code",
+                props: { code: token.content.replace(/\n$/, ""), lang: "plaintext" },
+            });
+            i++;
+            continue;
+        }
+        if (token.type === "bullet_list_open" || token.type === "ordered_list_open") {
+            const listType = token.type === "ordered_list_open" ? "ordered-list" : "unordered-list";
+            const items = [];
+            i++;
+            while (i < tokens.length && tokens[i].type !== "bullet_list_close" && tokens[i].type !== "ordered_list_close") {
+                if (tokens[i].type === "list_item_open") {
+                    i++;
+                    const itemChildren = [];
+                    while (i < tokens.length && tokens[i].type !== "list_item_close") {
+                        if (tokens[i].type === "paragraph_open") {
+                            const inline = tokens[i + 1];
+                            if (inline?.children?.length) {
+                                itemChildren.push({
+                                    type: "paragraph",
+                                    children: [
+                                        { type: "prose", inline: inlineTokensToInline(inline.children) },
+                                    ],
+                                });
+                            }
+                            i += 3;
+                        }
+                        else {
+                            // Nested lists, etc.
+                            const nested = tokensToTree([tokens[i]]);
+                            itemChildren.push(...nested);
+                            i++;
+                        }
+                    }
+                    items.push({ type: "list-item", children: itemChildren });
+                    i++; // skip list_item_close
+                }
+                else {
+                    i++;
+                }
+            }
+            i++; // skip list close
+            tree.push({ type: listType, children: items });
+            continue;
+        }
+        if (token.type === "table_open") {
+            const tableNode = parseTable(tokens, i);
+            tree.push(tableNode.node);
+            i = tableNode.endIndex;
+            continue;
+        }
+        if (token.type === "hr") {
+            tree.push({ type: "thematic-break" });
+            i++;
+            continue;
+        }
+        if (token.type === "blockquote_open") {
+            const children = [];
+            i++;
+            while (i < tokens.length && tokens[i].type !== "blockquote_close") {
+                const nested = tokensToTree([tokens[i]]);
+                children.push(...nested);
+                i++;
+            }
+            i++; // skip close
+            tree.push({ type: "blockquote", children });
+            continue;
+        }
+        if (token.type === "html_block") {
+            tree.push({ type: "html", html: token.content });
+            i++;
+            continue;
+        }
+        // Skip unhandled tokens
+        i++;
+    }
+    return tree;
+}
+function parseTable(tokens, start) {
+    const rows = [];
+    let i = start + 1; // skip table_open
+    let inHead = false;
+    while (i < tokens.length && tokens[i].type !== "table_close") {
+        if (tokens[i].type === "thead_open") {
+            inHead = true;
+            i++;
+            continue;
+        }
+        if (tokens[i].type === "thead_close") {
+            inHead = false;
+            i++;
+            continue;
+        }
+        if (tokens[i].type === "tbody_open" || tokens[i].type === "tbody_close") {
+            i++;
+            continue;
+        }
+        if (tokens[i].type === "tr_open") {
+            const cells = [];
+            i++;
+            while (i < tokens.length && tokens[i].type !== "tr_close") {
+                if (tokens[i].type === "th_open" || tokens[i].type === "td_open") {
+                    const cellType = inHead ? "th" : "td";
+                    const inline = tokens[i + 1];
+                    const inlineNodes = inline?.type === "inline"
+                        ? inlineTokensToInline(inline.children || [])
+                        : [];
+                    cells.push({
+                        type: cellType,
+                        children: [{ type: "prose", inline: inlineNodes }],
+                    });
+                    i += 3; // open, inline, close
+                }
+                else {
+                    i++;
+                }
+            }
+            rows.push({ type: "tr", children: cells });
+            i++; // skip tr_close
+            continue;
+        }
+        i++;
+    }
+    return {
+        node: { type: "table", children: rows },
+        endIndex: i + 1,
+    };
+}
+// ── Inline token conversion ─────────────────────────────
+function inlineTokensToInline(tokens) {
+    const result = [];
+    let bold = false;
+    let italic = false;
+    let inLink = false;
+    let linkHref = "";
+    let linkChildren = [];
+    const push = (node) => {
+        if (inLink)
+            linkChildren.push(node);
+        else
+            result.push(node);
+    };
+    for (const token of tokens) {
+        if (token.type === "text") {
+            push({ type: "text", text: token.content, bold, italic });
+        }
+        else if (token.type === "code_inline") {
+            push({ type: "code", text: token.content });
+        }
+        else if (token.type === "softbreak" || token.type === "hardbreak") {
+            push({ type: "text", text: "\n" });
+        }
+        else if (token.type === "strong_open") {
+            bold = true;
+        }
+        else if (token.type === "strong_close") {
+            bold = false;
+        }
+        else if (token.type === "em_open") {
+            italic = true;
+        }
+        else if (token.type === "em_close") {
+            italic = false;
+        }
+        else if (token.type === "link_open") {
+            const href = token.attrGet("href") || "";
+            linkHref = href;
+            linkChildren = [];
+            inLink = true;
+        }
+        else if (token.type === "link_close") {
+            if (inLink) {
+                result.push({ type: "link", href: linkHref, children: linkChildren });
+                inLink = false;
+                linkHref = "";
+                linkChildren = [];
+            }
+        }
+        else if (token.type === "image") {
+            result.push({
+                type: "image",
+                src: token.attrGet("src") || "",
+                alt: token.content || token.attrGet("alt") || "",
+            });
+        }
+        else if (token.type === "html_inline") {
+            result.push({ type: "text", text: token.content });
+        }
+    }
+    return result;
+}
+// ── Main entry point ────────────────────────────────────
+/**
+ * Parse Starlight MDX source into TreeNode[].
+ *
+ * Pipeline:
+ *   1. Strip import lines
+ *   2. Convert :::directive admonitions to <Aside> component syntax
+ *   3. Segment into markdown + component blocks
+ *   4. Convert segments to TreeNode[]
+ */
+/**
+ * Group consecutive card nodes into a cards container.
+ * Feature and RelatedProduct components parse as individual card nodes —
+ * this groups them so the serializer renders them in a grid.
+ */
+function groupConsecutiveCards(tree) {
+    const result = [];
+    let cardBuf = [];
+    function flushCards() {
+        if (cardBuf.length > 0) {
+            result.push({ type: "cards", children: [...cardBuf] });
+            cardBuf = [];
+        }
+    }
+    for (const node of tree) {
+        if (node.type === "card") {
+            cardBuf.push(node);
+        }
+        else {
+            flushCards();
+            result.push(node);
+        }
+    }
+    flushCards();
+    return result;
+}
+export function starlightToTree(source, options) {
+    const adapter = options?.adapter;
+    const { map: componentMap, selfClosing } = buildComponentMap(adapter);
+    let cleaned = stripImports(source);
+    cleaned = applyInlineTransforms(cleaned, adapter);
+    cleaned = convertDirectives(cleaned);
+    // Strip raw HTML div wrappers (e.g. <div style="font-size:87%">)
+    cleaned = cleaned.replace(/<div[^>]*>\s*\n?/g, "");
+    cleaned = cleaned.replace(/<\/div>\s*\n?/g, "");
+    const segments = segmentMdx(cleaned, componentMap, selfClosing);
+    const tree = segmentsToTree(segments, componentMap, adapter);
+    const unwrapped = unwrapStepsOrderedList(tree);
+    return groupConsecutiveCards(unwrapped);
+}
+/**
+ * Starlight's <Steps> wraps a markdown ordered list, producing:
+ *   steps > ordered-list > list-item
+ *
+ * Serializers expect:
+ *   steps > step
+ *
+ * This unwraps the ordered-list: each list-item becomes a step node.
+ * Applied recursively so nested steps (e.g. inside tabs) also work.
+ */
+function unwrapStepsOrderedList(nodes) {
+    return nodes.map((node) => {
+        // Recurse into children first
+        if (node.children) {
+            node.children = unwrapStepsOrderedList(node.children);
+        }
+        if (node.type !== "steps")
+            return node;
+        // Check for steps > ordered-list pattern
+        const children = node.children ?? [];
+        if (children.length === 1 &&
+            children[0].type === "ordered-list" &&
+            children[0].children) {
+            // Promote each list-item to a step node
+            node.children = children[0].children.map((listItem) => ({
+                type: "step",
+                props: {},
+                children: listItem.children,
+            }));
+        }
+        return node;
+    });
+}