@dogsbay/format-starlight 0.2.0-beta.0

package/dist/parse-mdx.js

@@ -0,0 +1,554 @@
+/**
+ * Starlight MDX parser — v2 using markdown-it-mdx-jsx plugin.
+ *
+ * Replaces the regex-based segmentMdx approach with proper JSX token handling.
+ * The markdown-it-mdx-jsx plugin emits jsx_open/jsx_close/jsx_self_closing
+ * tokens that are handled alongside standard markdown tokens.
+ *
+ * Keeps:
+ * - Directive conversion (:::note → <Aside>)
+ * - Inline transforms (adapter.inlineTransforms)
+ * - Steps unwrapping (steps > ordered-list > list-item → steps > step)
+ * - Card grouping (consecutive cards → cards container)
+ * - The adapter pattern for site-specific component mappings
+ */
+import MarkdownIt from "markdown-it";
+import mdxJsx, { normalizeJsxLines } from "@dogsbay/markdown-it-mdx-jsx";
+import { extractFrontmatter } from "./parse.js";
+// ── Component mapping ───────────────────────────────────
+// Starlight built-in components (always available)
+const STARLIGHT_COMPONENTS = {
+    Tabs: { type: "tabs", container: true },
+    TabItem: {
+        type: "tab",
+        container: true,
+        propsFromAttrs: (attrs) => ({ title: attrs.label || "" }),
+    },
+    Steps: { type: "steps", container: true },
+    Aside: {
+        type: "callout",
+        container: true,
+        propsFromAttrs: (attrs) => ({
+            variant: attrs.type || "note",
+            title: attrs.title,
+        }),
+    },
+    Card: {
+        type: "card",
+        container: true,
+        propsFromAttrs: (attrs) => ({
+            title: attrs.title || "",
+            icon: attrs.icon,
+            href: attrs.href,
+        }),
+    },
+    CardGrid: { type: "cards", container: true },
+    LinkCard: {
+        type: "card",
+        container: false,
+        propsFromAttrs: (attrs) => ({
+            title: attrs.title || "",
+            description: attrs.description,
+            href: attrs.href,
+        }),
+    },
+    LinkButton: {
+        type: "link-button",
+        container: true,
+        propsFromAttrs: (attrs) => ({
+            href: attrs.href || "",
+            variant: attrs.variant,
+        }),
+    },
+    Badge: {
+        type: "badge",
+        container: false,
+        propsFromAttrs: (attrs) => ({
+            text: attrs.text || "",
+            variant: attrs.variant,
+        }),
+    },
+    Code: {
+        type: "code",
+        container: false,
+        propsFromAttrs: (attrs) => ({
+            code: attrs.code || "",
+            lang: attrs.lang || "plaintext",
+            title: attrs.title,
+        }),
+    },
+};
+const CORE_SELF_CLOSING = ["LinkCard", "Badge", "Code"];
+function buildComponentMap(adapter) {
+    const map = { ...STARLIGHT_COMPONENTS };
+    const selfClosing = [...CORE_SELF_CLOSING];
+    if (adapter) {
+        Object.assign(map, adapter.components);
+        for (const tag of adapter.selfClosingTags ?? []) {
+            selfClosing.push(tag);
+        }
+    }
+    return {
+        components: Object.keys(map),
+        selfClosing,
+        map,
+    };
+}
+// ── Pre-processing ──────────────────────────────────────
+function applyInlineTransforms(source, adapter) {
+    if (!adapter?.inlineTransforms)
+        return source;
+    let result = source;
+    for (const [regex, replacement] of adapter.inlineTransforms) {
+        result = result.replace(regex, replacement);
+    }
+    return result;
+}
+/**
+ * Rewrite ~/assets/ Vite alias to relative path.
+ */
+function rewriteAssetPaths(source) {
+    return source.replace(/~\/assets\//g, "assets/");
+}
+/**
+ * Convert Starlight :::note / :::caution / :::tip / :::danger directives
+ * to <Aside> component syntax before parsing.
+ */
+function convertDirectives(source) {
+    const lines = source.split("\n");
+    const result = [];
+    let i = 0;
+    while (i < lines.length) {
+        const line = lines[i];
+        // Match directives with optional leading whitespace (for indented contexts)
+        const directiveMatch = line.match(/^(\s*):::(note|tip|caution|danger|warning|info)(?:\[([^\]]*)\])?\s*$/);
+        if (directiveMatch) {
+            const indent = directiveMatch[1] || "";
+            const variant = directiveMatch[2] === "info" ? "note" : directiveMatch[2];
+            const title = directiveMatch[3] || "";
+            const titleAttr = title ? ` title="${title}"` : "";
+            result.push(`${indent}<Aside type="${variant}"${titleAttr}>`);
+            i++;
+            while (i < lines.length && lines[i]?.trim() !== ":::") {
+                result.push(lines[i]);
+                i++;
+            }
+            result.push(`${indent}</Aside>`);
+            if (i < lines.length)
+                i++; // skip closing :::
+        }
+        else {
+            result.push(line);
+            i++;
+        }
+    }
+    return result.join("\n");
+}
+// ── Token → TreeNode conversion ─────────────────────────
+/**
+ * Convert markdown-it tokens (including jsx_* tokens) to TreeNode[].
+ * This replaces both segmentsToTree and tokensToTree from the old parser.
+ */
+function tokensToTree(tokens, componentMap) {
+    const tree = [];
+    let i = 0;
+    while (i < tokens.length) {
+        const token = tokens[i];
+        // ── JSX tokens ─────────────────────────────────────
+        if (token.type === "jsx_self_closing") {
+            const mapping = componentMap[token.tag];
+            if (mapping) {
+                if (!mapping.strip) {
+                    const attrs = token.meta?.attrs || {};
+                    const props = mapping.propsFromAttrs?.(attrs) || {};
+                    tree.push({ type: mapping.type, props, children: [] });
+                }
+            }
+            i++;
+            continue;
+        }
+        if (token.type === "jsx_open") {
+            const tag = token.tag;
+            const mapping = componentMap[tag];
+            // Collect children until matching jsx_close
+            const children = [];
+            let depth = 1;
+            i++;
+            while (i < tokens.length && depth > 0) {
+                if (tokens[i].type === "jsx_open" && tokens[i].tag === tag) {
+                    depth++;
+                }
+                else if (tokens[i].type === "jsx_close" && tokens[i].tag === tag) {
+                    depth--;
+                    if (depth === 0) {
+                        i++; // skip close token
+                        break;
+                    }
+                }
+                children.push(tokens[i]);
+                i++;
+            }
+            if (mapping) {
+                if (mapping.strip) {
+                    // Strip but keep inner content parsed
+                    if (mapping.container && children.length > 0) {
+                        const innerNodes = tokensToTree(children, componentMap);
+                        tree.push(...innerNodes);
+                    }
+                }
+                else {
+                    const attrs = token.meta?.attrs || {};
+                    const props = mapping.propsFromAttrs?.(attrs) || {};
+                    const childNodes = tokensToTree(children, componentMap);
+                    tree.push({ type: mapping.type, props, children: childNodes });
+                }
+            }
+            else {
+                // Unknown component — parse inner content as markdown
+                const childNodes = tokensToTree(children, componentMap);
+                tree.push(...childNodes);
+            }
+            continue;
+        }
+        if (token.type === "jsx_close") {
+            // Orphan close token — skip (shouldn't happen with proper nesting)
+            i++;
+            continue;
+        }
+        // ── Standard markdown tokens ───────────────────────
+        if (token.type === "heading_open") {
+            const level = parseInt(token.tag.slice(1), 10);
+            const inline = tokens[i + 1];
+            const text = inline?.content || "";
+            const slug = text
+                .toLowerCase()
+                .replace(/[^a-z0-9]+/g, "-")
+                .replace(/^-|-$/g, "");
+            tree.push({
+                type: "heading",
+                props: { level, text, slug },
+                inline: inline ? inlineTokensToInline(inline.children || []) : [],
+            });
+            i += 3; // heading_open, inline, heading_close
+            continue;
+        }
+        if (token.type === "paragraph_open") {
+            const inline = tokens[i + 1];
+            if (inline?.children?.length) {
+                tree.push({
+                    type: "paragraph",
+                    children: [
+                        {
+                            type: "prose",
+                            inline: inlineTokensToInline(inline.children),
+                        },
+                    ],
+                });
+            }
+            i += 3; // paragraph_open, inline, paragraph_close
+            continue;
+        }
+        if (token.type === "fence") {
+            tree.push({
+                type: "code",
+                props: {
+                    code: token.content.replace(/\n$/, ""),
+                    lang: token.info?.split(/\s/)[0] || "plaintext",
+                    title: token.info?.match(/title="([^"]*)"/)?.[1],
+                },
+            });
+            i++;
+            continue;
+        }
+        if (token.type === "code_block") {
+            tree.push({
+                type: "code",
+                props: { code: token.content.replace(/\n$/, ""), lang: "plaintext" },
+            });
+            i++;
+            continue;
+        }
+        if (token.type === "bullet_list_open" ||
+            token.type === "ordered_list_open") {
+            const listType = token.type === "ordered_list_open"
+                ? "ordered-list"
+                : "unordered-list";
+            const start = token.type === "ordered_list_open"
+                ? parseInt(token.attrGet("start") || "1", 10)
+                : undefined;
+            const items = [];
+            i++;
+            while (i < tokens.length &&
+                tokens[i].type !== "bullet_list_close" &&
+                tokens[i].type !== "ordered_list_close") {
+                if (tokens[i].type === "list_item_open") {
+                    i++;
+                    // Collect all tokens until matching list_item_close (track depth for nested lists)
+                    const itemTokens = [];
+                    let liDepth = 1;
+                    while (i < tokens.length && liDepth > 0) {
+                        if (tokens[i].type === "list_item_open")
+                            liDepth++;
+                        if (tokens[i].type === "list_item_close") {
+                            liDepth--;
+                            if (liDepth === 0)
+                                break; // This is OUR close, don't include it
+                        }
+                        itemTokens.push(tokens[i]);
+                        i++;
+                    }
+                    // Recursively parse the list item content using tokensToTree
+                    // This handles nested lists, JSX components, paragraphs, etc.
+                    const itemChildren = tokensToTree(itemTokens, componentMap);
+                    items.push({ type: "list-item", children: itemChildren });
+                    i++; // skip list_item_close
+                }
+                else {
+                    i++;
+                }
+            }
+            i++; // skip list close
+            const listProps = start && start !== 1 ? { start } : undefined;
+            tree.push({ type: listType, props: listProps, children: items });
+            continue;
+        }
+        if (token.type === "table_open") {
+            const tableNode = parseTable(tokens, i);
+            tree.push(tableNode.node);
+            i = tableNode.endIndex;
+            continue;
+        }
+        if (token.type === "hr") {
+            tree.push({ type: "thematic-break" });
+            i++;
+            continue;
+        }
+        if (token.type === "blockquote_open") {
+            const children = [];
+            i++;
+            while (i < tokens.length && tokens[i].type !== "blockquote_close") {
+                const nested = tokensToTree(tokens.slice(i, i + 1), componentMap);
+                children.push(...nested);
+                i++;
+            }
+            i++; // skip close
+            tree.push({ type: "blockquote", children });
+            continue;
+        }
+        if (token.type === "html_block") {
+            // Check if this is a leftover JSX component that wasn't caught
+            const trimmed = token.content.trim();
+            if (trimmed.match(/^<[A-Z]/)) {
+                // Likely a JSX component — skip it (already handled or unrecognized)
+                i++;
+                continue;
+            }
+            tree.push({ type: "html", html: token.content });
+            i++;
+            continue;
+        }
+        // Skip unhandled tokens
+        i++;
+    }
+    return tree;
+}
+function parseTable(tokens, start) {
+    const rows = [];
+    let i = start + 1; // skip table_open
+    let inHead = false;
+    while (i < tokens.length && tokens[i].type !== "table_close") {
+        if (tokens[i].type === "thead_open") {
+            inHead = true;
+            i++;
+            continue;
+        }
+        if (tokens[i].type === "thead_close") {
+            inHead = false;
+            i++;
+            continue;
+        }
+        if (tokens[i].type === "tbody_open" ||
+            tokens[i].type === "tbody_close") {
+            i++;
+            continue;
+        }
+        if (tokens[i].type === "tr_open") {
+            const cells = [];
+            i++;
+            while (i < tokens.length && tokens[i].type !== "tr_close") {
+                if (tokens[i].type === "th_open" || tokens[i].type === "td_open") {
+                    const cellType = inHead ? "th" : "td";
+                    const inline = tokens[i + 1];
+                    const inlineNodes = inline?.type === "inline"
+                        ? inlineTokensToInline(inline.children || [])
+                        : [];
+                    cells.push({
+                        type: cellType,
+                        children: [{ type: "prose", inline: inlineNodes }],
+                    });
+                    i += 3; // open, inline, close
+                }
+                else {
+                    i++;
+                }
+            }
+            rows.push({ type: "tr", children: cells });
+            i++; // skip tr_close
+            continue;
+        }
+        i++;
+    }
+    return {
+        node: { type: "table", children: rows },
+        endIndex: i + 1,
+    };
+}
+// ── Inline token conversion ─────────────────────────────
+function inlineTokensToInline(tokens) {
+    const result = [];
+    let bold = false;
+    let italic = false;
+    let inLink = false;
+    let linkHref = "";
+    let linkChildren = [];
+    const push = (node) => {
+        if (inLink)
+            linkChildren.push(node);
+        else
+            result.push(node);
+    };
+    for (const token of tokens) {
+        if (token.type === "text") {
+            push({ type: "text", text: token.content, bold, italic });
+        }
+        else if (token.type === "code_inline") {
+            push({ type: "code", text: token.content });
+        }
+        else if (token.type === "softbreak" || token.type === "hardbreak") {
+            push({ type: "text", text: "\n" });
+        }
+        else if (token.type === "strong_open") {
+            bold = true;
+        }
+        else if (token.type === "strong_close") {
+            bold = false;
+        }
+        else if (token.type === "em_open") {
+            italic = true;
+        }
+        else if (token.type === "em_close") {
+            italic = false;
+        }
+        else if (token.type === "link_open") {
+            linkHref = token.attrGet("href") || "";
+            linkChildren = [];
+            inLink = true;
+        }
+        else if (token.type === "link_close") {
+            if (inLink) {
+                result.push({
+                    type: "link",
+                    href: linkHref,
+                    children: linkChildren,
+                });
+                inLink = false;
+                linkHref = "";
+                linkChildren = [];
+            }
+        }
+        else if (token.type === "image") {
+            push({
+                type: "image",
+                src: token.attrGet("src") || "",
+                alt: token.content || token.attrGet("alt") || "",
+            });
+        }
+        else if (token.type === "html_inline") {
+            push({ type: "text", text: token.content });
+        }
+        else if (token.type === "jsx_inline_self_closing" ||
+            token.type === "jsx_inline_open" ||
+            token.type === "jsx_inline_close") {
+            // Inline JSX tokens — skip (stripped components like <Render />)
+        }
+    }
+    return result;
+}
+// ── Post-processing ─────────────────────────────────────
+function unwrapStepsOrderedList(nodes) {
+    return nodes.map((node) => {
+        if (node.children) {
+            node.children = unwrapStepsOrderedList(node.children);
+        }
+        if (node.type !== "steps")
+            return node;
+        const children = node.children ?? [];
+        if (children.length === 1 &&
+            children[0].type === "ordered-list" &&
+            children[0].children) {
+            node.children = children[0].children.map((listItem) => ({
+                type: "step",
+                props: {},
+                children: listItem.children,
+            }));
+        }
+        return node;
+    });
+}
+function groupConsecutiveCards(tree) {
+    const result = [];
+    let cardBuf = [];
+    function flushCards() {
+        if (cardBuf.length > 0) {
+            result.push({ type: "cards", children: [...cardBuf] });
+            cardBuf = [];
+        }
+    }
+    for (const node of tree) {
+        if (node.type === "card") {
+            cardBuf.push(node);
+        }
+        else {
+            flushCards();
+            result.push(node);
+        }
+    }
+    flushCards();
+    return result;
+}
+export function starlightToTree(source, options) {
+    const adapter = options?.adapter;
+    const { components, selfClosing, map: componentMap } = buildComponentMap(adapter);
+    // Strip frontmatter block before parsing. Without this, markdown-it sees
+    // `---...---` as thematic-break + paragraph + thematic-break, which leaks the
+    // YAML into the rendered body. Frontmatter is extracted separately at the
+    // CLI level; here we only need to remove it from the content stream.
+    const { body } = extractFrontmatter(source);
+    // Pre-process source
+    // Normalize tabs to spaces — MDX source often mixes tabs and spaces
+    let cleaned = body.replace(/\t/g, "    ");
+    // Resolve include/partial tags via adapter (e.g. Cloudflare's <Render>)
+    if (adapter?.resolveIncludes) {
+        cleaned = adapter.resolveIncludes(cleaned, options?.partialsDir);
+    }
+    cleaned = applyInlineTransforms(cleaned, adapter);
+    cleaned = rewriteAssetPaths(cleaned);
+    cleaned = convertDirectives(cleaned);
+    // Strip raw HTML div wrappers
+    cleaned = cleaned.replace(/<div[^>]*>\s*\n?/g, "");
+    cleaned = cleaned.replace(/<\/div>\s*\n?/g, "");
+    // Normalize JSX lines — split multi-tag lines for the line-based parser
+    cleaned = normalizeJsxLines(cleaned);
+    // Create markdown-it instance with JSX plugin
+    // The plugin handles import stripping, JSX comments, and component tokenization
+    const md = new MarkdownIt({ html: true, linkify: true }).use(mdxJsx, {
+        components,
+        selfClosing,
+    });
+    // Parse into tokens (including jsx_* tokens)
+    const tokens = md.parse(cleaned, {});
+    // Convert tokens to tree
+    const tree = tokensToTree(tokens, componentMap);
+    const unwrapped = unwrapStepsOrderedList(tree);
+    return groupConsecutiveCards(unwrapped);
+}

package/dist/parse.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Starlight MDX parser — parse Starlight source .mdx files into TreeNode[].
+ *
+ * Handles:
+ *   - YAML frontmatter extraction (title, sidebar.order, sidebar.label)
+ *   - Import line stripping (import { ... } from "~/components")
+ *   - Starlight directive admonitions (:::note, :::caution, :::tip, :::danger)
+ *   - Starlight JSX components (Tabs, TabItem, Steps, Card, etc.)
+ *   - Cloudflare custom components (Feature, Plan, Description, etc.) via customization layer
+ *   - Standard CommonMark (headings, paragraphs, lists, tables, code, links)
+ */
+import type { TreeNode } from "@dogsbay/types";
+import type { SiteAdapter } from "./adapter.js";
+export interface StarlightFrontmatter {
+    title: string;
+    sidebarLabel?: string;
+    sidebarOrder?: number;
+    sidebarHidden?: boolean;
+    /** sidebar.group.hideIndex — when true, index page is not shown as child in group */
+    hideIndex?: boolean;
+    description?: string;
+    contentType?: string;
+}
+export declare function extractFrontmatter(source: string): {
+    frontmatter: StarlightFrontmatter;
+    /** Full YAML as a flat record — includes all fields, not just the typed subset. */
+    raw: Record<string, unknown>;
+    body: string;
+};
+export interface ParseOptions {
+    /** Site-specific adapter for custom component mappings */
+    adapter?: SiteAdapter;
+}
+export declare function starlightToTree(source: string, options?: ParseOptions): TreeNode[];