@dogsbay/format-mdx 0.2.0-beta.14 → 0.2.0-beta.141

package/dist/mintlify-nav.js

@@ -0,0 +1,417 @@
+/**
+ * Mintlify `docs.json` / legacy `mint.json` → NavItem[].
+ *
+ * Mintlify's navigation is polymorphic and recursive. A "navigation node" is an
+ * object that carries one (schema says one; we tolerate several) of these
+ * container arrays, each of which nests another navigation node:
+ *
+ *   languages · versions · tabs · dropdowns · anchors · groups · pages
+ *
+ * plus leaf `pages` entries that are bare path strings, `{ group, pages }`
+ * sub-groups, or `{ page/anchor/..., href }` external links. `versions` and
+ * `languages` wrap any other structure, so a fully-loaded site can be
+ * `versions → languages → tabs → groups → pages`.
+ *
+ * The legacy `mint.json` is flatter: `navigation` is an ARRAY of
+ * `{ group, pages }`, plus top-level `anchors: [{ name, url }]`. Both are
+ * handled here.
+ *
+ * Page labels are NOT in the config — a page is a bare slug, and its
+ * human-readable title lives in the MDX file's frontmatter. We resolve it the
+ * same way the Fumadocs nav builder does (frontmatter `title`, else a humanized
+ * last path segment).
+ */
+import { existsSync, readFileSync, readdirSync, realpathSync } from "node:fs";
+import { basename, dirname, join, sep } from "node:path";
+import YAML from "yaml";
+import { openApiPageKey, openApiRefSlug } from "./mintlify-openapi.js";
+/**
+ * True when `dir` looks like a Mintlify docs source: a `docs.json`/`mint.json`
+ * that actually reads as a Mintlify config, plus md/mdx pages. The config sniff
+ * (a `navigation` key or a `mintlify` `$schema`) keeps an unrelated `docs.json`
+ * — e.g. TypeDoc's output — from being claimed by the mdx importer.
+ */
+export function detectMintlifyDir(dir) {
+    if (!hasMdFiles(dir))
+        return false;
+    const config = readConfig(dir);
+    if (!config)
+        return false;
+    return (config.navigation != null ||
+        Array.isArray(config.anchors) ||
+        (typeof config.$schema === "string" && config.$schema.includes("mintlify")));
+}
+/**
+ * Build NavItem[] from a Mintlify config, or null when no config is present or
+ * it declares no navigation.
+ */
+export function buildMintlifyNav(source, opts = {}) {
+    const config = readConfig(source);
+    if (!config)
+        return null;
+    const ctx = { source, prefix: opts.hrefPrefix ?? "", depth: 0, budget: { spent: 0, warned: false } };
+    const items = [];
+    // `navigation` is an object (docs.json) or an array (legacy mint.json).
+    if (config.navigation != null) {
+        items.push(...navNodeToItems(config.navigation, ctx));
+    }
+    // Legacy top-level anchors sit alongside `navigation`, not inside it.
+    for (const a of objs(config.anchors))
+        items.push(anchorItem(a, ctx));
+    return items.length > 0 ? items : null;
+}
+/** Depth cap for the navigation walk — far above any real config (max seen: 6). */
+const MAX_NAV_DEPTH = 64;
+/**
+ * Total nav nodes one build may visit.
+ *
+ * Depth alone does not bound WORK: a cyclic `$ref` with two children per level
+ * is O(2^depth) — measured 4x per two levels, each node doing a readFileSync —
+ * so a depth cap of 64 still hangs the import forever. The bug this replaced
+ * (an unguarded RangeError) at least TERMINATED. Mintlify's own docs are ~1000
+ * items, so this is ~50x the largest real config.
+ */
+const MAX_NAV_NODES = 50_000;
+function readConfig(source) {
+    for (const name of ["docs.json", "mint.json"]) {
+        const p = join(source, name);
+        if (existsSync(p)) {
+            try {
+                return JSON.parse(readFileSync(p, "utf-8"));
+            }
+            catch {
+                return null;
+            }
+        }
+    }
+    return null;
+}
+/**
+ * A navigation node: an array (legacy nav / a bare pages list) or an object
+ * carrying container arrays. Containers are checked in outer-to-inner order and
+ * concatenated — the schema says one type per level, but concatenating is safe
+ * and tolerant of hand-written configs that mix them.
+ */
+function navNodeToItems(node, ctx) {
+    if (Array.isArray(node))
+        return node.flatMap((e) => entryToItems(e, ctx));
+    if (!node || typeof node !== "object")
+        return [];
+    // Container elements must be objects — a stray null/scalar in an array must
+    // be skipped, not throw (readConfig's contract is "bad config → null nav",
+    // and buildMintlifyNav is called uncaught). `pages` is exempt: entryToItems
+    // already tolerates strings and null.
+    if (++ctx.budget.spent > MAX_NAV_NODES) {
+        if (!ctx.budget.warned) {
+            console.warn(`warning: navigation exceeded ${MAX_NAV_NODES} nodes — the rest was skipped ` +
+                "(a branching $ref cycle looks like this)");
+            ctx.budget.warned = true;
+        }
+        return [];
+    }
+    if (ctx.depth >= MAX_NAV_DEPTH) {
+        console.warn(`warning: navigation nesting exceeded ${MAX_NAV_DEPTH} levels — the rest of ` +
+            "this branch was skipped (a $ref cycle looks like this)");
+        return [];
+    }
+    const inner = { ...ctx, depth: ctx.depth + 1 };
+    const items = [];
+    // `$ref` is resolved at every container level, not just languages: the same
+    // split-file pattern is used for versions and tabs in large configs.
+    for (const l of objs(node.languages))
+        items.push(variantItem(resolveRef(l, inner), "language", inner));
+    for (const v of objs(node.versions))
+        items.push(variantItem(resolveRef(v, inner), "version", inner));
+    for (const p of objs(node.products))
+        items.push(containerItem(resolveRef(p, inner), "product", inner));
+    for (const t of objs(node.tabs))
+        items.push(containerItem(resolveRef(t, inner), "tab", inner));
+    for (const a of objs(node.anchors))
+        items.push(containerItem(resolveRef(a, inner), "anchor", inner));
+    for (const d of objs(node.dropdowns))
+        items.push(containerItem(resolveRef(d, inner), "dropdown", inner));
+    for (const g of objs(node.groups))
+        items.push(...groupItems(resolveRef(g, inner), inner));
+    // `global` holds divisions shown on EVERY page (global anchors/tabs — a
+    // changelog or blog link that persists across sections). Documented in
+    // Mintlify's navigation reference; unhandled it silently dropped those
+    // links. Recursed like any other node, since it just wraps divisions.
+    if (node.global && typeof node.global === "object") {
+        items.push(...navNodeToItems(resolveRef(node.global, inner), inner));
+    }
+    // `menu` holds `{ item, href|groups }` entries — the dropdown under a tab.
+    // Missing it cost Mintlify's own docs their entire "Guides" section: 30 real
+    // pages imported from disk but appeared in no navigation, which is also why
+    // they looked like "pages outside the nav" when auditing coverage.
+    for (const m of objs(node.menu))
+        items.push(menuItem(resolveRef(m, inner), inner));
+    if (Array.isArray(node.pages))
+        items.push(...node.pages.flatMap((p) => entryToItems(p, inner)));
+    return items.filter(Boolean);
+}
+/** Array elements that are non-null objects (skips stray null/scalar entries). */
+function objs(v) {
+    return Array.isArray(v) ? v.filter((e) => e != null && typeof e === "object") : [];
+}
+/**
+ * Resolve a `{ "$ref": "./fr.json" }` node by loading that file.
+ *
+ * Mintlify splits large multi-language configs across files: the `en` entry is
+ * inlined in `docs.json` and the rest are `$ref`s to siblings. Not following
+ * them cost Mintlify's own docs their ENTIRE French, Spanish and Chinese
+ * navigation — three nav entries labelled `language` with no children, while
+ * the fr/es/zh pages imported fine and were simply unreachable.
+ *
+ * Resolved relative to the docs root, like every other Mintlify path. A ref
+ * that cannot be read returns the node untouched, so it degrades to today's
+ * behaviour rather than throwing.
+ *
+ * Two guards, both learned from `mintlify-openapi.ts` closing the same holes:
+ *
+ *  - CONFINEMENT. `docs.json` ships inside the untrusted repo being converted,
+ *    so "the operator chose it" covers which DIRECTORY to convert, not the
+ *    contents of a file in it. `{"$ref":"../../secret.json"}` put an
+ *    out-of-corpus file's contents into the exported navigation as labels and
+ *    hrefs.
+ *  - CYCLES. A self-referential ref recursed until the stack was exhausted,
+ *    producing a nav ~1800 levels deep whose depth varied by machine; it only
+ *    failed to crash because the RangeError happened to land inside the
+ *    try/catch below.
+ */
+function resolveRef(node, ctx) {
+    if (!node || typeof node !== "object" || typeof node.$ref !== "string")
+        return node;
+    const target = join(ctx.source, node.$ref.replace(/^\.\//, "").replace(/^\//, ""));
+    if (!withinSource(target, ctx.source)) {
+        console.warn(`warning: navigation $ref "${node.$ref}" points outside the docs directory — ignored`);
+        return node;
+    }
+    if (!existsSync(target))
+        return node;
+    try {
+        const loaded = JSON.parse(readFileSync(target, "utf-8"));
+        // A ref can point at an ARRAY (redirects.json is a bare list) — spreading
+        // that into an object would turn it into {0:…,1:…}.
+        if (Array.isArray(loaded))
+            return loaded;
+        // Keep any sibling keys the ref node carried (e.g. an explicit `language`).
+        const { $ref: _ref, ...rest } = node;
+        return { ...loaded, ...rest };
+    }
+    catch {
+        return node;
+    }
+}
+/** Is `target` inside the docs root once symlinks are resolved? */
+function withinSource(target, source) {
+    const real = (p) => {
+        try {
+            return realpathSync(p);
+        }
+        catch {
+            const parent = dirname(p);
+            return parent === p ? p : join(real(parent), basename(p));
+        }
+    };
+    const root = real(source);
+    const abs = real(target);
+    return abs === root || abs.startsWith(root + sep);
+}
+/** A single entry inside a `pages` array. */
+function entryToItems(entry, ctx) {
+    if (typeof entry === "string") {
+        // A `METHOD /path` string references an OpenAPI operation generated from the
+        // container's `openapi` spec (ollama's "Endpoints" group) — it is not a
+        // file. Point at the slug the page generator uses (openApiRefSlug) so the
+        // link resolves; treating it as a file slug produced a broken `/POST /x`.
+        const oaKey = openApiPageKey(entry);
+        if (oaKey) {
+            const path = oaKey.slice(oaKey.indexOf(" ") + 1);
+            const slug = openApiRefSlug(path);
+            const last = slug.split("/").pop() || slug;
+            return [{ label: humanize(last), href: `${ctx.prefix}/${slug}` }];
+        }
+        return [pageItem(entry, ctx)];
+    }
+    if (!entry || typeof entry !== "object")
+        return [];
+    // Sub-group: `{ group, pages }`.
+    if (typeof entry.group === "string")
+        return groupItems(entry, ctx);
+    // External or explicit page link: `{ page?, href }`.
+    if (typeof entry.href === "string") {
+        const label = str(entry.page ?? entry.group ?? entry.title) ?? entry.href;
+        return [{ label, href: resolveHref(entry.href, ctx) }];
+    }
+    // A nested navigation node (e.g. a page entry that itself carries groups).
+    const nested = navNodeToItems(entry, ctx);
+    return nested;
+}
+/** `{ group, pages, root?, tag? }` → a group NavItem (empty label = splice up). */
+function groupItems(g, ctx) {
+    const children = navNodeToItems(g, ctx);
+    const label = str(g.group);
+    // A group with an empty/absent name (legacy root group) has no header — its
+    // pages belong at the parent level.
+    if (!label)
+        return children;
+    const item = { label, children };
+    // `root` names a landing page for the group.
+    if (typeof g.root === "string")
+        item.href = pageHref(g.root, ctx);
+    return [item];
+}
+/** tab / dropdown / anchor: labelled by its own key; may be external or nest. */
+function containerItem(node, kind, ctx) {
+    const label = str(node[kind]) ?? str(node.name) ?? kind;
+    const children = navNodeToItems(node, ctx);
+    if (children.length > 0)
+        return { label, children };
+    // Leaf: an external href, or a bare path (legacy anchor `url`).
+    const target = str(node.href) ?? str(node.url);
+    return target ? { label, href: resolveHref(target, ctx) } : { label };
+}
+/** version / language: labelled by its key, tag appended; wraps nested nav. */
+function variantItem(node, kind, ctx) {
+    const base = str(node[kind]) ?? kind;
+    const tag = str(node.tag);
+    const label = tag ? `${base} (${tag})` : base;
+    const children = navNodeToItems(node, ctx);
+    const item = { label };
+    if (children.length > 0)
+        item.children = children;
+    else if (str(node.href))
+        item.href = resolveHref(node.href, ctx);
+    return item;
+}
+/**
+ * Mintlify `redirects` as a source → destination map, `$ref` resolved.
+ *
+ * A docs site accumulates these as pages move, and its OWN pages then link to
+ * the old paths — Mintlify serves the redirect, so the link works and nobody
+ * notices it is stale. On import the redirect does not exist, so those links
+ * 404: measured on Mintlify's own docs, 16 internal links point at a redirect
+ * source, including `/installation` on the front page (→ `/cli/install`).
+ *
+ * `redirects` is itself commonly a `$ref` to a sibling file (121 entries in
+ * `redirects.json` here), so it is resolved the same way as navigation.
+ */
+export function mintlifyRedirects(source) {
+    const config = readConfig(source);
+    if (!config)
+        return {};
+    const ctx = { source, prefix: "", depth: 0, budget: { spent: 0, warned: false } };
+    const raw = config.redirects;
+    const list = Array.isArray(raw)
+        ? raw
+        : (() => {
+            const resolved = resolveRef(raw, ctx);
+            return Array.isArray(resolved)
+                ? resolved
+                : Array.isArray(resolved?.redirects)
+                    ? resolved.redirects
+                    : [];
+        })();
+    const map = {};
+    for (const entry of list) {
+        const from = str(entry?.source);
+        const to = str(entry?.destination);
+        if (from && to)
+            map[from.replace(/\/$/, "")] = to;
+    }
+    return map;
+}
+/** A `menu` entry: `{ item, href? , groups?/pages? }`. */
+function menuItem(node, ctx) {
+    const label = str(node.item) ?? str(node.group) ?? "Menu";
+    const children = navNodeToItems(node, ctx);
+    const item = { label };
+    if (children.length > 0)
+        item.children = children;
+    else if (str(node.href))
+        item.href = resolveHref(node.href, ctx);
+    return item;
+}
+/** Legacy top-level anchor: `{ name, url, icon? }`. */
+function anchorItem(a, ctx) {
+    const label = str(a.name) ?? str(a.anchor) ?? "Anchor";
+    const target = str(a.url) ?? str(a.href);
+    return target ? { label, href: resolveHref(target, ctx) } : { label };
+}
+/** A bare page slug → NavItem, label resolved from frontmatter title. */
+function pageItem(slug, ctx) {
+    return { label: pageLabel(slug, ctx), href: pageHref(slug, ctx) };
+}
+// ── href / label helpers ────────────────────────────────
+function resolveHref(target, ctx) {
+    if (isExternal(target) || target.startsWith("#") || target.startsWith("mailto:"))
+        return target;
+    return pageHref(target, ctx);
+}
+function pageHref(slug, ctx) {
+    const clean = slug.replace(/^\//, "").replace(/\.mdx?$/, "");
+    // An `index` page is routed to its folder root by Astro (root `index.mdx` → `/`,
+    // `foo/index.mdx` → `/foo`), so the nav href must match — linking to `/index`
+    // 404s while the page lives at `/`. Mirrors the Fumadocs nav builder.
+    if (clean === "index")
+        return `${ctx.prefix}/`;
+    return `${ctx.prefix}/${clean.replace(/\/index$/, "")}`;
+}
+function isExternal(s) {
+    return /^https?:\/\//.test(s);
+}
+function pageLabel(slug, ctx) {
+    const clean = slug.replace(/^\//, "").replace(/\.mdx?$/, "");
+    const file = resolvePageFile(ctx.source, clean);
+    if (file) {
+        const title = frontmatterTitle(file);
+        if (title)
+            return title;
+    }
+    const last = clean.split("/").pop() || clean;
+    return humanize(last);
+}
+function resolvePageFile(source, slug) {
+    for (const ext of [".mdx", ".md"]) {
+        const p = join(source, slug + ext);
+        if (existsSync(p))
+            return p;
+        // A folder index: `slug/index.mdx`.
+        const idx = join(source, slug, "index" + ext);
+        if (existsSync(idx))
+            return idx;
+    }
+    return null;
+}
+function frontmatterTitle(file) {
+    try {
+        const src = readFileSync(file, "utf-8");
+        const m = /^---\n([\s\S]*?)\n---/.exec(src);
+        if (m) {
+            const fm = YAML.parse(m[1]);
+            if (fm && typeof fm.title === "string")
+                return fm.title;
+        }
+    }
+    catch {
+        // fall through
+    }
+    return undefined;
+}
+function humanize(name) {
+    return name.replace(/[-_]/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
+}
+function str(v) {
+    if (typeof v !== "string")
+        return undefined;
+    const t = v.trim();
+    return t.length > 0 ? t : undefined;
+}
+function hasMdFiles(dir) {
+    try {
+        return readdirSync(dir).some((e) => (e.endsWith(".md") || e.endsWith(".mdx")) && e !== "llms.txt");
+    }
+    catch {
+        return false;
+    }
+}

package/dist/mintlify-openapi.d.ts

@@ -0,0 +1,108 @@
+import type { TreeNode, ExportPage } from "@dogsbay/types";
+/**
+ * The OpenAPI spec reference declared in a Mintlify config, or null. Reads
+ * `docs.json` / `mint.json` `api.openapi` (string, or first of an array) and the
+ * legacy top-level `openapi`. Only an `http(s)://` value is a URL; everything
+ * else — including a ROOT-relative path like `/api-reference/openapi.json`,
+ * which Mintlify resolves against the docs root, not the filesystem root — is
+ * resolved against `source`.
+ */
+export declare function mintlifyOpenApiSpecRef(source: string): string | null;
+/**
+ * Where a spec reference came from — which decides how far we trust it.
+ *
+ * `config` is `docs.json` / `mint.json`: the operator pointed us at this site,
+ * so its config is a deliberate choice. `page` is a `.mdx` file's own
+ * frontmatter — that is CONTENT, authored by whoever wrote the repo we are
+ * converting, and converting an arbitrary public repo is the whole point of
+ * the migration story. Content does not get to name a network endpoint or a
+ * file outside the corpus.
+ */
+export type SpecRefOrigin = "config" | "page";
+/**
+ * Fetch the raw spec document, unparsed.
+ *
+ * An OpenAPI spec is a CANONICAL SOURCE ARTIFACT, not prose: round-tripping it
+ * through the TreeNode IR is lossy by construction ($ref/component schemas
+ * inline, security schemes and callbacks vanish). Exporters that can consume a
+ * spec directly should pass THIS through rather than rebuild it from endpoint
+ * nodes. Returns null when the spec cannot be read.
+ */
+export declare function readOpenApiSpecSource(specRef: string): Promise<{
+    text: string;
+    filename: string;
+} | null>;
+export interface EndpointMapResult {
+    /** `"<method> <path>"` (lowercase method) → the `endpoint` TreeNode(s). */
+    map: Map<string, TreeNode[]>;
+    /** The same keys → the page title format-openapi gives the operation. */
+    titles?: Map<string, string>;
+    /** Spec load/parse failure message, or null. Lets the caller distinguish a
+     *  failed fetch (blame the spec source) from a genuine no-match (blame the
+     *  page). Without this, every page reads "no matching operation" on a 404. */
+    loadError: string | null;
+}
+/**
+ * Parse a spec into an endpoint map. A fetch/parse error must degrade to blank
+ * pages (never crash the import), but is reported via `loadError` rather than
+ * silently swallowed.
+ */
+export declare function buildMintlifyEndpointMap(specRef: string): Promise<EndpointMapResult>;
+/**
+ * Normalise a page's `openapi` frontmatter value to a map key, or null when the
+ * page is not an OpenAPI page. Accepts `"post /partners/ban"` (method + path);
+ * a leading named-spec token (`"api POST /x"`) is tolerated by taking the last
+ * two whitespace-separated tokens.
+ */
+export declare function openApiPageKey(value: unknown): string | null;
+/**
+ * The spec a PAGE names in its own `openapi:` frontmatter, or null.
+ *
+ * Mintlify lets a page point at a specific spec inline:
+ *
+ *     openapi: "/openapi.json GET /project/update-status/{statusId}"
+ *
+ * `openApiPageKey` already tolerates that leading token (it takes the last two
+ * tokens); this reads the token itself, which nothing did. A site whose
+ * `docs.json` has no `api.openapi` key therefore resolved NO endpoints even
+ * though every page stated exactly which spec to use — measured on Mintlify's
+ * own documentation: 100 pages imported without their endpoint content and 20
+ * of them came out completely blank, next to an `openapi.json` sitting in the
+ * repo root.
+ *
+ * Resolved the same way as a config-declared ref: a leading `/` is the DOCS
+ * root, not the filesystem root.
+ */
+export declare function openApiPageSpecRef(source: string, value: unknown): string | null;
+/**
+ * The vault/site slug for a nav-referenced OpenAPI operation (a `pages` string
+ * of the form `"POST /api/generate"` under a tab/group that declares an
+ * `openapi` spec). The NAV and the GENERATED PAGE both call this, so their
+ * href and slug always agree. Path-based (`/api/generate` → `api/generate`) for
+ * clean URLs; same-path-different-method operations would collide — rare, and
+ * documented (ollama and typical REST doc sets use unique paths per page).
+ */
+export declare function openApiRefSlug(path: string): string;
+/**
+ * Every distinct OpenAPI spec a Mintlify config references — the top-level
+ * `api.openapi` AND any `openapi` declared on a nav container (tab/group).
+ *
+ * Both are real source artifacts. Capturing only the top-level one meant a site
+ * like ollama (whose spec is declared on an "API Reference" tab) fell back to
+ * lossy reconstruction despite having a perfectly good spec on disk.
+ */
+export declare function collectMintlifyOpenApiSpecRefs(source: string, 
+/** Spec refs named by individual pages' `openapi:` frontmatter. */
+pageSpecRefs?: readonly string[]): string[];
+/**
+ * Generate an ExportPage for every OpenAPI operation referenced ONLY by the nav
+ * (a `METHOD /path` `pages` string under a container that declares `openapi`) —
+ * these pages exist in no `.mdx` file (unlike the frontmatter case), so without
+ * this the nav links are dead and the API section is empty. ollama's docs.json
+ * is exactly this shape (an "Endpoints" group of 12 such strings under a tab
+ * with `openapi: /openapi.yaml`).
+ */
+export declare function collectMintlifyOpenApiNavPages(source: string): Promise<{
+    pages: ExportPage[];
+    loadError: string | null;
+}>;