@writedocs/generator 0.4.7 → 0.4.9

package/src/lib/mdx-mintlify.js

@@ -0,0 +1,99 @@
+import { visit } from 'unist-util-visit';
+
+// Remark plugins for Mintlify components whose input can't be read from a
+// component's own props/slot at render time. Registered in
+// astro.config.mjs's markdown processor, next to the other remark plugins.
+
+const TREE_NAMES = new Set(['Tree', 'FileTree']);
+
+function textOf(node) {
+  if (!node) return '';
+  if (node.type === 'text' || node.type === 'inlineCode') return node.value;
+  return (node.children ?? []).map(textOf).join('');
+}
+
+function jsxAttr(name, value) {
+  return { type: 'mdxJsxAttribute', name, value };
+}
+
+function listToTreeNodes(list) {
+  return list.children
+    .filter((item) => item.type === 'listItem')
+    .map((item) => {
+      const label = textOf(item.children.find((c) => c.type === 'paragraph')).trim();
+      const nested = item.children.find((c) => c.type === 'list');
+      const isFolder = label.endsWith('/') || Boolean(nested);
+      const name = label.replace(/\/+$/, '');
+      if (!isFolder) {
+        return { type: 'mdxJsxFlowElement', name: 'Tree.File', attributes: [jsxAttr('name', name)], children: [] };
+      }
+      const children = nested ? listToTreeNodes(nested) : [];
+      const attributes = [jsxAttr('name', name)];
+      // "Folders that contain nested items expand by default" - Mintlify's
+      // rule for the list syntax.
+      if (children.length > 0) attributes.push(jsxAttr('defaultOpen', null));
+      return { type: 'mdxJsxFlowElement', name: 'Tree.Folder', attributes, children };
+    });
+}
+
+/**
+ * Mintlify's Markdown-list form of <Tree>/<FileTree>:
+ *
+ *   <FileTree>
+ *
+ *   - docs/
+ *     - index.mdx
+ *   - docs.config.ts
+ *
+ *   </FileTree>
+ *
+ * A trailing slash or nested items make an entry a folder; formatting in a
+ * name (`code`, **bold**) is dropped. Rewritten here into the same
+ * <Tree.Folder>/<Tree.File> elements the component form uses, so Tree.astro
+ * only ever renders one shape.
+ */
+export function remarkMintlifyTreeLists() {
+  return (tree) => {
+    visit(tree, 'mdxJsxFlowElement', (node) => {
+      if (!TREE_NAMES.has(node.name)) return;
+      if (!node.children.some((c) => c.type === 'list')) return;
+      node.children = node.children.flatMap((c) => (c.type === 'list' ? listToTreeNodes(c) : [c]));
+    });
+  };
+}
+
+function dedent(text) {
+  const lines = text.split('\n');
+  const indents = lines.filter((l) => l.trim()).map((l) => l.match(/^[ \t]*/)[0].length);
+  const min = indents.length ? Math.min(...indents) : 0;
+  return lines
+    .map((l) => l.slice(min))
+    .join('\n')
+    .trim();
+}
+
+/**
+ * Mintlify's <Prompt> copies (and sends to Cursor) the prompt's own source
+ * text - Markdown included, e.g. "- Use second-person voice" - not the
+ * rendered HTML's plain text. A component only receives the rendered slot,
+ * so the source is read here, from the file between the element's first
+ * and last child, and passed down as a hidden `_text` prop.
+ */
+export function remarkMintlifyPromptText() {
+  return (tree, file) => {
+    const source = String(file.value ?? '');
+    visit(tree, 'mdxJsxFlowElement', (node) => {
+      if (node.name !== 'Prompt' || node.children.length === 0) return;
+      if (node.attributes.some((a) => a.type === 'mdxJsxAttribute' && a.name === '_text')) return;
+      const start = node.children[0].position?.start?.offset;
+      const end = node.children[node.children.length - 1].position?.end?.offset;
+      if (typeof start !== 'number' || typeof end !== 'number') return;
+      // Back up to the start of the first child's line, so its own
+      // indentation is part of what dedent() measures - but only over
+      // whitespace, never into the <Prompt ...> tag on the same line.
+      const lineStart = source.lastIndexOf('\n', start - 1) + 1;
+      const from = /^[ \t]*$/.test(source.slice(lineStart, start)) ? lineStart : start;
+      node.attributes.push(jsxAttr('_text', dedent(source.slice(from, end))));
+    });
+  };
+}

package/src/lib/mdx-title-anchor-ids.js

@@ -8,7 +8,11 @@ import { visit } from 'unist-util-visit';
 // (Card, Tabs, AccordionGroup, etc. don't). AccordionGroup is deliberately
 // excluded - it has no `title` of its own, just wraps Accordion children
 // that each get their own entry here independently.
-const TITLED_COMPONENT_NAMES = ['Callout', 'Note', 'Info', 'Tip', 'Warning', 'Danger', 'Check', 'Accordion'];
+const TITLED_COMPONENT_NAMES = ['Callout', 'Note', 'Info', 'Tip', 'Warning', 'Danger', 'Check', 'Accordion', 'Update'];
+
+// The attribute each one's anchor text comes from - `title`, except
+// Mintlify's <Update>, whose anchor is its `label`.
+const ANCHOR_ATTRIBUTE = { Update: 'label' };
 
 // Deliberately the same slugify Callout.astro/Accordion.astro themselves
 // fall back to (see their own comments) - kept as a literal duplicate
@@ -67,8 +71,9 @@ export function remarkTitleAnchorIds() {
     const seen = new Map();
     visit(tree, ['mdxJsxFlowElement', 'mdxJsxTextElement'], (node) => {
       if (!node.name || !TITLED_COMPONENT_NAMES.includes(node.name)) return;
+      const attrName = ANCHOR_ATTRIBUTE[node.name] ?? 'title';
       const titleAttr = node.attributes?.find(
-        (attr) => attr.type === 'mdxJsxAttribute' && attr.name === 'title'
+        (attr) => attr.type === 'mdxJsxAttribute' && attr.name === attrName
       );
       if (!titleAttr || typeof titleAttr.value !== 'string') return;
 

package/src/lib/mdx-unknown-components.js

@@ -0,0 +1,96 @@
+import path from 'node:path';
+import { visit } from 'unist-util-visit';
+import { BUILTIN_COMPONENT_NAMES } from './mdx-inject-builtins.js';
+
+// Which JSX elements in an MDX file are components writedocs can't resolve
+// - neither a built-in nor something the file imports or defines itself.
+// MDX fails the whole build on one ("Expected component `X` to be
+// defined"), which turned every custom component in a migrated Mintlify
+// project into a build-breaking hunt, one component per build. Shared by
+// the build (remarkUnknownComponentFallback below) and `writedocs
+// validate` (lib/content-check.js), so both agree on exactly which
+// elements are unknown. Plain JavaScript so validate can load it with
+// plain Node.
+
+// Dotted built-ins: the root and the parts it has (see
+// src/components/compound.ts). `<Tree.Leaf>` is unknown even though
+// `Tree` isn't.
+const BUILTIN_MEMBERS = {
+  Tree: ['Folder', 'File'],
+  FileTree: ['Folder', 'File'],
+  Color: ['Row', 'Item'],
+  GitHub: ['Repo'],
+};
+
+function localBindings(tree) {
+  const names = new Set();
+  visit(tree, 'mdxjsEsm', (node) => {
+    for (const stmt of node.data?.estree?.body ?? []) {
+      if (stmt.type === 'ImportDeclaration') {
+        for (const s of stmt.specifiers) if (s.local?.name) names.add(s.local.name);
+      } else if (stmt.type === 'ExportNamedDeclaration') {
+        const decl = stmt.declaration;
+        if (decl?.type === 'VariableDeclaration') {
+          for (const d of decl.declarations) if (d.id?.type === 'Identifier') names.add(d.id.name);
+        } else if (decl?.id?.name) {
+          names.add(decl.id.name); // export function X / export class X
+        }
+        for (const s of stmt.specifiers ?? []) if (s.exported?.name) names.add(s.exported.name);
+      }
+    }
+  });
+  return names;
+}
+
+/** Every JSX element in an MDX tree that names a component this file can't
+ *  resolve, in document order: [{ node, name, line, column }]. Lowercase
+ *  names are plain HTML elements and never count. */
+export function findUnknownComponents(tree) {
+  const bound = localBindings(tree);
+  const unknown = [];
+  visit(tree, ['mdxJsxFlowElement', 'mdxJsxTextElement'], (node) => {
+    if (!node.name) return; // a fragment, <>...</>
+    const [root, ...members] = node.name.split('.');
+    if (!/^[A-Z]/.test(root)) return; // <div>, <svg>, <foo.bar> - HTML or a lowercase member expression
+    if (bound.has(root)) return;
+    if (BUILTIN_COMPONENT_NAMES.includes(root)) {
+      if (members.length === 0) return;
+      if (members.length === 1 && BUILTIN_MEMBERS[root]?.includes(members[0])) return;
+    }
+    unknown.push({
+      node,
+      name: node.name,
+      line: node.position?.start?.line,
+      column: node.position?.start?.column,
+    });
+  });
+  return unknown;
+}
+
+/**
+ * Remark plugin: an unknown component renders its children and nothing
+ * else - the element becomes a fragment - with a warning naming the file
+ * and line, instead of failing the build. The site still builds and reads
+ * sensibly (the component's text is still there), and the warning, plus
+ * `writedocs validate`, say what to fix.
+ */
+export function remarkUnknownComponentFallback() {
+  return (tree, file) => {
+    const found = findUnknownComponents(tree);
+    if (found.length === 0) return;
+    const contentDir = process.env.WRITEDOCS_CONTENT_DIR;
+    const filePath = file.path
+      ? contentDir
+        ? path.relative(contentDir, file.path).split(path.sep).join('/')
+        : file.path
+      : '(unknown file)';
+    for (const { node, name, line } of found) {
+      console.warn(
+        `[writedocs] ${filePath}${line ? `:${line}` : ''} - unknown component <${name}>, showing only its content. ` +
+          'Remove it, or define it in a snippet.'
+      );
+      node.name = null;
+      node.attributes = [];
+    }
+  };
+}

package/src/lib/pages.js

@@ -0,0 +1,78 @@
+// Which files in a content directory are pages. Plain JavaScript, not
+// TypeScript, so `writedocs validate` can load it with plain Node (Node
+// refuses to strip types under node_modules - see lib/icons.js's comment);
+// lib/config.ts re-exports findAllPages() for everything inside Astro.
+import fs from 'node:fs';
+import path from 'node:path';
+import matter from 'gray-matter';
+
+// Top-level folders of a content directory that are never scanned for
+// pages - build output, dependencies, and static assets.
+export const EXCLUDED_TOP_LEVEL_DIRS = new Set(['node_modules', 'dist', '.astro', '.writedocs', 'public', '.git']);
+
+/** Recursively finds every .md/.mdx file under `contentDir` that has a
+ *  frontmatter block, skipping the handful of build/dependency
+ *  directories a real content directory tends to also contain (see
+ *  EXCLUDED_TOP_LEVEL_DIRS above) - docs/ is scanned exactly like any
+ *  other folder, no special-casing. Returns POSIX-relative paths (from
+ *  `contentDir`) suitable to hand straight to Astro's `glob()` loader as
+ *  a literal `pattern` array - see content.config.ts's `pages`
+ *  collection, and [...slug].astro, which needs the identical list to
+ *  decide whether calling `getCollection('pages')` is worth doing at all
+ *  (see content.config.ts's own comment on why an empty collection still
+ *  needs to exist, just backed by a no-op loader, to avoid Astro's "does
+ *  not exist or is empty" warning). Also reused directly by
+ *  astro.config.mjs's noindex/sitemap scan and by `writedocs validate`, so
+ *  both always see exactly the same file set that actually becomes a page.
+ *  Synchronous and re-run from scratch wherever it's called rather than
+ *  cached and shared across modules - cheap enough in practice (a docs
+ *  site's own file count) not to matter. */
+export function findAllPages(contentDir) {
+  const results = [];
+  function walk(dir, relBase) {
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const rel = relBase ? `${relBase}/${entry.name}` : entry.name;
+      const abs = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (relBase === '' && EXCLUDED_TOP_LEVEL_DIRS.has(entry.name)) continue;
+        walk(abs, rel);
+        continue;
+      }
+      if (!entry.isFile() || !/\.mdx?$/i.test(entry.name)) continue;
+      let raw;
+      try {
+        raw = fs.readFileSync(abs, 'utf-8');
+      } catch {
+        continue;
+      }
+      let data;
+      try {
+        ({ data } = matter(raw));
+      } catch {
+        // Frontmatter that isn't valid YAML: still a page (it opens a
+        // frontmatter block), so the error surfaces with this file's name -
+        // from `writedocs validate`, or Astro's own loader in the build -
+        // instead of as a bare YAML error thrown from here.
+        if (/^---\r?\n/.test(raw)) results.push(rel);
+        continue;
+      }
+      if (Object.keys(data).length === 0) continue; // no frontmatter at all - not a page
+      results.push(rel);
+    }
+  }
+  walk(contentDir, '');
+  return results;
+}
+
+/** The id writedocs.json's navigation refers to a page by: its path from
+ *  the content directory, without the extension or a trailing `/index`
+ *  (same algorithm as fileIdForEntry() in lib/config.ts). */
+export function fileIdForPath(relativePath) {
+  return relativePath.replace(/\.mdx?$/i, '').replace(/\/index$/, '');
+}

package/src/lib/visibility.js

@@ -0,0 +1,29 @@
+// Mintlify's <Visibility for="humans|agents"> in the page source text that
+// writedocs serves to agents - the per-page `.md` route and llms-full.txt,
+// which both hand out a page's raw MDX body rather than its rendered HTML.
+// The rendered site shows `humans` content and hides `agents` content (see
+// Visibility.astro); the text versions do the opposite: `humans` blocks are
+// removed, and `agents` blocks are unwrapped so only their contents remain.
+// A <Visibility> with no `for` counts as `humans`, like the component.
+//
+// Works on source text, not an AST, since that's what these routes serve.
+// Nested <Visibility> blocks aren't supported (Mintlify doesn't document
+// nesting either) - the regex pairs each opening tag with the first
+// closing tag after it.
+
+const VISIBILITY_BLOCK_RE =
+  /<Visibility(?:\s+for\s*=\s*(?:"(humans|agents)"|'(humans|agents)'|\{\s*["'](humans|agents)["']\s*\}))?\s*>([\s\S]*?)<\/Visibility>/g;
+
+function dedent(text) {
+  const lines = text.replace(/^\n+|\s+$/g, '').split('\n');
+  const indents = lines.filter((l) => l.trim()).map((l) => l.match(/^[ \t]*/)[0].length);
+  const min = indents.length ? Math.min(...indents) : 0;
+  return lines.map((l) => l.slice(min)).join('\n');
+}
+
+export function applyVisibilityForAgents(body) {
+  return body.replace(VISIBILITY_BLOCK_RE, (_, dq, sq, expr, inner) => {
+    const audience = dq ?? sq ?? expr ?? 'humans';
+    return audience === 'agents' ? dedent(inner) : '';
+  });
+}

package/src/pages/[...slug].astro

@@ -56,6 +56,13 @@ import ParamField from "../components/ParamField.astro";
 import ResponseField from "../components/ResponseField.astro";
 import Columns from "../components/Columns.astro";
 import Tooltip from "../components/Tooltip.astro";
+import Update from "../components/Update.astro";
+import Tile from "../components/Tile.astro";
+import Panel from "../components/Panel.astro";
+import Prompt from "../components/Prompt.astro";
+import View from "../components/View.astro";
+import Visibility from "../components/Visibility.astro";
+import { Tree, FileTree, Color, GitHub } from "../components/compound";
 import ApiPlayground from "../components/ApiPlayground.astro";
 import ApiReferencePanel from "../components/ApiReferencePanel.astro";
 
@@ -438,6 +445,16 @@ const components = {
   ResponseField,
   Columns,
   Tooltip,
+  Update,
+  Tile,
+  Panel,
+  Prompt,
+  View,
+  Visibility,
+  Tree,
+  FileTree,
+  Color,
+  GitHub,
 };
 ---
 
@@ -727,7 +744,12 @@ const components = {
     const tocCol = document.querySelector<HTMLElement>(".wd-toc-col");
     if (!tocCol) return; // no toc column on this page (mode !== 'default') - nothing to dock into
 
-    const panels = Array.from(root.querySelectorAll<HTMLElement>(".wd-example-panel"));
+    // Top-level panels only: Mintlify's <Panel> (Panel.astro) is itself a
+    // .wd-example-panel, and a RequestExample/ResponseExample inside it
+    // moves with it rather than on its own.
+    const panels = Array.from(root.querySelectorAll<HTMLElement>(".wd-example-panel")).filter(
+      (panel) => !panel.parentElement?.closest(".wd-example-panel"),
+    );
     if (panels.length === 0) return;
     if (tocCol.dataset.wdExampleInit) return;
     tocCol.dataset.wdExampleInit = "true";

package/src/pages/[...slug].md.ts

@@ -4,6 +4,7 @@ import path from 'node:path';
 import type { APIRoute } from 'astro';
 import { loadDocsConfig, normalizeEntryId, findAllPages } from '../lib/config';
 import { writedocsTempDir } from '../lib/writedocs-temp-dir.js';
+import { applyVisibilityForAgents } from '../lib/visibility.js';
 
 // The raw-Markdown twin of [...slug].astro: every content page is also
 // reachable at the exact same slug with a literal ".md" suffix (e.g.
@@ -73,7 +74,9 @@ export const GET: APIRoute = ({ props }) => {
   // it here means this route needs no separate file read/parse of its
   // own, and always matches exactly what [...slug].astro rendered from
   // (same entry, same collection query).
-  const body = entry.body ?? '';
+  // Mintlify's <Visibility>: this is the agents' view of the page - see
+  // lib/visibility.js.
+  const body = applyVisibilityForAgents(entry.body ?? '');
   const markdown = `# ${entry.data.title}\n\n${body}`;
   return new Response(markdown, {
     headers: { 'Content-Type': 'text/markdown; charset=utf-8' },

package/src/pages/llms-full.txt.ts

@@ -4,6 +4,7 @@ import path from 'node:path';
 import type { APIRoute } from 'astro';
 import { loadDocsConfig, normalizeEntryId, findAllPages } from '../lib/config';
 import { writedocsTempDir } from '../lib/writedocs-temp-dir.js';
+import { applyVisibilityForAgents } from '../lib/visibility.js';
 
 // The "everything, concatenated" half of the llms.txt pair - see
 // llms.txt.ts (right next to this file) for the lightweight index half,
@@ -53,7 +54,8 @@ export const GET: APIRoute = async () => {
     .filter((entry) => !entry.data.url)
     .map((entry: DocsEntry) => {
       const slug = normalizeEntryId(entry.id);
-      const body = entry.body ?? '';
+      // Agents' view of <Visibility> blocks, same as the .md route.
+      const body = applyVisibilityForAgents(entry.body ?? '');
       return { slug, title: entry.data.title, body };
     })
     .sort((a, b) => a.slug.localeCompare(b.slug));