@writedocs/generator 0.4.8 → 0.4.10

package/src/lib/mdx-inline-react.js

@@ -0,0 +1,202 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { parse as acornParse } from 'acorn';
+import { writedocsTempDir } from './writedocs-temp-dir.js';
+import { MINTLIFY_HOOKS } from './mdx-mintlify.js';
+import { BUILTIN_COMPONENT_NAMES } from './mdx-inject-builtins.js';
+
+// Mintlify lets a page define a React component inline and use it right
+// there:
+//
+//   export const Counter = () => {
+//     const [count, setCount] = useState(0)
+//     return <button onClick={() => setCount(count + 1)}>{count}</button>
+//   }
+//
+//   <Counter />
+//
+// Under Astro that can't work as written: JSX inside an .mdx file compiles
+// to Astro elements, not React ones, so a component that calls a hook fails
+// the build ("Objects are not valid as a React child"). A .jsx/.tsx file,
+// on the other hand, is compiled as React and hydrated in the browser (see
+// mdx-auto-hydrate.js) - which is exactly what a snippet is.
+//
+// So an exported component that calls a React hook - or that's passed as a
+// prop to a React snippet, which needs it to be React too - is moved out of
+// the page into a generated .jsx file, and the page imports it from there,
+// the same as if the author had written it as a snippet. It renders on the
+// server and is interactive in the browser, like on Mintlify. Other
+// components stay in the page (they render fine as they are).
+//
+// The generated file also gets: the page's own imports of JavaScript
+// modules (snippets, packages), with relative paths made absolute; the
+// page's other exports, in case the component uses them; and an import of
+// the hooks it calls. It's written to writedocs' temp directory - never into
+// the site's own folder - named by a hash of its content, so an edit in
+// `writedocs dev` produces a new file instead of a stale cached one.
+//
+// writedocs' built-in components are Astro components, which React can't
+// run - inside a moved component each one is a simple React stand-in, so it
+// renders simplified. findInteractiveComponents() reports every such use,
+// and both the build and `writedocs validate` warn about them with
+// simplifiedBuiltinMessage(). ROADMAP.md tracks real React versions.
+
+const HOOK_CALL = new RegExp(`\\b(?:${MINTLIFY_HOOKS.join('|')})\\s*\\(`);
+const HOOK_NAME = new RegExp(`\\b(${MINTLIFY_HOOKS.join('|')})\\s*\\(`, 'g');
+const BUILTIN_TAG = new RegExp(`<(${BUILTIN_COMPONENT_NAMES.join('|')})(?=[\\s/>.])`, 'g');
+
+function declaredNames(stmt) {
+  const decl = stmt.declaration;
+  if (!decl) return [];
+  if (decl.type === 'VariableDeclaration') return decl.declarations.map((d) => d.id?.name).filter(Boolean);
+  return decl.id?.name ? [decl.id.name] : [];
+}
+
+function isJsModule(specifier) {
+  // A snippet or package the component could use - not MDX/Astro, which
+  // aren't importable from React, and not writedocs' own components.
+  if (/\.(mdx|md|astro)$/i.test(specifier)) return false;
+  if (specifier === 'writedocs/components') return false;
+  return true;
+}
+
+/**
+ * The page components that will run as React - and the built-ins inside
+ * them that will render simplified. `source` is the text the tree was
+ * parsed from (statement offsets index into it). Returns:
+ *   statements - every top-level import/export, as { node, stmt }
+ *   extracted  - the ones to move: { node, stmt, names, reason }, where
+ *                reason is 'hooks' or 'prop'
+ *   simplified - { component, builtin, reason, offset } for every built-in
+ *                tag inside an extracted component (offset into `source`)
+ */
+export function findInteractiveComponents(tree, source) {
+  const esmNodes = tree.children.filter((n) => n.type === 'mdxjsEsm' && n.data?.estree);
+  const statements = esmNodes.flatMap((n) => n.data.estree.body.map((stmt) => ({ node: n, stmt })));
+  const text = (stmt) => source.slice(stmt.start, stmt.end);
+
+  // Components imported from .jsx/.tsx - React islands. A page component
+  // handed to one of them as a prop has to be React too, or React gets an
+  // Astro element it can't render.
+  const reactTags = statements
+    .filter(({ stmt }) => stmt.type === 'ImportDeclaration' && /\.[jt]sx$/.test(stmt.source.value))
+    .flatMap(({ stmt }) => stmt.specifiers.map((sp) => sp.local.name));
+  const passedToReact = (name) => reactTags.some((tag) => new RegExp(`<${tag}\\b[^>]*=\\{\\s*${name}\\s*\\}`).test(source));
+
+  // A built-in the page imports under its own name from somewhere else (its
+  // own `Card` snippet, say) isn't writedocs' built-in.
+  const importedNames = new Set(
+    statements
+      .filter(({ stmt }) => stmt.type === 'ImportDeclaration' && isJsModule(stmt.source.value))
+      .flatMap(({ stmt }) => stmt.specifiers.map((sp) => sp.local.name))
+  );
+
+  const extracted = [];
+  const simplified = [];
+  for (const { node, stmt } of statements) {
+    if (stmt.type !== 'ExportNamedDeclaration') continue;
+    const names = declaredNames(stmt).filter((n) => /^[A-Z]/.test(n));
+    if (names.length === 0) continue;
+    const code = text(stmt);
+    const reason = HOOK_CALL.test(code) ? 'hooks' : names.some(passedToReact) ? 'prop' : null;
+    if (!reason) continue;
+    extracted.push({ node, stmt, names, reason });
+    for (const m of code.matchAll(BUILTIN_TAG)) {
+      if (importedNames.has(m[1])) continue;
+      simplified.push({ component: names[0], builtin: m[1], reason, offset: stmt.start + m.index });
+    }
+  }
+  return { statements, extracted, simplified, text };
+}
+
+/** The warning for one simplified built-in - worded once, for the build and
+ *  for `writedocs validate`. Returns { message, suggestion }. */
+export function simplifiedBuiltinMessage({ component, builtin, reason }) {
+  const why = reason === 'hooks' ? 'it uses React hooks' : "it's passed to a React snippet";
+  const result =
+    builtin === 'Icon'
+      ? 'renders nothing'
+      : builtin === 'CodeBlock'
+        ? 'renders as a plain code block, without syntax highlighting'
+        : 'shows only its content';
+  return {
+    message: `<${builtin}> inside <${component}> ${result}: <${component}> runs as a React component (${why}), and writedocs' built-in components can't run inside React.`,
+    suggestion: `Move the <${builtin}> out of <${component}>, or write <${component}> as a .jsx snippet with its own markup.`,
+  };
+}
+
+export function remarkExtractInlineReactComponents() {
+  return (tree, file) => {
+    const source = String(file.value ?? '');
+    const { statements, extracted, simplified, text } = findInteractiveComponents(tree, source);
+    if (extracted.length === 0 || !file.path) return;
+
+    // Same warnings `writedocs validate` gives (lib/content-check.js).
+    const contentDir = process.env.WRITEDOCS_CONTENT_DIR || process.cwd();
+    const relPath = path.relative(contentDir, file.path).split(path.sep).join('/');
+    for (const use of simplified) {
+      const line = source.slice(0, use.offset).split('\n').length;
+      console.warn(`[writedocs] ${relPath}:${line} - ${simplifiedBuiltinMessage(use).message}`);
+    }
+
+    const pageDir = path.dirname(file.path);
+    const absolute = (spec) => (spec.startsWith('.') ? path.resolve(pageDir, spec).split(path.sep).join('/') : spec);
+
+    const imports = statements
+      .filter(({ stmt }) => stmt.type === 'ImportDeclaration' && isJsModule(stmt.source.value))
+      .map(({ stmt }) => text(stmt).replace(stmt.source.raw, JSON.stringify(absolute(stmt.source.value))));
+    const otherExports = statements
+      .filter(({ stmt }) => stmt.type === 'ExportNamedDeclaration' && stmt.declaration && !extracted.some((e) => e.stmt === stmt))
+      .map(({ stmt }) => text(stmt));
+    const componentCode = extracted.map(({ stmt }) => text(stmt));
+    const hooks = [...new Set(componentCode.join('\n').matchAll(HOOK_NAME))].map((m) => m[1]);
+    const alreadyImported = imports.join('\n');
+    const hookImport = [...new Set(hooks)].filter((h) => !new RegExp(`\\b${h}\\b`).test(alreadyImported));
+
+    // A simple React stand-in for each built-in the moved code uses:
+    // CodeBlock a plain (unhighlighted) code block, the rest their children.
+    const movedCode = [...otherExports, ...componentCode].join('\n');
+    const standIns = BUILTIN_COMPONENT_NAMES.filter(
+      (name) => new RegExp(`<${name}[\\s/>.]`).test(movedCode) && !new RegExp(`\\b${name}\\b`).test(alreadyImported)
+    ).map((name) =>
+      name === 'CodeBlock'
+        ? 'const CodeBlock = ({ children, filename }) => (<div className="wd-code-block">{filename ? <div className="wd-code-title">{filename}</div> : null}<pre className="astro-code"><code>{children}</code></pre></div>);'
+        : `const ${name} = ({ children }) => <>{children}</>;`
+    );
+
+    const moduleSource = [
+      '// Generated by writedocs from an inline component in:',
+      `// ${file.path.split(path.sep).join('/')}`,
+      hookImport.length ? `import { ${hookImport.join(', ')} } from 'react';` : '',
+      ...imports,
+      ...standIns,
+      ...otherExports,
+      ...componentCode,
+      '',
+    ].join('\n');
+
+    const hash = crypto.createHash('sha1').update(moduleSource).digest('hex').slice(0, 12);
+    const dir = path.join(writedocsTempDir(contentDir), 'inline-components');
+    const modulePath = path.join(dir, `${path.basename(file.path).replace(/\.mdx$/i, '')}-${hash}.jsx`);
+    if (!fs.existsSync(modulePath)) {
+      fs.mkdirSync(dir, { recursive: true });
+      fs.writeFileSync(modulePath, moduleSource);
+    }
+
+    // Drop the moved declarations from the page, and import them instead.
+    for (const { node, stmt } of extracted) {
+      node.data.estree.body = node.data.estree.body.filter((s) => s !== stmt);
+      node.value = node.value.replace(text(stmt), '');
+    }
+    // Imported for this file's own use, and exported again under the same
+    // names - an .mdx snippet that defines the component is imported by
+    // other pages, which still expect the export to be there.
+    const names = extracted.flatMap(({ stmt }) => declaredNames(stmt));
+    const importSource =
+      `import { ${names.join(', ')} } from ${JSON.stringify(modulePath.split(path.sep).join('/'))};\n` +
+      `export { ${names.join(', ')} };\n`;
+    const estree = acornParse(importSource, { ecmaVersion: 'latest', sourceType: 'module' });
+    tree.children.unshift({ type: 'mdxjsEsm', value: importSource, data: { estree } });
+  };
+}

package/src/lib/mdx-mintlify.js

@@ -1,4 +1,5 @@
 import { visit } from 'unist-util-visit';
+import { parse as acornParse } from 'acorn';
 
 // Remark plugins for Mintlify components whose input can't be read from a
 // component's own props/slot at render time. Registered in
@@ -97,3 +98,67 @@ export function remarkMintlifyPromptText() {
     });
   };
 }
+
+// Mintlify pre-injects these React hooks into every page and snippet, so
+// Mintlify content calls them without importing them.
+export const MINTLIFY_HOOKS = ['useState', 'useEffect', 'useRef', 'useCallback', 'useMemo', 'useContext', 'useReducer'];
+
+/** The hooks `code` calls but doesn't import or declare itself. */
+export function missingHooks(code, alreadyBound = new Set()) {
+  return MINTLIFY_HOOKS.filter((hook) => {
+    if (alreadyBound.has(hook)) return false;
+    if (!new RegExp(`\\b${hook}\\s*\\(`).test(code)) return false;
+    // Imported (import { useState } / import { useState as x }) or declared
+    // (const useState = ...) by the file itself - leave it alone.
+    const declared = new RegExp(`(import\\s*\\{[^}]*\\b${hook}\\b[^}]*\\}\\s*from)|((const|let|var|function)\\s+${hook}\\b)`);
+    return !declared.test(code);
+  });
+}
+
+/**
+ * Mintlify's pre-injected React hooks, for MDX pages: a page whose own
+ * code (an `export const Counter = () => { const [n, setN] = useState(0) }`)
+ * calls a hook it never imported gets `import { ... } from 'react'` added.
+ * Snippet .jsx/.tsx files get the same from mintlifyReactHooksPlugin()
+ * below. (A component defined inside a page renders once, at build time -
+ * only components imported from a .jsx/.tsx snippet are hydrated in the
+ * browser; see mdx-auto-hydrate.js.)
+ */
+export function remarkMintlifyReactHooks() {
+  return (tree) => {
+    const bound = new Set();
+    const code = [];
+    visit(tree, ['mdxjsEsm', 'mdxFlowExpression', 'mdxTextExpression'], (node) => {
+      code.push(node.value ?? '');
+      if (node.type !== 'mdxjsEsm') return;
+      for (const stmt of node.data?.estree?.body ?? []) {
+        if (stmt.type === 'ImportDeclaration') for (const s of stmt.specifiers) if (s.local?.name) bound.add(s.local.name);
+      }
+    });
+    const missing = missingHooks(code.join('\n'), bound);
+    if (missing.length === 0) return;
+    const source = `import { ${missing.join(', ')} } from 'react';\n`;
+    const estree = acornParse(source, { ecmaVersion: 'latest', sourceType: 'module' });
+    tree.children.unshift({ type: 'mdxjsEsm', value: source, data: { estree } });
+  };
+}
+
+/** The same for a site's own .jsx/.tsx files (snippets) - a Vite plugin,
+ *  since those never go through the MDX pipeline. Only files inside the
+ *  site's content directory are touched, never this package's own. */
+export function mintlifyReactHooksPlugin(contentDir) {
+  const root = contentDir.split('\\').join('/').replace(/\/+$/, '') + '/';
+  return {
+    name: 'writedocs:mintlify-react-hooks',
+    enforce: 'pre',
+    transform(code, id) {
+      const file = id.split('?')[0].split('\\').join('/');
+      if (!/\.[jt]sx$/.test(file) || !file.startsWith(root) || file.includes('/node_modules/')) return null;
+      const missing = missingHooks(code);
+      if (missing.length === 0) return null;
+      // After any leading directives ('use client'), which must stay first.
+      const directives = /^(\s*(['"])use [a-z]+\2;?\s*)*/.exec(code)[0];
+      return { code: `${directives}import { ${missing.join(', ')} } from 'react';\n${code.slice(directives.length)}`, map: null };
+    },
+  };
+}

package/src/lib/mdx-substitute-variables.js

@@ -25,6 +25,8 @@ import { visit } from 'unist-util-visit';
 // either, so it round-trips as plain text the same way `{{key}}` would
 // have in an ordinary (non-MDX) Markdown file.
 const PLACEHOLDER = /\[\[\s*([\w.-]+)\s*\]\]/g;
+// The inside of a `{{key}}` expression, as MDX parsed it: `{key}`.
+const MINTLIFY_PLACEHOLDER = /^\s*\{\s*([A-Za-z_$][\w$]*)\s*\}\s*$/;
 
 /**
  * A remark plugin that replaces `[[key]]` placeholders in a page's prose
@@ -62,5 +64,20 @@ export function remarkSubstituteVariables(variables) {
         Object.prototype.hasOwnProperty.call(variables, key) ? variables[key] : match
       );
     });
+
+    // Mintlify's syntax for the same thing, `{{key}}`. MDX has already
+    // parsed that as a JavaScript expression by the time this runs - an
+    // object literal `{key}` inside `{...}` - so it's matched as an
+    // expression node, not text, and replaced with the value as text. Only
+    // for keys the site defines; any other expression is left alone. (A key
+    // with a hyphen isn't valid JavaScript there, so MDX rejects
+    // `{{my-key}}` before this runs - `writedocs convert` warns about those.)
+    visit(tree, ['mdxTextExpression', 'mdxFlowExpression'], (node, index, parent) => {
+      const key = MINTLIFY_PLACEHOLDER.exec(node.value ?? '')?.[1];
+      if (!key || !parent || index === undefined) return;
+      if (!Object.prototype.hasOwnProperty.call(variables, key)) return;
+      const text = { type: 'text', value: String(variables[key]) };
+      parent.children[index] = node.type === 'mdxFlowExpression' ? { type: 'paragraph', children: [text] } : text;
+    });
   };
 }

package/src/lib/mdx-unknown-components.js

@@ -0,0 +1,149 @@
+import path from 'node:path';
+import { visit } from 'unist-util-visit';
+import { parse as acornParse } from 'acorn';
+import { BUILTIN_COMPONENT_NAMES } from './mdx-inject-builtins.js';
+
+// Which JSX elements in an MDX file are components writedocs can't resolve
+// - neither a built-in nor something the file imports or defines itself.
+// MDX fails the whole build on one ("Expected component `X` to be
+// defined"), which turned every custom component in a migrated Mintlify
+// project into a build-breaking hunt, one component per build. Shared by
+// the build (remarkUnknownComponentFallback below) and `writedocs
+// validate` (lib/content-check.js), so both agree on exactly which
+// elements are unknown. Plain JavaScript so validate can load it with
+// plain Node.
+
+// Dotted built-ins: the root and the parts it has (see
+// src/components/compound.ts). `<Tree.Leaf>` is unknown even though
+// `Tree` isn't.
+const BUILTIN_MEMBERS = {
+  Tree: ['Folder', 'File'],
+  FileTree: ['Folder', 'File'],
+  Color: ['Row', 'Item'],
+  GitHub: ['Repo'],
+};
+
+function localBindings(tree) {
+  const names = new Set();
+  visit(tree, 'mdxjsEsm', (node) => {
+    for (const stmt of node.data?.estree?.body ?? []) {
+      if (stmt.type === 'ImportDeclaration') {
+        for (const s of stmt.specifiers) if (s.local?.name) names.add(s.local.name);
+      } else if (stmt.type === 'ExportNamedDeclaration') {
+        const decl = stmt.declaration;
+        if (decl?.type === 'VariableDeclaration') {
+          for (const d of decl.declarations) if (d.id?.type === 'Identifier') names.add(d.id.name);
+        } else if (decl?.id?.name) {
+          names.add(decl.id.name); // export function X / export class X
+        }
+        for (const s of stmt.specifiers ?? []) if (s.exported?.name) names.add(s.exported.name);
+      }
+    }
+  });
+  return names;
+}
+
+/** Every JSX element in an MDX tree that names a component this file can't
+ *  resolve, in document order: [{ node, name, line, column }]. Lowercase
+ *  names are plain HTML elements and never count. */
+export function findUnknownComponents(tree) {
+  const bound = localBindings(tree);
+  const unknown = [];
+  visit(tree, ['mdxJsxFlowElement', 'mdxJsxTextElement'], (node) => {
+    if (!node.name) return; // a fragment, <>...</>
+    const [root, ...members] = node.name.split('.');
+    if (!/^[A-Z]/.test(root)) return; // <div>, <svg>, <foo.bar> - HTML or a lowercase member expression
+    if (bound.has(root)) return;
+    if (BUILTIN_COMPONENT_NAMES.includes(root)) {
+      if (members.length === 0) return;
+      if (members.length === 1 && BUILTIN_MEMBERS[root]?.includes(members[0])) return;
+    }
+    unknown.push({
+      node,
+      name: node.name,
+      line: node.position?.start?.line,
+      column: node.position?.start?.column,
+    });
+  });
+  unknown.push(...findUnknownInCode(tree, bound));
+  return unknown;
+}
+
+/** The same, for JSX inside the page's own code - `export const X = () =>
+ *  <Widget />` or a `{...}` expression - which isn't part of the content
+ *  tree. Matched by text, so a name counts as known when the code declares
+ *  it anywhere (`const Widget`, `function Widget`, a destructured
+ *  `{ Widget }` parameter), not only at the top level. These entries have
+ *  no `node` (nothing to turn into a fragment) and `inCode: true`. */
+function findUnknownInCode(tree, bound) {
+  const found = [];
+  const seen = new Set();
+  const codeNodes = [];
+  visit(tree, ['mdxjsEsm', 'mdxFlowExpression', 'mdxTextExpression'], (node) => {
+    if (node.value) codeNodes.push(node);
+  });
+  const allCode = codeNodes.map((n) => n.value).join('\n');
+  const declaredInCode = (name) =>
+    new RegExp(`\\b(?:const|let|var|function|class)\\s+${name}\\b|[{,(]\\s*(?:\\.\\.\\.)?${name}\\s*[,}=):]`).test(allCode);
+  for (const node of codeNodes) {
+    for (const m of node.value.matchAll(/<([A-Z][A-Za-z0-9]*)((?:\.[A-Za-z0-9]+)*)[\s/>]/g)) {
+      const root = m[1];
+      const members = m[2] ? m[2].slice(1).split('.') : [];
+      const name = [root, ...members].join('.');
+      if (seen.has(name) || bound.has(root) || declaredInCode(root)) continue;
+      if (BUILTIN_COMPONENT_NAMES.includes(root) && (members.length === 0 || (members.length === 1 && BUILTIN_MEMBERS[root]?.includes(members[0])))) continue;
+      seen.add(name);
+      const before = node.value.slice(0, m.index);
+      const line = (node.position?.start?.line ?? 0) + (before.match(/\n/g)?.length ?? 0);
+      found.push({ node: null, name, line: line || undefined, column: undefined, inCode: true, root });
+    }
+  }
+  return found;
+}
+
+/**
+ * Remark plugin: an unknown component renders its children and nothing
+ * else - the element becomes a fragment - with a warning naming the file
+ * and line, instead of failing the build. The site still builds and reads
+ * sensibly (the component's text is still there), and the warning, plus
+ * `writedocs validate`, say what to fix.
+ */
+export function remarkUnknownComponentFallback() {
+  return (tree, file) => {
+    const found = findUnknownComponents(tree);
+    if (found.length === 0) return;
+    const contentDir = process.env.WRITEDOCS_CONTENT_DIR;
+    const filePath = file.path
+      ? contentDir
+        ? path.relative(contentDir, file.path).split(path.sep).join('/')
+        : file.path
+      : '(unknown file)';
+    const stubs = new Set();
+    for (const { node, name, line, inCode, root } of found) {
+      console.warn(
+        `[writedocs] ${filePath}${line ? `:${line}` : ''} - unknown component <${name}>, showing only its content. ` +
+          'Remove it, or define it in a snippet.'
+      );
+      if (inCode) {
+        stubs.add(root);
+      } else {
+        node.name = null;
+        node.attributes = [];
+      }
+    }
+    // Inside code there's no element to turn into a fragment - the name
+    // itself has to exist, or rendering throws "X is not defined". Each one
+    // gets a stand-in component that renders its children (a dotted
+    // `<Widget.Part>` gets its parts on the same stand-in).
+    if (stubs.size > 0) {
+      const source = [...stubs]
+        .map(
+          (name) =>
+            `export const ${name} = new Proxy((props) => props?.children ?? null, { get: (fn, key) => (key in fn ? fn[key] : (props) => props?.children ?? null) });`
+        )
+        .join('\n');
+      const estree = acornParse(source, { ecmaVersion: 'latest', sourceType: 'module' });
+      tree.children.unshift({ type: 'mdxjsEsm', value: source, data: { estree } });
+    }
+  };
+}