@writedocs/generator 0.4.11 → 0.5.0

package/src/lib/link-check.js

@@ -0,0 +1,473 @@
+// `writedocs broken-links`: every internal link in the content - page
+// bodies, the snippets they import, and writedocs.json - checked against the
+// site the build would produce, without building it. Plain JavaScript,
+// loaded by plain Node from an installed package (see lib/icons.js's
+// comment on why not TypeScript).
+//
+// A link is broken when, on the built site:
+//   - no page, redirect or file is at its address;
+//   - its #anchor isn't on the target page (headings, titled components -
+//     Callout/Accordion/Update and friends - and elements with an `id`);
+//   - it names a file the build doesn't publish (anything outside public/
+//     other than images and fonts - see lib/styles-asset-integration.js).
+//
+// Addresses are worked out the way the build works them out:
+//   - a page's URL is its frontmatter `slug`, or else its file path with
+//     every segment put through github-slugger - Astro's own glob-loader
+//     id (getContentEntryIdAndSlug in astro/dist/content/utils.js), which
+//     is why `docs/API Guide/v1.2.mdx` is served at /docs/api-guide/v12/;
+//   - heading ids come from github-slugger too, deduped per page the way
+//     Astro's rehypeHeadingIds does; titled components' ids from
+//     lib/mdx-title-anchor-ids.js's own rule;
+//   - a relative link resolves the way a browser resolves it, from the
+//     page's URL - which ends in a slash (/docs/guides/setup/), so
+//     `install` on that page means /docs/guides/setup/install/, not a
+//     sibling page. Mintlify's URLs have no trailing slash, so migrated
+//     relative links are the usual casualty; the message says so.
+//
+// External links (http:, mailto:, ...) aren't checked.
+import fs from 'node:fs';
+import path from 'node:path';
+import { createRequire } from 'node:module';
+import { pathToFileURL } from 'node:url';
+import matter from 'gray-matter';
+import { visit } from 'unist-util-visit';
+import { findAllPages } from './pages.js';
+import { createJsonLocator } from './config-schema.js';
+import { writedocsTempDir } from './writedocs-temp-dir.js';
+
+// The same github-slugger Astro builds page URLs and heading ids with -
+// resolved through Astro itself, so the two can't drift apart.
+let sluggerModule = null;
+async function loadSlugger() {
+  if (!sluggerModule) {
+    const require = createRequire(import.meta.url);
+    const fromAstro = createRequire(require.resolve('astro/package.json'));
+    sluggerModule = await import(pathToFileURL(fromAstro.resolve('github-slugger')).href);
+  }
+  return sluggerModule;
+}
+
+const processors = {};
+async function parse(text, format) {
+  if (!processors[format]) {
+    const { createProcessor } = await import('@mdx-js/mdx');
+    processors[format] = createProcessor({ format });
+  }
+  return processors[format].parse(text);
+}
+
+// Images and fonts are published from anywhere in the project; every other
+// file only from public/ (lib/styles-asset-integration.js).
+const ASSET_EXTENSIONS = new Set(['.svg', '.png', '.jpg', '.jpeg', '.gif', '.webp', '.avif', '.ico', '.woff', '.woff2', '.ttf', '.otf']);
+// lib/mdx-title-anchor-ids.js
+const TITLED_COMPONENTS = ['Callout', 'Note', 'Info', 'Tip', 'Warning', 'Danger', 'Check', 'Accordion', 'Update'];
+const titleSlug = (value) =>
+  value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+
+const BASE = 'http://writedocs.invalid';
+
+function normalizeEntryId(id) {
+  const trimmed = String(id).replace(/^\/+/, '').replace(/\/+$/, '');
+  return trimmed === '' ? 'index' : trimmed;
+}
+const urlForId = (id) => (id === 'index' ? '/' : `/${id}/`);
+
+function textOf(node) {
+  if (typeof node.value === 'string' && (node.type === 'text' || node.type === 'inlineCode')) return node.value;
+  return (node.children ?? []).map(textOf).join('');
+}
+
+function attribute(node, name) {
+  const attr = node.attributes?.find((a) => a.type === 'mdxJsxAttribute' && a.name === name);
+  return attr && typeof attr.value === 'string' ? attr.value : null;
+}
+
+/** Everything a page or snippet file links to and can be linked to. */
+async function scanFile(absPath, { slug, Slugger }) {
+  let raw;
+  try {
+    raw = fs.readFileSync(absPath, 'utf-8').replace(/\r\n/g, '\n');
+  } catch {
+    return null;
+  }
+  let parsed;
+  try {
+    parsed = matter(raw, {});
+  } catch {
+    return null; // invalid frontmatter - `writedocs validate` reports it
+  }
+  const bodyStart = raw.lastIndexOf(parsed.content);
+  const lineOffset = bodyStart > 0 ? raw.slice(0, bodyStart).split('\n').length - 1 : 0;
+  let tree;
+  try {
+    tree = await parse(parsed.content, /\.mdx$/i.test(absPath) ? 'mdx' : 'md');
+  } catch {
+    return null; // doesn't parse - `writedocs validate` reports it
+  }
+
+  const links = []; // { href, line, kind: 'link' | 'image' | 'src' }
+  const anchors = new Set();
+  const imports = [];
+  const lineOf = (node) => (node.position?.start?.line ? node.position.start.line + lineOffset : undefined);
+  const headingSlugger = new Slugger();
+  const titleSeen = new Map();
+
+  visit(tree, (node) => {
+    switch (node.type) {
+      case 'link':
+      case 'definition':
+        links.push({ href: node.url, line: lineOf(node), kind: 'link' });
+        break;
+      case 'image':
+        links.push({ href: node.url, line: lineOf(node), kind: 'image' });
+        break;
+      case 'heading': {
+        const text = textOf(node);
+        if (text.trim()) anchors.add(headingSlugger.slug(text));
+        break;
+      }
+      case 'html': {
+        // Raw HTML in a .md page.
+        for (const m of node.value.matchAll(/\bhref\s*=\s*["']([^"']*)["']/gi)) links.push({ href: m[1], line: lineOf(node), kind: 'link' });
+        for (const m of node.value.matchAll(/\bsrc\s*=\s*["']([^"']*)["']/gi)) links.push({ href: m[1], line: lineOf(node), kind: 'src' });
+        for (const m of node.value.matchAll(/\b(?:id|name)\s*=\s*["']([^"']+)["']/gi)) anchors.add(m[1]);
+        break;
+      }
+      case 'mdxjsEsm':
+        for (const m of node.value.matchAll(/\bfrom\s*['"]([^'"]+\.mdx?)['"]/g)) imports.push(m[1]);
+        break;
+      case 'mdxJsxFlowElement':
+      case 'mdxJsxTextElement': {
+        const href = attribute(node, 'href');
+        if (href !== null) links.push({ href, line: lineOf(node), kind: 'link' });
+        if (!/^(iframe|script|embed)$/i.test(node.name ?? '')) {
+          for (const name of ['src', 'img']) {
+            const value = attribute(node, name);
+            if (value !== null) links.push({ href: value, line: lineOf(node), kind: 'src' });
+          }
+        }
+        const id = attribute(node, 'id');
+        if (id) anchors.add(id);
+        if (TITLED_COMPONENTS.includes(node.name)) {
+          const title = attribute(node, node.name === 'Update' ? 'label' : 'title');
+          const base = title ? titleSlug(title) : '';
+          if (base) {
+            const prior = titleSeen.get(base) ?? 0;
+            titleSeen.set(base, prior + 1);
+            anchors.add(prior === 0 ? base : `${base}-${prior + 1}`);
+          }
+        }
+        break;
+      }
+    }
+  });
+  return { data: parsed.data ?? {}, links, anchors, imports };
+}
+
+function editDistance(a, b) {
+  const row = Array.from({ length: b.length + 1 }, (_, i) => i);
+  for (let i = 1; i <= a.length; i++) {
+    let prev = row[0];
+    row[0] = i;
+    for (let j = 1; j <= b.length; j++) {
+      const tmp = row[j];
+      row[j] = Math.min(row[j] + 1, row[j - 1] + 1, prev + (a[i - 1] === b[j - 1] ? 0 : 1));
+      prev = tmp;
+    }
+  }
+  return row[b.length];
+}
+
+function closest(value, candidates) {
+  let best = null;
+  let bestDistance = Infinity;
+  for (const c of candidates) {
+    const d = editDistance(value.toLowerCase(), c.toLowerCase());
+    if (d < bestDistance) {
+      best = c;
+      bestDistance = d;
+    }
+  }
+  return best !== null && bestDistance <= Math.max(2, Math.floor(value.length * 0.3)) ? best : null;
+}
+
+/** Generated OpenAPI pages' URLs, from the manifests generateApiPages()
+ *  (src/cli/generate-api-pages.js) wrote - it runs first. */
+function generatedPageUrls(contentDir) {
+  const urls = new Set();
+  const root = path.join(writedocsTempDir(contentDir), 'openapi');
+  (function walk(dir) {
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const abs = path.join(dir, entry.name);
+      if (entry.isDirectory()) walk(abs);
+      else if (entry.name === 'manifest.json') {
+        try {
+          for (const op of JSON.parse(fs.readFileSync(abs, 'utf-8'))) if (op.generated) urls.add(urlForId(normalizeEntryId(op.slug)));
+        } catch {}
+      }
+    }
+  })(root);
+  return urls;
+}
+
+/** Every `href` in writedocs.json, and redirect destinations. */
+function configLinks(config) {
+  const found = [];
+  (function walk(node, trail) {
+    if (Array.isArray(node)) node.forEach((item, i) => walk(item, [...trail, i]));
+    else if (node && typeof node === 'object') {
+      for (const [key, value] of Object.entries(node)) {
+        if (key === 'href' && typeof value === 'string') found.push({ href: value, path: [...trail, key] });
+        else walk(value, [...trail, key]);
+      }
+    }
+  })(config, []);
+  (Array.isArray(config?.redirects) ? config.redirects : []).forEach((r, i) => {
+    if (typeof r?.destination === 'string') found.push({ href: r.destination, path: ['redirects', i, 'destination'] });
+  });
+  return found;
+}
+
+/**
+ * Checks every internal link in `contentDir`. `configText` is writedocs.json's
+ * raw text (valid JSON - the CLI checks that first). Returns
+ * { pages, links, external, broken } - `broken` is a list of issues shaped
+ * like lib/content-check.js's: { file, line?, message, suggestion? }.
+ */
+export async function checkLinks(contentDir, configText) {
+  const { slug, default: Slugger } = await loadSlugger();
+  const config = JSON.parse(configText);
+
+  // --- What the site has -------------------------------------------------
+  const pageFiles = findAllPages(contentDir);
+  const pages = new Map(); // url -> { rel, abs, scan }
+  const urlOfFile = new Map(); // rel -> url
+  const scans = new Map(); // abs -> scanFile() result (or null)
+  const scanOnce = async (abs) => {
+    if (!scans.has(abs)) scans.set(abs, await scanFile(abs, { slug, Slugger }));
+    return scans.get(abs);
+  };
+  for (const rel of pageFiles) {
+    const abs = path.join(contentDir, rel);
+    const scan = await scanOnce(abs);
+    const id = scan?.data?.slug
+      ? normalizeEntryId(scan.data.slug)
+      : normalizeEntryId(
+          rel
+            .replace(/\.mdx?$/i, '')
+            .split('/')
+            .map((segment) => slug(segment))
+            .join('/')
+            .replace(/\/index$/, '')
+        );
+    const url = urlForId(id);
+    pages.set(url, { rel, abs, scan });
+    urlOfFile.set(rel, url);
+  }
+  const generated = generatedPageUrls(contentDir);
+  const redirects = new Set(
+    (Array.isArray(config.redirects) ? config.redirects : [])
+      .map((r) => (typeof r?.source === 'string' ? `/${r.source.replace(/^\/+|\/+$/g, '')}/`.replace(/^\/\/$/, '/') : null))
+      .filter(Boolean)
+  );
+  const special = new Set(['/llms.txt', '/llms-full.txt', '/404.html', '/404/']);
+  if (config.domain) ['/sitemap.xml', '/sitemap-index.xml'].forEach((p) => special.add(p));
+  const knownUrls = [...pages.keys(), ...generated, ...redirects];
+
+  // Anchors a page can be linked to: its own, plus those of every snippet
+  // it imports (they render inside it).
+  const snippetPath = (spec, fromAbs) =>
+    spec.startsWith('/') ? path.join(contentDir, spec) : path.resolve(path.dirname(fromAbs), spec);
+  const anchorCache = new Map();
+  async function anchorsOf(abs, seen = new Set()) {
+    if (anchorCache.has(abs)) return anchorCache.get(abs);
+    if (seen.has(abs)) return new Set();
+    seen.add(abs);
+    const scan = await scanOnce(abs);
+    const all = new Set(scan?.anchors ?? []);
+    for (const spec of scan?.imports ?? []) for (const a of await anchorsOf(snippetPath(spec, abs), seen)) all.add(a);
+    anchorCache.set(abs, all);
+    return all;
+  }
+
+  // Root-relative image/font paths something displays - see the "Any other
+  // file" case below. Snippets are scanned here too, so the set is complete
+  // before the first link is checked.
+  const displayedAssets = new Set();
+  async function collectDisplayed(abs, seen = new Set()) {
+    if (seen.has(abs)) return;
+    seen.add(abs);
+    const scan = await scanOnce(abs);
+    for (const link of scan?.links ?? []) if (link.kind !== 'link' && link.href.startsWith('/')) displayedAssets.add(link.href.split(/[?#]/)[0]);
+    for (const spec of scan?.imports ?? []) await collectDisplayed(snippetPath(spec, abs), seen);
+  }
+  for (const page of pages.values()) await collectDisplayed(page.abs);
+  (function walk(node) {
+    if (typeof node === 'string') {
+      if (node.startsWith('/') && ASSET_EXTENSIONS.has(path.posix.extname(node).toLowerCase())) displayedAssets.add(node);
+    } else if (Array.isArray(node)) node.forEach(walk);
+    else if (node && typeof node === 'object') Object.values(node).forEach(walk);
+  })(config);
+
+  // --- Checking one link -------------------------------------------------
+  const broken = [];
+  const reported = new Set();
+  let checked = 0;
+  let external = 0;
+
+  /** `pageUrl`: the URL of the page the link renders on (relative links
+   *  resolve from it). `fileAbs`: the file the link is written in. */
+  async function check(href, { file, line, kind, pageUrl, fileAbs }) {
+    const value = String(href ?? '').trim();
+    if (!value || value === '#' || value.startsWith('{') || value.includes('{{')) return;
+    if (/^[a-z][a-z0-9+.-]*:/i.test(value) || value.startsWith('//')) {
+      external += 1;
+      return;
+    }
+    checked += 1;
+    const report = (message, suggestion) => {
+      const key = `${file}\n${line}\n${value}`;
+      if (reported.has(key)) return;
+      reported.add(key);
+      broken.push({ file, line, message, suggestion });
+    };
+
+    // A Markdown image with a relative path is resolved from the file's own
+    // folder at build time (Astro's image pipeline), not from the URL.
+    if (kind === 'image' && fileAbs && !value.startsWith('/') && !value.startsWith('#')) {
+      const target = path.resolve(path.dirname(fileAbs), decodeURI(value.split(/[?#]/)[0]));
+      if (!fs.existsSync(target)) report(`Image ${value} - there's no file at ${path.relative(contentDir, target).split(path.sep).join('/')}.`);
+      return;
+    }
+
+    let url;
+    try {
+      url = new URL(value, BASE + (pageUrl ?? '/'));
+    } catch {
+      report(`${value} - not a valid link.`);
+      return;
+    }
+    let pathname;
+    try {
+      pathname = decodeURI(url.pathname);
+    } catch {
+      pathname = url.pathname;
+    }
+    const hash = url.hash ? decodeURIComponent(url.hash.slice(1)) : '';
+    const extension = path.posix.extname(pathname.replace(/\/+$/, '')).toLowerCase();
+    const isRelative = !value.startsWith('/') && !value.startsWith('#');
+
+    // A page (or /x/index.html).
+    let pageKey = null;
+    if (!extension || pathname.endsWith('/index.html')) {
+      pageKey = pathname.replace(/\/index\.html$/, '/').replace(/\/?$/, '/');
+    }
+    if (pageKey !== null) {
+      if (pages.has(pageKey)) {
+        if (!hash || hash === 'top') return;
+        const anchors = await anchorsOf(pages.get(pageKey).abs);
+        if (anchors.has(hash)) return;
+        const suggestion = closest(hash, [...anchors]);
+        const onPage = pageKey === pageUrl && value.startsWith('#') ? 'this page' : pageKey;
+        report(
+          `${value} - ${onPage} has no heading or anchor "#${hash}".`,
+          suggestion ? `Did you mean #${suggestion}?` : undefined
+        );
+        return;
+      }
+      // "/" always exists: the home page, or a redirect to the first page.
+      if (pageKey === '/' || generated.has(pageKey) || redirects.has(pageKey) || special.has(pageKey.replace(/\/$/, '') || '/')) return;
+      if (special.has(pathname)) return;
+
+      // Mintlify-style relative link: resolved from the file's folder
+      // instead of the page's URL, would it land on a page?
+      if (isRelative && pageUrl && pageUrl !== '/') {
+        const siblingBase = pageUrl.replace(/[^/]+\/$/, '');
+        const alt = new URL(value, BASE + siblingBase);
+        const altKey = decodeURI(alt.pathname).replace(/\/?$/, '/');
+        if (pages.has(altKey) || generated.has(altKey)) {
+          report(
+            `${value} - relative links resolve from the page's own URL (${pageUrl}), so this one points to ${pageKey}, which doesn't exist.`,
+            `Write ${altKey}${alt.hash} instead.`
+          );
+          return;
+        }
+      }
+      const suggestion = closest(pageKey, knownUrls);
+      report(`${value} - there's no page at ${pageKey}.`, suggestion ? `Did you mean ${suggestion}?` : undefined);
+      return;
+    }
+
+    // A link to a page's source file (docs/setup.mdx) instead of its URL.
+    if (extension === '.mdx' || extension === '.md') {
+      const bare = pathname.replace(/\.mdx?$/i, '').replace(/\/?$/, '/');
+      if (extension === '.md' && config.contextMenu && pages.has(bare)) return; // the page's Markdown copy
+      // Written as a file path, so look it up as one: from the file's own
+      // folder when relative.
+      const filePath = isRelative && fileAbs ? path.resolve(path.dirname(fileAbs), decodeURI(value.split(/[?#]/)[0])) : path.join(contentDir, pathname);
+      const rel = path.relative(contentDir, filePath).split(path.sep).join('/');
+      const byFile = urlOfFile.get(rel) ?? (pages.has(bare) ? bare : null);
+      report(
+        `${value} - links to a source file, which isn't published.`,
+        byFile ? `Link to the page's URL instead: ${byFile}` : undefined
+      );
+      return;
+    }
+
+    // Any other file.
+    if (special.has(pathname)) return;
+    const segments = pathname.replace(/^\/+/, '').split('/');
+    if (fs.existsSync(path.join(contentDir, 'public', ...segments))) return;
+    const inProject = path.join(contentDir, ...segments);
+    if (fs.existsSync(inProject)) {
+      // Outside public/, an image or font is published only when something
+      // displays it - an <img src>, a Markdown image, a writedocs.json
+      // field - not for a plain link to it (lib/styles-asset-integration.js
+      // copies what the built pages' `src` attributes name).
+      if (ASSET_EXTENSIONS.has(extension) && (kind !== 'link' || displayedAssets.has(pathname))) return;
+      report(
+        `${value} - ${pathname} is in the project, but only files in public/ are published${
+          ASSET_EXTENSIONS.has(extension) ? ' for a link (an image outside it is published only where a page displays it)' : ''
+        }.`,
+        `Move it to public${pathname}.`
+      );
+      return;
+    }
+    report(`${value} - there's no file at ${pathname}.`);
+  }
+
+  // --- The pages, their snippets, and writedocs.json ---------------------
+  const relOf = (abs) => path.relative(contentDir, abs).split(path.sep).join('/');
+  const snippetsChecked = new Set();
+  async function checkFile(abs, pageUrl, seen) {
+    const scan = await scanOnce(abs);
+    if (!scan) return;
+    const file = relOf(abs);
+    for (const link of scan.links) await check(link.href, { file, line: link.line, kind: link.kind, pageUrl, fileAbs: abs });
+    for (const spec of scan.imports) {
+      const snippet = snippetPath(spec, abs);
+      if (seen.has(snippet) || snippetsChecked.has(`${snippet}\n${pageUrl}`)) continue;
+      snippetsChecked.add(`${snippet}\n${pageUrl}`);
+      await checkFile(snippet, pageUrl, new Set([...seen, snippet]));
+    }
+  }
+  for (const [url, page] of pages) await checkFile(page.abs, url, new Set([page.abs]));
+
+  const locate = createJsonLocator(configText);
+  for (const { href, path: jsonPath } of configLinks(config)) {
+    await check(href, { file: 'writedocs.json', line: locate(jsonPath)?.line, kind: 'link', pageUrl: '/' });
+  }
+
+  broken.sort((a, b) => a.file.localeCompare(b.file) || (a.line ?? 0) - (b.line ?? 0));
+  return { pages: pages.size, links: checked, external, broken };
+}

package/src/lib/mdx-inline-react.js

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import crypto from 'node:crypto';
 import { parse as acornParse } from 'acorn';
+import { report } from './cli-report.js';
 import { writedocsTempDir } from './writedocs-temp-dir.js';
 import { MINTLIFY_HOOKS } from './mdx-mintlify.js';
 import { BUILTIN_COMPONENT_NAMES } from './mdx-inject-builtins.js';
@@ -137,7 +138,7 @@ export function remarkExtractInlineReactComponents() {
     const relPath = path.relative(contentDir, file.path).split(path.sep).join('/');
     for (const use of simplified) {
       const line = source.slice(0, use.offset).split('\n').length;
-      console.warn(`[writedocs] ${relPath}:${line} - ${simplifiedBuiltinMessage(use).message}`);
+      report('warn', simplifiedBuiltinMessage(use).message, `${relPath}:${line}`);
     }
 
     const pageDir = path.dirname(file.path);

package/src/lib/mdx-unknown-components.js

@@ -2,6 +2,7 @@ import path from 'node:path';
 import { visit } from 'unist-util-visit';
 import { parse as acornParse } from 'acorn';
 import { BUILTIN_COMPONENT_NAMES } from './mdx-inject-builtins.js';
+import { report } from './cli-report.js';
 
 // Which JSX elements in an MDX file are components writedocs can't resolve
 // - neither a built-in nor something the file imports or defines itself.
@@ -120,9 +121,10 @@ export function remarkUnknownComponentFallback() {
       : '(unknown file)';
     const stubs = new Set();
     for (const { node, name, line, inCode, root } of found) {
-      console.warn(
-        `[writedocs] ${filePath}${line ? `:${line}` : ''} - unknown component <${name}>, showing only its content. ` +
-          'Remove it, or define it in a snippet.'
+      report(
+        'warn',
+        `Unknown component <${name}> - showing only its content. Remove it, or define it in a snippet.`,
+        `${filePath}${line ? `:${line}` : ''}`
       );
       if (inCode) {
         stubs.add(root);