@pterodoc/core 0.2.0

package/src/render/images.ts

@@ -0,0 +1,113 @@
+/**
+ * Finding and resolving the files a document references.
+ *
+ * Images are found wherever they sit — in a paragraph, behind a reference,
+ * inside raw HTML — because the one place they never sit is at the top level
+ * of a document, which is where an earlier version of this code looked.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { visit } from 'unist-util-visit';
+import type { Image, Root } from 'mdast';
+import { parseMarkdown, type MarkdownFormat } from './parse';
+import { resolveReferences } from './references';
+import { isAbsoluteUrl } from './links';
+
+/** An image a document points at. */
+export interface ImageReference {
+  /** The URL exactly as written in the source. */
+  url: string;
+  /** Alternative text. */
+  alt: string;
+  /** Title, which becomes a caption. */
+  title?: string | undefined;
+}
+
+/** Where an image URL actually points. */
+export type ResolvedImage =
+  | { kind: 'external'; url: string }
+  | { kind: 'file'; url: string; file: string }
+  | { kind: 'missing'; url: string };
+
+/** Where to look when resolving a relative or absolute reference. */
+export interface ImageResolutionContext {
+  /** Absolute path of the document's own source file. */
+  sourceAbsolutePath: string;
+  /** The version's content directory. */
+  contentPath: string;
+  /** The localised content directory, preferred when the file exists in both. */
+  contentPathLocalized: string;
+  /** Absolute paths of the site's static directories. */
+  staticDirs: string[];
+  /** The site's base URL, stripped from absolute references when present. */
+  baseUrl: string;
+  /** The Docusaurus site directory, for `@site/` references. */
+  siteDir: string;
+  /** Injected so resolution can be tested without a filesystem. */
+  exists?: (file: string) => boolean;
+}
+
+/** Every image a document references, including ones behind a reference definition. */
+export function collectImages(markdown: string, format: MarkdownFormat = 'md'): ImageReference[] {
+  const root: Root = parseMarkdown(markdown, format);
+  // Reference-style images become ordinary ones first, so they are not missed.
+  resolveReferences(root);
+
+  const found: ImageReference[] = [];
+  const seen = new Set<string>();
+  visit(root, 'image', (node: Image) => {
+    if (seen.has(node.url)) return;
+    seen.add(node.url);
+    found.push({
+      url: node.url,
+      alt: node.alt ?? '',
+      ...(node.title != null ? { title: node.title } : {}),
+    });
+  });
+  return found;
+}
+
+/**
+ * Work out which file an image URL points at.
+ *
+ * The order matches how Docusaurus itself resolves them: an absolute URL is
+ * left alone, `@site/` is site-relative, a leading slash is served from a
+ * static directory, and anything else is relative to the document — preferring
+ * the localised copy when there is one.
+ */
+export function resolveImage(url: string, ctx: ImageResolutionContext): ResolvedImage {
+  const exists = ctx.exists ?? ((file: string) => fs.existsSync(file));
+
+  if (!url || url.startsWith('data:') || isAbsoluteUrl(url)) return { kind: 'external', url };
+
+  if (url.startsWith('@site/')) {
+    const file = path.resolve(ctx.siteDir, url.slice('@site/'.length));
+    return exists(file) ? { kind: 'file', url, file } : { kind: 'missing', url };
+  }
+
+  if (url.startsWith('/')) {
+    const base = ctx.baseUrl.replace(/\/+$/, '');
+    const withoutBase = base && url.startsWith(`${base}/`) ? url.slice(base.length) : url;
+    for (const dir of ctx.staticDirs) {
+      for (const candidate of [path.join(dir, url), path.join(dir, withoutBase)]) {
+        if (exists(candidate)) return { kind: 'file', url, file: candidate };
+      }
+    }
+    return { kind: 'missing', url };
+  }
+
+  const sourceDir = path.dirname(ctx.sourceAbsolutePath);
+  const direct = path.resolve(sourceDir, url);
+  if (exists(direct)) return { kind: 'file', url, file: direct };
+
+  // The document may be the unlocalised original while its assets sit beside
+  // the translated copy, or the other way round.
+  if (ctx.contentPath && ctx.contentPathLocalized && ctx.contentPath !== ctx.contentPathLocalized) {
+    const relative = path.relative(ctx.contentPath, direct);
+    const twin = path.resolve(ctx.contentPathLocalized, relative);
+    if (!relative.startsWith('..') && exists(twin)) return { kind: 'file', url, file: twin };
+  }
+
+  return { kind: 'missing', url };
+}

package/src/render/index.ts

@@ -0,0 +1,129 @@
+/**
+ * Rendering a document: parse, resolve, and serialise to blocks.
+ *
+ * This layer is pure. It never reaches the network, never loads Docusaurus and
+ * knows nothing about the target beyond the URLs a resolver hands back.
+ */
+
+import matter from 'gray-matter';
+import { parseMarkdown, detectFormat, type MarkdownFormat } from './parse';
+import { resolveReferences } from './references';
+import { lowerMdx, type UnknownPolicy } from './mdx';
+import { rewriteLinks, type LinkResolver } from './links';
+import { createRenderContext, renderBody, type RenderContext } from './renderers';
+import { createSlugger } from './slug';
+import { excerptFrom } from './excerpt';
+import { DEFAULT_ADMONITION_KEYWORDS } from './admonitions';
+import type { Theme } from './theme';
+import { IssueCollector } from '../util/issues';
+
+export { createTheme, DEFAULT_STRINGS } from './theme';
+export type { Theme, Strings, BlockVocabulary } from './theme';
+export { parseMarkdown, detectFormat } from './parse';
+export type { MarkdownFormat } from './parse';
+export { renderBlock, renderBody, createRenderContext } from './renderers';
+export type { RenderContext } from './renderers';
+export { excerptFrom } from './excerpt';
+export { serializeBlock, serializeVoidBlock, serializeAttrs, escapeCode, escapeText, joinBlocks } from './blocks';
+export { renderInline } from './inline';
+export { headingIdFor, createSlugger } from './slug';
+export { resolveReferences } from './references';
+export { rewriteLinks, toInternalPath, isAbsoluteUrl } from './links';
+export type { LinkResolver, ResolvedLink } from './links';
+export { DEFAULT_ADMONITION_KEYWORDS } from './admonitions';
+export { lowerMdx, isTranslatable, KNOWN_COMPONENTS } from './mdx';
+export type { UnknownPolicy } from './mdx';
+export { renderJsxBlock, attribute } from './components';
+export { collectImages, resolveImage } from './images';
+export type { ImageReference, ResolvedImage } from './images';
+export { composePage, DEFAULT_LAYOUT, renderNavigationStub, renderVersionBanner } from './page';
+export type { ComposePageInput, PageLayout, PageLike } from './page';
+
+/** What to render, and how. */
+export interface RenderDocInput {
+  /** The document body, front matter included or not. */
+  markdown: string;
+  /** Absolute or site-relative path, used for issue positions and format detection. */
+  file?: string;
+  /** Permalink of this document, which relative links resolve against. */
+  permalink: string;
+  /** Which flavour to parse as; detected from `file` when omitted. */
+  format?: MarkdownFormat;
+  /** Class names and strings. */
+  theme: Theme;
+  /** Where each link should point. */
+  resolveLink?: LinkResolver;
+  /** Directive names that mean "admonition" on this site. */
+  admonitionKeywords?: Iterable<string>;
+  /** Whether Docusaurus keeps heading case when generating anchors. */
+  maintainCase?: boolean;
+  /** Drop a leading H1 that repeats the page title. */
+  dedupeTitle?: boolean;
+  /** Uploaded media, keyed by the URL as written in the source. */
+  media?: Map<string, { id: number; url: string }>;
+  /** What to do about JSX with no translation. */
+  onUnknownJsx?: UnknownPolicy;
+  /** Collector to record issues into; a fresh one is made when omitted. */
+  issues?: IssueCollector;
+}
+
+/** A rendered document body and what was learned while rendering it. */
+export interface RenderedDoc {
+  /** The block markup. */
+  body: string;
+  /** Tree paths of the pages this document links to. */
+  links: Set<string>;
+  /** Plain text of the first paragraph, for an excerpt. */
+  firstParagraph: string;
+  /** Everything worth telling the user about this document. */
+  issues: IssueCollector;
+}
+
+/**
+ * Render one document to Gutenberg block markup.
+ */
+export function renderDoc(input: RenderDocInput): RenderedDoc {
+  const issues = input.issues ?? new IssueCollector();
+  const parsed = matter(input.markdown);
+  const source = parsed.content.replace(/\r\n/g, '\n').replace(/^\n+/, '');
+
+  const format = input.format ?? (input.file ? detectFormat(input.file) : 'md');
+  const root = parseMarkdown(source, format);
+
+  if (format === 'mdx') {
+    lowerMdx(root, {
+      source,
+      file: input.file,
+      issues,
+      onUnknown: input.onUnknownJsx ?? 'report',
+    });
+  }
+
+  resolveReferences(root);
+  const links = input.resolveLink
+    ? rewriteLinks(root, input.permalink, input.resolveLink, issues)
+    : new Set<string>();
+
+  const ctx: RenderContext = createRenderContext({
+    theme: input.theme,
+    slugger: createSlugger(),
+    issues,
+    maintainCase: input.maintainCase ?? false,
+    admonitionKeywords: new Set(input.admonitionKeywords ?? DEFAULT_ADMONITION_KEYWORDS),
+    file: input.file,
+    source,
+    media: input.media,
+    onUnknownJsx: input.onUnknownJsx ?? 'report',
+  });
+
+  const { body, firstParagraph } = renderBody(root, ctx, {
+    dedupeTitle: input.dedupeTitle ?? true,
+  });
+
+  return { body, links, firstParagraph, issues };
+}
+
+/** Build an excerpt from a description, falling back to the opening paragraph. */
+export function excerptFor(description: string, firstParagraph: string, max = 160): string {
+  return excerptFrom(description || firstParagraph || '', max);
+}

package/src/render/inline.ts

@@ -0,0 +1,62 @@
+/**
+ * Phrasing content to HTML.
+ *
+ * The whole run of nodes is converted in one pass, so `mdast-util-to-hast`
+ * keeps whatever state it needs, and the conversion happens inside a synthetic
+ * paragraph so it does not separate the children with newlines the way it does
+ * for a root.
+ */
+
+import { toHast } from 'mdast-util-to-hast';
+import { toHtml } from 'hast-util-to-html';
+import type { PhrasingContent } from 'mdast';
+import type { Element, Node as HastNode, Parent as HastParent, RootContent, Text } from 'hast';
+
+/**
+ * Escape shortcode brackets inside text, leaving tags and attributes alone.
+ *
+ * Done on the tree rather than on the serialised string: a `>` inside an
+ * attribute value would fool any regex that tries to find tag boundaries.
+ * A `raw` node carries the entity through serialisation unescaped.
+ */
+export function escapeShortcodesInHast(nodes: RootContent[]): RootContent[] {
+  const out: RootContent[] = [];
+  for (const node of nodes) {
+    if (node.type === 'text' && (node as Text).value.includes('[')) {
+      const parts = (node as Text).value.split('[');
+      parts.forEach((part, index) => {
+        if (index > 0) out.push({ type: 'raw', value: '[' } as unknown as RootContent);
+        if (part !== '') out.push({ type: 'text', value: part } as Text);
+      });
+      continue;
+    }
+    if ('children' in node && Array.isArray((node as HastParent).children)) {
+      (node as HastParent).children = escapeShortcodesInHast(
+        (node as HastParent).children as RootContent[],
+      );
+    }
+    out.push(node);
+  }
+  return out;
+}
+
+/** Convert phrasing content to the hast nodes that back it. */
+export function toInlineHast(nodes: PhrasingContent[]): RootContent[] {
+  const paragraph = toHast(
+    { type: 'paragraph', children: nodes },
+    { allowDangerousHtml: true },
+  ) as Element | undefined;
+  const children = (paragraph?.children ?? []) as RootContent[];
+  return escapeShortcodesInHast(children);
+}
+
+/** Serialise hast nodes to HTML. */
+export function hastToHtml(nodes: RootContent[] | HastNode): string {
+  const tree = Array.isArray(nodes) ? { type: 'root' as const, children: nodes } : nodes;
+  return toHtml(tree as never, { allowDangerousHtml: true });
+}
+
+/** Render phrasing content to HTML. */
+export function renderInline(nodes: PhrasingContent[]): string {
+  return hastToHtml(toInlineHast(nodes));
+}

package/src/render/links.ts

@@ -0,0 +1,94 @@
+/**
+ * Link rewriting.
+ *
+ * A documentation link can be relative to the source file, absolute against
+ * the site's base URL, or an anchor. Whichever it is, it has to end up
+ * pointing at the right page on the target — or, when the target has no such
+ * page, somewhere honest.
+ */
+
+import path from 'node:path';
+import { visit } from 'unist-util-visit';
+import type { Link, Root } from 'mdast';
+import type { IssueCollector } from '../util/issues';
+
+/** What a link should become. */
+export interface ResolvedLink {
+  /** The replacement URL, or null to keep the text and drop the link. */
+  href: string | null;
+  /** Target page's tree path, when the link points at a published page. */
+  path?: string | undefined;
+}
+
+/** Decides where a link should point. */
+export type LinkResolver = (href: string, fromPermalink: string) => ResolvedLink;
+
+/** True for a URL that already names its own scheme or host. */
+export function isAbsoluteUrl(href: string): boolean {
+  return /^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(href) || href.startsWith('//');
+}
+
+/**
+ * Resolve a documentation link to the URL path it refers to.
+ *
+ * Relative targets are resolved against the linking document's own permalink,
+ * which is what makes this work regardless of where the docs are mounted.
+ *
+ * @param href The link as written.
+ * @param fromPermalink Permalink of the document containing the link.
+ * @returns The URL path and fragment, or undefined when the link is not internal.
+ */
+export function toInternalPath(
+  href: string,
+  fromPermalink: string,
+): { urlPath: string; hash: string } | undefined {
+  if (!href || href.startsWith('#') || isAbsoluteUrl(href)) return undefined;
+
+  const hashIndex = href.indexOf('#');
+  const hash = hashIndex === -1 ? '' : href.slice(hashIndex);
+  const target = hashIndex === -1 ? href : href.slice(0, hashIndex);
+
+  const withoutExtension = target.replace(/\.mdx?$/, '');
+  const base = fromPermalink.endsWith('/') ? fromPermalink : `${fromPermalink}/`;
+
+  const urlPath = withoutExtension.startsWith('/')
+    ? path.posix.normalize(withoutExtension)
+    : path.posix.normalize(path.posix.join(base, '..', withoutExtension));
+
+  return { urlPath, hash };
+}
+
+/**
+ * Rewrite every link in a document.
+ *
+ * @param root The document, modified in place.
+ * @param fromPermalink Permalink of the document.
+ * @param resolve Where each link should point.
+ * @returns Tree paths of the pages this document links to.
+ */
+export function rewriteLinks(
+  root: Root,
+  fromPermalink: string,
+  resolve: LinkResolver,
+  _issues?: IssueCollector,
+): Set<string> {
+  const linked = new Set<string>();
+
+  visit(root, 'link', (node: Link, index, parent) => {
+    const resolved = resolve(node.url, fromPermalink);
+    if (resolved.path !== undefined) linked.add(resolved.path);
+
+    if (resolved.href === null) {
+      // The target is not published, so keep the words and drop the link.
+      if (parent && index !== undefined) {
+        parent.children.splice(index, 1, ...(node.children as never[]));
+        return index;
+      }
+      return;
+    }
+    node.url = resolved.href;
+    return;
+  });
+
+  return linked;
+}

package/src/render/mdx.ts

@@ -0,0 +1,178 @@
+/**
+ * Lowering MDX into something a Gutenberg page can hold.
+ *
+ * MDX is JavaScript, and a WordPress page is not. What can be translated is
+ * translated by the component table; the rest is reported with its position
+ * rather than dropped, because a page that quietly loses a third of its
+ * content is worse than one that tells you it did.
+ */
+
+import { visit } from 'unist-util-visit';
+import type { Root, RootContent } from 'mdast';
+import type { IssueCollector, Severity } from '../util/issues';
+
+/** What lowering learned on the way through. */
+export interface MdxLoweringResult {
+  /** Bindings from `import X from './y.png'`, so `src={X}` can be resolved. */
+  imports: Map<string, string>;
+}
+
+/** How to treat content that cannot be translated. */
+export type UnknownPolicy = 'report' | 'placeholder' | 'error';
+
+/** Elements that are plain HTML and can be carried through as written. */
+const HTML_ELEMENTS = new Set([
+  'a', 'abbr', 'b', 'br', 'code', 'div', 'em', 'figcaption', 'figure', 'h1', 'h2', 'h3',
+  'h4', 'h5', 'h6', 'hr', 'i', 'img', 'kbd', 'li', 'mark', 'ol', 'p', 'pre', 's', 'samp',
+  'small', 'span', 'strong', 'sub', 'sup', 'table', 'tbody', 'td', 'th', 'thead', 'tr',
+  'u', 'ul', 'var',
+]);
+
+/** Components the renderer knows how to turn into blocks. */
+export const KNOWN_COMPONENTS = new Set([
+  'Tabs',
+  'TabItem',
+  'Details',
+  'details',
+  'summary',
+  'CodeBlock',
+  'Admonition',
+]);
+
+/** A node with a position we can quote from the source. */
+interface Positioned {
+  type: string;
+  position?: { start: { line: number; column: number; offset?: number }; end: { offset?: number } };
+}
+
+/** The original text a node was parsed from. */
+function sourceOf(node: Positioned, source: string): string {
+  const start = node.position?.start.offset;
+  const end = node.position?.end.offset;
+  return start !== undefined && end !== undefined ? source.slice(start, end) : '';
+}
+
+/** Read the import specifiers out of an ESM block, without evaluating it. */
+function readImports(value: string, into: Map<string, string>): void {
+  const pattern = /import\s+([A-Za-z_$][\w$]*)\s+from\s+['"]([^'"]+)['"]/g;
+  for (const match of value.matchAll(pattern)) into.set(match[1]!, match[2]!);
+}
+
+/** True for an expression that holds nothing but a comment. */
+function isCommentOnly(node: { value?: string; data?: { estree?: { body?: unknown[]; comments?: unknown[] } } }): boolean {
+  const estree = node.data?.estree;
+  if (estree) return (estree.body?.length ?? 0) === 0 && (estree.comments?.length ?? 0) > 0;
+  return /^\s*\/[/*]/.test(node.value ?? '');
+}
+
+/**
+ * Replace MDX-only nodes so nothing downstream has to know about them.
+ *
+ * @param root The document, modified in place.
+ * @param ctx Where to report, and what to do about the untranslatable.
+ */
+export function lowerMdx(
+  root: Root,
+  ctx: {
+    source: string;
+    file?: string | undefined;
+    issues: IssueCollector;
+    onUnknown: UnknownPolicy;
+  },
+): MdxLoweringResult {
+  const imports = new Map<string, string>();
+  const severity: Severity = ctx.onUnknown === 'error' ? 'error' : 'warning';
+
+  const report = (node: Positioned, message: string, code: string): RootContent | undefined => {
+    ctx.issues.add({
+      code,
+      severity,
+      message,
+      file: ctx.file,
+      line: node.position?.start.line,
+      column: node.position?.start.column,
+    });
+    if (ctx.onUnknown !== 'placeholder') return undefined;
+    return { type: 'html', value: `<!-- pterodoc: ${message} -->` } as RootContent;
+  };
+
+  const replacements: { parent: { children: RootContent[] }; index: number; with: RootContent | undefined }[] = [];
+
+  visit(root, (node, index, parent) => {
+    if (!parent || index === undefined) return;
+    const typed = node as Positioned;
+
+    switch (typed.type) {
+      case 'mdxjsEsm': {
+        readImports((node as { value?: string }).value ?? '', imports);
+        replacements.push({ parent: parent as never, index, with: undefined });
+        return;
+      }
+
+      case 'mdxFlowExpression':
+      case 'mdxTextExpression': {
+        const expression = node as { value?: string };
+        if (isCommentOnly(expression)) {
+          replacements.push({ parent: parent as never, index, with: undefined });
+          return;
+        }
+        replacements.push({
+          parent: parent as never,
+          index,
+          with: report(
+            typed,
+            `An MDX expression ({${(expression.value ?? '').trim().slice(0, 40)}}) has no fixed value outside the site, so it was left out.`,
+            'mdx-expression',
+          ),
+        });
+        return;
+      }
+
+      case 'mdxJsxTextElement': {
+        const element = node as { name?: string | null; children?: RootContent[] };
+        const name = element.name ?? '';
+        if (name && HTML_ELEMENTS.has(name)) {
+          // Plain HTML: keep exactly what the author wrote.
+          replacements.push({
+            parent: parent as never,
+            index,
+            with: { type: 'html', value: sourceOf(typed, ctx.source) } as RootContent,
+          });
+          return;
+        }
+        if (name === '') {
+          // A fragment: keep the children, drop the wrapper.
+          return;
+        }
+        replacements.push({
+          parent: parent as never,
+          index,
+          with: report(
+            typed,
+            `<${name}> is a React component, which a page cannot run, so it was left out.`,
+            'mdx-unknown-component',
+          ),
+        });
+        return;
+      }
+
+      default:
+        return;
+    }
+  });
+
+  // Applied afterwards so the walk is not disturbed by its own edits.
+  for (const replacement of replacements.reverse()) {
+    if (replacement.with) replacement.parent.children.splice(replacement.index, 1, replacement.with);
+    else replacement.parent.children.splice(replacement.index, 1);
+  }
+
+  return { imports };
+}
+
+/** True for an element name pterodoc can translate into blocks. */
+export function isTranslatable(name: string): boolean {
+  return KNOWN_COMPONENTS.has(name) || HTML_ELEMENTS.has(name);
+}
+
+export { HTML_ELEMENTS };