create-eziwiki 0.1.0

package/template/lib/markdown/languages.test.ts

@@ -0,0 +1,85 @@
+import { describe, it, expect } from 'vitest';
+import { bundledLanguages } from 'shiki';
+import { clearLanguageCache, findFenceLanguages, getUsedLanguages } from './languages';
+
+describe('findFenceLanguages', () => {
+  it('finds the language on a fence', () => {
+    expect(findFenceLanguages('```typescript\nconst x = 1;\n```')).toEqual(['typescript']);
+  });
+
+  it('lower-cases identifiers', () => {
+    expect(findFenceLanguages('```TypeScript\nx\n```')).toEqual(['typescript']);
+  });
+
+  it('deduplicates', () => {
+    expect(findFenceLanguages('```js\na\n```\n\n```js\nb\n```')).toEqual(['js']);
+  });
+
+  it('finds several languages', () => {
+    expect(findFenceLanguages('```ts\na\n```\n\n```python\nb\n```').sort()).toEqual([
+      'python',
+      'ts',
+    ]);
+  });
+
+  it('handles tilde fences', () => {
+    expect(findFenceLanguages('~~~rust\nx\n~~~')).toEqual(['rust']);
+  });
+
+  it('ignores fences with no language', () => {
+    expect(findFenceLanguages('```\nplain\n```')).toEqual([]);
+  });
+
+  it('accepts identifiers with punctuation', () => {
+    expect(findFenceLanguages('```c++\nx\n```')).toEqual(['c++']);
+    expect(findFenceLanguages('```objective-c\nx\n```')).toEqual(['objective-c']);
+  });
+
+  it('returns nothing for text with no fences', () => {
+    expect(findFenceLanguages('just prose')).toEqual([]);
+  });
+});
+
+describe('getUsedLanguages', () => {
+  it('returns only grammars Shiki actually bundles', () => {
+    const known = new Set(Object.keys(bundledLanguages));
+
+    for (const language of getUsedLanguages()) {
+      expect(known.has(language), language).toBe(true);
+    }
+  });
+
+  it('loads far fewer than the full bundle', () => {
+    // The whole point is to avoid loading every grammar; if this ever
+    // approaches the bundle size, first-render cost has regressed badly.
+    expect(getUsedLanguages().length).toBeLessThan(Object.keys(bundledLanguages).length / 4);
+  });
+
+  it('always includes the base languages', () => {
+    const languages = getUsedLanguages();
+
+    for (const base of ['typescript', 'bash', 'json', 'markdown']) {
+      expect(languages, base).toContain(base);
+    }
+  });
+
+  it('includes languages the content uses', () => {
+    // content/features/syntax-highlighting.md demonstrates several languages.
+    expect(getUsedLanguages().length).toBeGreaterThan(10);
+  });
+
+  it('is sorted and free of duplicates', () => {
+    const languages = getUsedLanguages();
+
+    expect(languages).toEqual([...languages].sort());
+    expect(new Set(languages).size).toBe(languages.length);
+  });
+
+  it('memoises until cleared', () => {
+    const first = getUsedLanguages();
+    expect(getUsedLanguages()).toBe(first);
+
+    clearLanguageCache();
+    expect(getUsedLanguages()).not.toBe(first);
+  });
+});

package/template/lib/markdown/languages.ts

@@ -0,0 +1,103 @@
+import { bundledLanguages, type BuiltinLanguage } from 'shiki';
+import { getContentRegistry } from '../content/registry';
+import { cached } from '../cache';
+
+/**
+ * Works out which syntax-highlighting grammars the site actually needs.
+ *
+ * Shiki loads every bundled grammar by default — around two hundred of them —
+ * which dominates the cost of the first render and, therefore, of the whole
+ * build. Almost none of them are ever used: a documentation site typically
+ * touches a handful of languages. Scanning the content for the fences that
+ * exist and loading only those cuts initialisation from tens of seconds to
+ * well under one.
+ *
+ * Server-only.
+ */
+
+/**
+ * Languages always loaded, whether or not the content currently uses them.
+ *
+ * These cover the fences most likely to be added next, so a new code block in
+ * a common language does not silently render unhighlighted until someone
+ * notices.
+ */
+const BASE_LANGUAGES = [
+  'bash',
+  'css',
+  'diff',
+  'html',
+  'javascript',
+  'json',
+  'jsx',
+  'markdown',
+  'python',
+  'shell',
+  'sql',
+  'tsx',
+  'typescript',
+  'yaml',
+] as const;
+
+/** Matches the language written on an opening code fence. */
+const FENCE_PATTERN = /^ {0,3}(?:`{3,}|~{3,})[ \t]*([A-Za-z0-9_+#-]+)/gm;
+
+/**
+ * Extracts the language identifiers used by fenced code blocks in a string.
+ *
+ * @param markdown - Markdown source
+ * @returns Lower-cased identifiers, deduplicated
+ *
+ * @example
+ * ```typescript
+ * findFenceLanguages('```ts\nx\n```\n```python\ny\n```');
+ * // ['ts', 'python']
+ * ```
+ */
+export function findFenceLanguages(markdown: string): string[] {
+  const found = new Set<string>();
+
+  for (const match of markdown.matchAll(FENCE_PATTERN)) {
+    found.add(match[1].toLowerCase());
+  }
+
+  return [...found];
+}
+
+let memo: BuiltinLanguage[] | null = null;
+
+/**
+ * Returns the grammars to load for this site.
+ *
+ * Identifiers that Shiki does not recognise are dropped rather than passed
+ * through: an unknown name makes the highlighter throw at construction, which
+ * would turn one typo in one fence into a failed build. Unrecognised fences
+ * fall back to plain text at render time instead.
+ *
+ * @returns Bundled language names, sorted
+ */
+export function getUsedLanguages(): BuiltinLanguage[] {
+  const hit = cached(memo);
+  if (hit) return hit;
+
+  const known = new Set(Object.keys(bundledLanguages));
+  const wanted = new Set<string>(BASE_LANGUAGES);
+
+  for (const doc of getContentRegistry().docs) {
+    for (const language of findFenceLanguages(doc.content)) {
+      wanted.add(language);
+    }
+  }
+
+  // The filter is what makes the assertion sound: only names present in the
+  // bundle survive, and those are exactly the BuiltinLanguage values.
+  memo = [...wanted].filter((language) => known.has(language)).sort() as BuiltinLanguage[];
+  return memo;
+}
+
+/**
+ * Discards the memoised language list. Intended for tests.
+ */
+export function clearLanguageCache(): void {
+  memo = null;
+}

package/template/lib/markdown/rehype-plugins.ts

@@ -0,0 +1,240 @@
+import { visit } from 'unist-util-visit';
+import { toString } from 'hast-util-to-string';
+import type { Element, Root } from 'hast';
+import { docPathToUrl, type UrlMap } from '../navigation/url';
+
+/**
+ * Custom rehype plugins used by the build-time Markdown pipeline.
+ *
+ * These run against the HTML AST, after Markdown has been converted but before
+ * it is serialised, which is the only place where heading anchors, resolved
+ * links, and code-block chrome can all be produced without shipping a Markdown
+ * parser to the browser.
+ */
+
+/** A heading extracted from a document, used to render the table of contents. */
+export interface Heading {
+  /** Anchor id, matching the `id` attribute rehype-slug assigned */
+  id: string;
+  /** Rendered text of the heading */
+  text: string;
+  /** Heading level, 1 for h1 through 6 for h6 */
+  depth: number;
+}
+
+/** Heading levels collected for the table of contents. */
+const TOC_LEVELS = new Set(['h2', 'h3', 'h4']);
+
+/**
+ * Collects document headings onto `file.data.headings`.
+ *
+ * Results are attached to the virtual file rather than to a closure so that a
+ * single compiled processor can be reused across every document in the build —
+ * instantiating the syntax highlighter per page is the dominant cost otherwise.
+ *
+ * Must run after `rehype-slug`, which assigns the `id` attributes the table of
+ * contents links to. Headings without an id are skipped rather than given a
+ * generated one, since an anchor absent from the document is a dead link.
+ *
+ * @example
+ * ```typescript
+ * const file = await unified().use(rehypeSlug).use(rehypeCollectHeadings).process(md);
+ * const headings = file.data.headings as Heading[];
+ * ```
+ */
+export function rehypeCollectHeadings() {
+  return (tree: Root, file: { data: Record<string, unknown> }) => {
+    const collected: Heading[] = [];
+
+    visit(tree, 'element', (node: Element) => {
+      if (!TOC_LEVELS.has(node.tagName)) return;
+
+      const id = node.properties?.id;
+      if (typeof id !== 'string' || !id) return;
+
+      collected.push({
+        id,
+        text: toString(node),
+        depth: Number(node.tagName.slice(1)),
+      });
+    });
+
+    file.data.headings = collected;
+  };
+}
+
+/**
+ * Determines whether an href points outside the site.
+ */
+function isExternal(href: string): boolean {
+  return /^[a-z][a-z0-9+.-]*:/i.test(href) || href.startsWith('//');
+}
+
+/**
+ * Rewrites internal Markdown links to the site's configured URL form, and
+ * hardens external links.
+ *
+ * Authors write links the way the content tree looks — `[Setup](guides/setup)`
+ * — and this resolves them through the URL map, so the same Markdown works
+ * under either URL strategy. Anchors, external URLs, and links to unknown
+ * documents are left untouched.
+ *
+ * @param urlMap - Precomputed mapping from content paths to URL segments
+ */
+export function rehypeInternalLinks(urlMap: UrlMap) {
+  return (tree: Root) => {
+    visit(tree, 'element', (node: Element) => {
+      if (node.tagName !== 'a') return;
+
+      const href = node.properties?.href;
+      if (typeof href !== 'string' || !href) return;
+
+      if (isExternal(href)) {
+        node.properties.target = '_blank';
+        // `rel` is a space-separated list property in hast, so it is modelled
+        // as an array rather than a single string.
+        node.properties.rel = ['noopener', 'noreferrer'];
+        return;
+      }
+
+      // In-page anchors already point at ids produced by rehype-slug.
+      if (href.startsWith('#')) return;
+
+      // Split off any anchor or query so the document part can be resolved.
+      const match = /^([^#?]*)(.*)$/.exec(href);
+      if (!match) return;
+
+      const [, rawPath, suffix] = match;
+      const docPath = rawPath.replace(/^\/+/, '').replace(/\.md$/, '').replace(/\/+$/, '');
+      if (!docPath) return;
+
+      const url = docPathToUrl(urlMap, docPath);
+      if (url) {
+        // Trailing slash to match `trailingSlash` in the Next config: that is
+        // the form every page is exported under. Hosts that redirect the
+        // slashless form cost a round trip per link; hosts that do not — a
+        // plain object store, say — answer it with a 404.
+        node.properties.href = `/${url}/${suffix}`;
+      }
+    });
+  };
+}
+
+/**
+ * Wraps fenced code blocks in a container with a language label and copy button.
+ *
+ * Runs *before* the syntax highlighter so that the language recorded in the
+ * Markdown fence is still available; the highlighter then rewrites the inner
+ * `<pre>` in place. The copy button carries no inline handler — a single
+ * delegated listener on the client picks it up — which keeps the entire
+ * highlighting stack out of the browser bundle.
+ */
+export function rehypeCodeShell() {
+  return (tree: Root) => {
+    visit(tree, 'element', (node: Element, index, parent) => {
+      if (node.tagName !== 'pre' || !parent || index === undefined) return;
+
+      // Skip blocks that have already been wrapped.
+      if (
+        parent.type === 'element' &&
+        (parent as Element).properties?.['data-ezw-code'] !== undefined
+      ) {
+        return;
+      }
+
+      const code = node.children.find(
+        (child): child is Element => child.type === 'element' && child.tagName === 'code',
+      );
+
+      const className = code?.properties?.className;
+      const classes = Array.isArray(className) ? className.map(String) : [];
+      const languageClass = classes.find((cls) => cls.startsWith('language-'));
+      const language = languageClass ? languageClass.slice('language-'.length) : 'text';
+
+      const wrapper: Element = {
+        type: 'element',
+        tagName: 'div',
+        properties: {
+          className: ['ezw-code'],
+          'data-ezw-code': '',
+          'data-language': language,
+        },
+        children: [
+          {
+            type: 'element',
+            tagName: 'div',
+            properties: { className: ['ezw-code__bar'] },
+            children: [
+              {
+                type: 'element',
+                tagName: 'span',
+                properties: { className: ['ezw-code__lang'] },
+                children: [{ type: 'text', value: language }],
+              },
+              {
+                type: 'element',
+                tagName: 'button',
+                properties: {
+                  type: 'button',
+                  className: ['ezw-code__copy'],
+                  'data-ezw-copy': '',
+                  'aria-label': 'Copy code',
+                },
+                children: [{ type: 'text', value: 'Copy' }],
+              },
+            ],
+          },
+          node,
+        ],
+      };
+
+      parent.children[index] = wrapper;
+    });
+  };
+}
+
+/**
+ * Prefixes root-relative asset URLs with the deployment base path.
+ *
+ * Static exports served from a subdirectory — GitHub Pages project sites, for
+ * one — need `/images/x.png` rewritten to `/eziwiki/images/x.png`. Next rewrites
+ * these for JSX it controls, but not for HTML produced by this pipeline.
+ *
+ * @param basePath - Deployment base path, or '' when served from the root
+ */
+export function rehypeBasePath(basePath: string) {
+  return (tree: Root) => {
+    if (!basePath) return;
+
+    visit(tree, 'element', (node: Element) => {
+      const attr = node.tagName === 'a' ? 'href' : node.tagName === 'img' ? 'src' : null;
+      if (!attr) return;
+
+      const value = node.properties?.[attr];
+      if (typeof value !== 'string') return;
+      if (!value.startsWith('/') || value.startsWith('//')) return;
+      if (value.startsWith(`${basePath}/`)) return;
+
+      node.properties[attr] = `${basePath}${value}`;
+    });
+  };
+}
+
+/**
+ * Adds lazy loading and consistent styling hooks to content images.
+ */
+export function rehypeImages() {
+  return (tree: Root) => {
+    visit(tree, 'element', (node: Element) => {
+      if (node.tagName !== 'img') return;
+
+      node.properties ??= {};
+      node.properties.loading ??= 'lazy';
+      node.properties.decoding ??= 'async';
+
+      const className = node.properties.className;
+      const classes = Array.isArray(className) ? className.map(String) : [];
+      node.properties.className = [...classes, 'ezw-img'];
+    });
+  };
+}

package/template/lib/markdown/remark-wikilink.ts

@@ -0,0 +1,141 @@
+import { visit } from 'unist-util-visit';
+import type { Root, Text, PhrasingContent, Parent } from 'mdast';
+import { WIKILINK_PATTERN, parseWikiLink, type WikiLink } from './wikilink';
+
+/**
+ * Turns `[[wiki links]]` into ordinary Markdown links.
+ *
+ * Runs on the Markdown AST, before conversion to HTML, so the resulting links
+ * pass through the rest of the pipeline like any other. Only text nodes are
+ * visited, which means links written inside code spans and fenced blocks are
+ * left alone for free — documentation that explains the syntax has to be able
+ * to show it.
+ */
+
+/** What a target resolved to. */
+export interface WikiLinkTarget {
+  /** Root-relative href for the document */
+  url: string;
+  /** Default display text when the author gave no label */
+  title: string;
+}
+
+/**
+ * Resolves a wiki-link target to a destination, or null when there is none.
+ */
+export type WikiLinkResolver = (target: string) => WikiLinkTarget | null;
+
+/**
+ * Builds the replacement node for one wiki link.
+ *
+ * An unresolved link renders as marked-up text rather than an anchor: a link
+ * that goes nowhere is worse than visibly broken text, because it looks
+ * clickable and silently is not.
+ */
+function toNode(link: WikiLink, resolve: WikiLinkResolver): PhrasingContent {
+  // An anchor-only link points within the current page, so there is nothing to
+  // resolve.
+  if (!link.target && link.anchor) {
+    return {
+      type: 'link',
+      url: `#${link.anchor}`,
+      children: [{ type: 'text', value: link.label ?? link.anchor }],
+    };
+  }
+
+  const resolved = resolve(link.target);
+
+  if (!resolved) {
+    return {
+      // `emphasis` is a carrier for the rendered span: it is a known phrasing
+      // type, so it degrades to <em> if the hName hint is ever ignored.
+      type: 'emphasis',
+      data: {
+        hName: 'span',
+        hProperties: {
+          className: ['ezw-broken-link'],
+          title: `Unresolved link: ${link.target}`,
+        },
+      },
+      children: [{ type: 'text', value: link.label ?? link.target }],
+    };
+  }
+
+  return {
+    type: 'link',
+    url: link.anchor ? `${resolved.url}#${link.anchor}` : resolved.url,
+    data: {
+      hProperties: { className: ['ezw-wikilink'] },
+    },
+    children: [{ type: 'text', value: link.label ?? resolved.title }],
+  };
+}
+
+/**
+ * Splits a text node into text and link nodes.
+ *
+ * @param node - The text node to split
+ * @param resolve - Target resolver
+ * @returns Replacement nodes, or null when the text contains no wiki links
+ */
+function splitText(node: Text, resolve: WikiLinkResolver): PhrasingContent[] | null {
+  const { value } = node;
+  if (!value.includes('[[')) return null;
+
+  const replacement: PhrasingContent[] = [];
+  let cursor = 0;
+  let matched = false;
+
+  // matchAll on a global pattern is safe here because the regex literal is
+  // re-evaluated per call; lastIndex never leaks between documents.
+  for (const match of value.matchAll(WIKILINK_PATTERN)) {
+    const parsed = parseWikiLink(match[1], match[0]);
+    if (!parsed) continue;
+
+    const start = match.index ?? 0;
+
+    if (start > cursor) {
+      replacement.push({ type: 'text', value: value.slice(cursor, start) });
+    }
+
+    replacement.push(toNode(parsed, resolve));
+    cursor = start + match[0].length;
+    matched = true;
+  }
+
+  if (!matched) return null;
+
+  if (cursor < value.length) {
+    replacement.push({ type: 'text', value: value.slice(cursor) });
+  }
+
+  return replacement;
+}
+
+/**
+ * Remark plugin factory.
+ *
+ * @param resolve - Resolves a target to its destination
+ *
+ * @example
+ * ```typescript
+ * unified().use(remarkParse).use(remarkWikiLinks, (target) =>
+ *   target === 'intro' ? { url: '/intro', title: 'Introduction' } : null,
+ * );
+ * ```
+ */
+export function remarkWikiLinks(resolve: WikiLinkResolver) {
+  return (tree: Root) => {
+    visit(tree, 'text', (node: Text, index, parent) => {
+      if (!parent || index === undefined) return;
+
+      const replacement = splitText(node, resolve);
+      if (!replacement) return;
+
+      (parent as Parent).children.splice(index, 1, ...replacement);
+
+      // Continue after the nodes just inserted, so their text is not rescanned.
+      return index + replacement.length;
+    });
+  };
+}