@doc-kit/core 1.2.1 → 1.3.0

package/src/plugins/shiki/highlighter.mjs

@@ -0,0 +1,373 @@
+'use strict';
+
+import { createRequire } from 'node:module';
+import { endianness } from 'node:os';
+
+import createSyntaxHighlighter from '@node-core/rehype-shiki/highlighter';
+import { isSpecialLang } from 'shiki/core';
+import { bundledLanguagesInfo } from 'shiki/langs';
+import { bundledThemes } from 'shiki/themes';
+
+import { importFromURL } from '#utils/loaders.mjs';
+import { getMarkdownPlugins } from '#utils/markdown/plugins.mjs';
+import { lazy } from '#utils/misc.mjs';
+
+const require = createRequire(import.meta.url);
+
+/**
+ * A language a highlighter highlights.
+ *
+ * @typedef {Pick<import('shiki').LanguageRegistration, 'name' | 'displayName' | 'aliases'>} Language
+ */
+
+/**
+ * A syntax highlighter, creating its Shiki instance on first use.
+ *
+ * @typedef {Object} SyntaxHighlighter
+ * @property {import('shiki').HighlighterCore} shiki - The Shiki instance
+ * @property {(languageId?: string) => string} resolveLanguage - Resolves a language, falling back to plain text for unknown ones
+ * @property {(code: string, lang: string, meta?: Record<string, unknown>) => string} highlightToHtml - Highlights code, returning the inner HTML of its `<code>` element
+ * @property {(code: string, lang: string, meta?: Record<string, unknown>) => ReturnType<import('shiki').HighlighterCore['codeToHast']>} highlightToHast - Highlights code, returning a HAST tree
+ * @property {Array<Language>} langs - The languages it highlights
+ */
+
+/**
+ * Creates Shiki's regular expression engine: the wasm (Oniguruma) one where
+ * it can, the JavaScript one otherwise.
+ *
+ * @returns {Promise<import('shiki').RegexEngine>}
+ */
+const createEngine = async () => {
+  // riscv64 with sv39 has limited virtual memory space, where creating
+  // too many (>20) wasm memory instances fails.
+  // https://github.com/nodejs/node/pull/60591
+  //
+  // The wasm highlighter is currently not compatible with big endian.
+  // https://github.com/nodejs/node/pull/62512#issuecomment-4243469950
+  if (process.arch !== 'riscv64' && endianness() === 'LE') {
+    const { createOnigurumaEngine } = await import('shiki/engine/oniguruma');
+
+    return createOnigurumaEngine(import('shiki/wasm'));
+  }
+
+  const { createJavaScriptRegexEngine } =
+    await import('shiki/engine/javascript');
+
+  // Not every bundled grammar compiles under the JavaScript engine,
+  // so skip the patterns it cannot handle instead of throwing.
+  return createJavaScriptRegexEngine({ forgiving: true });
+};
+
+// The regular expression engine of this thread, for all its highlighters
+let engine;
+
+// The highlighters of the options given, by their JSON
+const highlighters = new Map();
+
+// What code in an unknown language is highlighted as
+const FALLBACK_LANGUAGE = 'text';
+
+// The languages Shiki bundles. Registering a grammar takes time and memory,
+// and slows down every highlight after it, so each one is only imported and
+// registered once code in it is highlighted (see `resolveLanguage`).
+const BUNDLED_LANGUAGES = bundledLanguagesInfo.map(({ id, name, aliases }) => ({
+  name: id,
+  displayName: name,
+  aliases,
+}));
+
+// The bundled language each of their names and aliases stands for, as code
+// names its language by either
+const BUNDLED_NAMES = new Map(
+  BUNDLED_LANGUAGES.flatMap(({ name, aliases = [] }) =>
+    [name, ...aliases].map(alias => [alias, name])
+  )
+);
+
+/**
+ * Imports a bundled language: its grammar, and those of the languages it
+ * embeds, lazily too. Shiki only highlights a language embedded lazily (a
+ * fenced code block in Markdown) once it is registered, so registering it
+ * later would make code highlight differently depending on what was
+ * highlighted before it.
+ *
+ * For the same reason, a grammar injecting into others' (`ts-tags`', into
+ * JavaScript's tagged templates) only injects into the language it was
+ * imported for: Shiki only applies an injection to the grammars it creates
+ * after it. A language given in `langs` keeps its injections.
+ *
+ * @param {string} name - A name or alias of the language
+ * @returns {Array<import('shiki').LanguageRegistration>}
+ */
+const importBundledLanguage = name => {
+  const id = BUNDLED_NAMES.get(name);
+  const langs = new Map();
+  const pending = [id];
+
+  while (pending.length > 0) {
+    const grammars = require(`shiki/langs/${pending.pop()}.mjs`).default;
+
+    for (const lang of grammars) {
+      if (langs.has(lang.name)) {
+        continue;
+      }
+
+      langs.set(lang.name, lang);
+
+      const { embeddedLangs = [], embeddedLangsLazy = [] } = lang;
+
+      for (const embedded of [...embeddedLangs, ...embeddedLangsLazy]) {
+        if (BUNDLED_NAMES.has(embedded)) {
+          pending.push(BUNDLED_NAMES.get(embedded));
+        }
+      }
+    }
+  }
+
+  const { scopeName } = langs.get(id);
+
+  return [...langs.values()].map(lang =>
+    lang.injectTo ? { ...lang, injectTo: [scopeName] } : lang
+  );
+};
+
+/**
+ * Adds the bundled languages that languages embed, which Shiki registers
+ * along with them.
+ *
+ * @param {Array<import('shiki').LanguageRegistration>} langs - The languages
+ * @returns {Array<import('shiki').LanguageRegistration>}
+ */
+const withEmbeddedLanguages = langs => {
+  const names = new Set(langs.map(({ name }) => name));
+
+  const embedded = langs
+    .flatMap(({ embeddedLangs = [], embeddedLangsLazy = [] }) => [
+      ...embeddedLangs,
+      ...embeddedLangsLazy,
+    ])
+    .filter(name => !names.has(name) && BUNDLED_NAMES.has(name))
+    .flatMap(importBundledLanguage);
+
+  return [...embedded, ...langs];
+};
+
+/**
+ * Imports a list of options, each given as it is, or as a module (a path
+ * relative to the working directory, or a URL) default-exporting it, or a list
+ * of them.
+ *
+ * @template T
+ * @param {Array<string | T>} options - The options
+ * @returns {Promise<Array<T>>}
+ */
+const importList = async options => {
+  const imports = options.map(option =>
+    typeof option === 'string' ? importFromURL(option) : option
+  );
+
+  const imported = await Promise.all(imports);
+
+  return imported.flat();
+};
+
+/**
+ * Imports a theme: the name of one Shiki bundles, a module default-exporting
+ * one, or the theme itself. Shiki tells themes apart by their name, so a theme
+ * without one is named after its color scheme.
+ *
+ * @param {string | import('shiki').ThemeRegistration} theme - The theme
+ * @param {'light' | 'dark'} scheme - Its color scheme
+ * @returns {Promise<import('shiki').ThemeRegistration>}
+ */
+const importTheme = async (theme, scheme) => {
+  let imported = theme;
+
+  if (typeof theme === 'string') {
+    if (theme in bundledThemes) {
+      const { default: bundledTheme } = await bundledThemes[theme]();
+
+      imported = bundledTheme;
+    } else {
+      imported = await importFromURL(theme);
+    }
+  }
+
+  return { name: `custom-${scheme}`, ...imported };
+};
+
+/**
+ * Imports the options of a highlighter, and creates it.
+ *
+ * @param {import('./rehype.mjs').ShikiOptions} options - The options
+ * @returns {Promise<SyntaxHighlighter>}
+ */
+const importHighlighter = async ({
+  langs = [],
+  langAlias = {},
+  themes,
+  transformers = [],
+}) => {
+  engine ??= createEngine();
+
+  const [regexEngine, importedLangs, importedTransformers] = await Promise.all([
+    engine,
+    importList(langs),
+    importList(transformers),
+  ]);
+
+  const coreOptions = {
+    engine: regexEngine,
+    // The bundled languages are registered as code uses them
+    langs: withEmbeddedLanguages(importedLangs),
+    // A copy, as Shiki adds the aliases of the languages it registers to it
+    langAlias: { ...langAlias },
+  };
+
+  // Without themes of its own, the highlighter has a default light and dark one
+  if (themes) {
+    const [light, dark] = await Promise.all([
+      importTheme(themes.light, 'light'),
+      importTheme(themes.dark, 'dark'),
+    ]);
+
+    coreOptions.themes = [light, dark];
+  }
+
+  /**
+   * Creates the Shiki instance, on first use, and the options it highlights
+   * with.
+   */
+  const load = lazy(() => {
+    const { shiki } = createSyntaxHighlighter({ coreOptions });
+
+    // The themes are given by name: Shiki keeps the themes it loaded parsed,
+    // but parses a theme object it's given again for every highlight
+    const [light, dark] =
+      coreOptions.themes?.map(theme => theme.name) ?? shiki.getLoadedThemes();
+
+    const highlightOptions = {
+      themes: { light, dark },
+      defaultColor: 'light',
+      transformers: importedTransformers,
+    };
+
+    return { shiki, highlightOptions };
+  });
+
+  /**
+   * Resolves a language, falling back to plain text for unknown ones. A
+   * bundled language is registered the first time it's resolved.
+   *
+   * @param {string} [languageId]
+   * @returns {string}
+   */
+  const resolveLanguage = languageId => {
+    if (!languageId) {
+      return FALLBACK_LANGUAGE;
+    }
+
+    const instance = load().shiki;
+    const name = instance.resolveLangAlias(languageId.toLowerCase());
+
+    if (isSpecialLang(name) || instance.getLoadedLanguages().includes(name)) {
+      return languageId;
+    }
+
+    if (!BUNDLED_NAMES.has(name)) {
+      return FALLBACK_LANGUAGE;
+    }
+
+    instance.loadLanguageSync(importBundledLanguage(name));
+
+    return languageId;
+  };
+
+  /**
+   * The options Shiki highlights code with.
+   *
+   * @param {string} lang - The language of the code
+   * @param {Record<string, unknown>} meta - Its metadata
+   */
+  const optionsFor = (lang, meta) => ({
+    lang: resolveLanguage(lang),
+    ...load().highlightOptions,
+    meta,
+  });
+
+  return {
+    langs: [...BUNDLED_LANGUAGES, ...importedLangs],
+
+    /**
+     * The Shiki instance.
+     */
+    get shiki() {
+      return load().shiki;
+    },
+
+    resolveLanguage,
+
+    /**
+     * Highlights code, returning the inner HTML of its `<code>` element.
+     *
+     * @param {string} code - The code
+     * @param {string} lang - Its language
+     * @param {Record<string, unknown>} [meta] - Its metadata
+     */
+    highlightToHtml: (code, lang, meta = {}) =>
+      load()
+        .shiki.codeToHtml(code, optionsFor(lang, meta))
+        // Shiki wraps the highlighted code in a <pre> and a <code>
+        .match(/<code>(.+?)<\/code>/s)[1],
+
+    /**
+     * Highlights code, returning a HAST tree.
+     *
+     * @param {string} code - The code
+     * @param {string} lang - Its language
+     * @param {Record<string, unknown>} [meta] - Its metadata
+     */
+    highlightToHast: (code, lang, meta = {}) =>
+      load().shiki.codeToHast(code, optionsFor(lang, meta)),
+  };
+};
+
+/**
+ * Creates a highlighter of every language Shiki bundles, with the given
+ * options (see `./rehype.mjs`). Its Shiki instance is created on first use,
+ * each bundled language is registered once code in it is highlighted, and the
+ * same options give the same highlighter.
+ *
+ * @param {import('./rehype.mjs').ShikiOptions} [options] - The options
+ * @returns {Promise<SyntaxHighlighter>}
+ */
+export const createHighlighter = (options = {}) => {
+  const key = JSON.stringify(options);
+
+  if (!highlighters.has(key)) {
+    highlighters.set(key, importHighlighter(options));
+  }
+
+  return highlighters.get(key);
+};
+
+/**
+ * Gets the highlighter of the Shiki plugin of a generator's Markdown pipeline,
+ * as loaded on the current thread, to highlight code as the pipeline does.
+ *
+ * @param {string} generator - The name of the generator
+ * @returns {SyntaxHighlighter}
+ */
+export const getHighlighter = generator => {
+  const shiki = getMarkdownPlugins(generator).rehypePlugins.find(
+    plugin => Array.isArray(plugin) && plugin[0].highlighter
+  );
+
+  if (!shiki) {
+    throw new Error(
+      `The Markdown pipeline of "${generator}" does not highlight code`
+    );
+  }
+
+  return shiki[0].highlighter;
+};

package/src/plugins/shiki/rehype.mjs

@@ -0,0 +1,38 @@
+'use strict';
+
+import rehypeShikiji from '@node-core/rehype-shiki/plugin';
+
+import { createHighlighter } from './highlighter.mjs';
+
+/**
+ * The options of the Shiki plugin. Its modules are paths relative to the
+ * working directory, or URLs.
+ *
+ * @typedef {Object} ShikiOptions
+ * @property {Array<string | import('shiki').LanguageRegistration>} [langs] - More languages: grammars, or modules default-exporting grammars
+ * @property {Record<string, string>} [langAlias] - Aliases of languages, e.g. `{ conf: 'ini' }`
+ * @property {{ light: string | import('shiki').ThemeRegistration, dark: string | import('shiki').ThemeRegistration }} [themes] - The light and dark themes: names of themes Shiki bundles, modules default-exporting themes, or themes
+ * @property {Array<string>} [transformers] - Modules default-exporting Shiki transformers, or lists of them
+ */
+
+/**
+ * Loads the rehype plugin highlighting code blocks with Shiki, in every
+ * language it bundles. The plugin exposes its highlighter as `highlighter`
+ * (see `getHighlighter`).
+ *
+ * @param {ShikiOptions} [options] - The options
+ * @returns {Promise<import('unified').Plugin & { highlighter: import('./highlighter.mjs').SyntaxHighlighter }>}
+ */
+export async function load(options) {
+  const highlighter = await createHighlighter(options);
+  const transformer = await rehypeShikiji({ highlighter });
+
+  /**
+   * Highlights the code blocks of a tree.
+   */
+  const shiki = () => transformer;
+
+  shiki.highlighter = highlighter;
+
+  return shiki;
+}

package/src/{utils/type-annotations/highlighted.mjs → plugins/type-annotations/highlighter.mjs}

@@ -1,38 +1,33 @@
 'use strict';
 
-import { highlighter } from '#utils/highlighter.mjs';
-
 import { typeAnnotationToHast } from './hast.mjs';
 
-// Kept apart from `./hast.mjs` on purpose: importing this module loads Shiki
-// (every grammar plus the regex engine), which only the pipelines that
-// highlight should pay for. The `ast` and `metadata` stages never do.
-const [lightTheme, darkTheme] = highlighter.shiki.getLoadedThemes();
-
 /**
- * Syntax-highlighted mdast→hast handler for `typeAnnotation` nodes, used by
- * the web (JSX) pipeline. The whole type is highlighted as one inline
- * fragment, and each resolved identifier's exact character range is wrapped
- * in an `<a>` via Shiki decorations. Values that are not TypeScript (display
- * names such as `HTTP/2 Headers Object`) are highlighted as plain text, so
- * their prose is not coloured as operators and numeric literals.
+ * Creates the syntax-highlighted mdast→hast handler for `typeAnnotation`
+ * nodes, used by the web (JSX) pipeline. The whole type is highlighted as one
+ * inline fragment, and each resolved identifier's exact character range is
+ * wrapped in an `<a>` via Shiki decorations. Values that are not TypeScript
+ * (display names such as `HTTP/2 Headers Object`) are highlighted as plain
+ * text, so their prose is not coloured as operators and numeric literals.
  *
  * Falls back to the minimal handler when the type failed to parse or nothing
  * resolved (no point paying for highlighting then).
  *
- * @param {import('mdast-util-to-hast').State} state
- * @param {import('mdast').Node} node
- * @returns {import('hast').Element}
+ * @param {() => import('#plugins/shiki/highlighter.mjs').SyntaxHighlighter} getHighlighter - Gives the highlighter, once a type is highlighted
+ * @returns {(state: import('mdast-util-to-hast').State, node: import('mdast').Node) => import('hast').Element}
  */
-export const typeAnnotationToHighlightedHast = (state, node) => {
+export const createTypeAnnotationHandler = getHighlighter => (state, node) => {
   const links = node.data?.links ?? [];
 
   if (node.data?.parseError || links.length === 0) {
     return typeAnnotationToHast(state, node);
   }
 
-  const root = highlighter.shiki.codeToHast(node.value, {
-    lang: node.data?.typescript ? 'typescript' : 'text',
+  const { shiki, resolveLanguage } = getHighlighter();
+  const [lightTheme, darkTheme] = shiki.getLoadedThemes();
+
+  const root = shiki.codeToHast(node.value, {
+    lang: resolveLanguage(node.data?.typescript ? 'typescript' : 'text'),
     themes: { light: lightTheme, dark: darkTheme },
     decorations: links.map(({ start, end, href }) => ({
       start,

package/src/{utils → plugins}/type-annotations/remark.mjs

@@ -10,12 +10,16 @@ import { typeAnnotationSyntax } from './syntax.mjs';
  * Remark plugin that teaches the parser to treat any balanced `{...}` span in
  * text as a `typeAnnotation` node whose value is a TypeScript type expression.
  *
- * Only registered on the non-MDX pipeline — in MDX, `{...}` is a real
- * expression and is handled by remark-mdx instead.
+ * It does nothing in MDX, where `{...}` is a real expression, handled by
+ * remark-mdx instead.
  *
  * @this {import('unified').Processor}
  */
 export default function remarkTypeAnnotations() {
+  if (this.data('mdx')) {
+    return;
+  }
+
   const data = this.data();
 
   (data.micromarkExtensions ??= []).push(typeAnnotationSyntax());

package/src/threading/chunk-worker.mjs

@@ -3,6 +3,7 @@ import { workerData } from 'node:worker_threads';
 import { loadGenerator } from '#generators/loader.mjs';
 import logger from '#logger/index.mjs';
 import { setConfig } from '#utils/configuration/index.mjs';
+import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs';
 
 if (workerData?.logLevel !== undefined) {
   logger.setLogLevel(workerData.logLevel);
@@ -26,5 +27,8 @@ export default async ({
 
   const generator = await loadGenerator(generatorSpecifier);
 
+  // Plugins can't be sent to workers, which import the pipeline themselves
+  await loadMarkdownPlugins(generator, configuration[generator.name]?.markdown);
+
   return generator.processChunk(input, itemIndices, extra);
 };

package/src/threading/constants.mjs

@@ -0,0 +1,10 @@
+'use strict';
+
+/**
+ * The heap size limit of each worker thread (V8's old space), in MB. V8 lets a
+ * heap grow to several times its live data before collecting it, the more the
+ * higher its limit, so every worker held several times what it was using. The
+ * biggest page of the Node.js docs, `all.html`, takes ~300MB. Node.js's
+ * `--max-old-space-size` replaces it.
+ */
+export const WORKER_HEAP_SIZE = 512;

package/src/threading/index.mjs

@@ -2,6 +2,8 @@ import Piscina from 'piscina';
 
 import logger from '#logger/index.mjs';
 
+import { WORKER_HEAP_SIZE } from './constants.mjs';
+
 const poolLogger = logger.child('WorkerPool');
 
 const workerScript = import.meta.resolve('./chunk-worker.mjs');
@@ -22,7 +24,10 @@ export default function createWorkerPool(threads) {
     filename: workerScript,
     minThreads: 0,
     maxThreads: threads,
-    idleTimeout: 1_000,
+    // A worker idle for 250ms ends, so its heap is gone during a long stretch
+    // of work on the main thread, such as bundling the site
+    idleTimeout: 250,
+    resourceLimits: { maxOldGenerationSizeMb: WORKER_HEAP_SIZE },
     workerData: { logLevel: logger.getLogLevel() },
   });
 }

package/src/utils/configuration/constants.mjs

@@ -1,9 +1,9 @@
 'use strict';
 
 /**
- * The default `threads` ceiling. Each worker that highlights code holds Shiki's
- * grammars and regex engine (~300MB) on top of the pages it is building, so
- * past a few threads memory, not CPU, is what runs out. `--threads` raises it.
+ * The default `threads` ceiling. Each worker holds a heap of its own, with the
+ * libraries and the pages it is working on, so past a few threads memory, not
+ * CPU, is what runs out. `--threads` raises it.
  */
 export const DEFAULT_MAX_THREADS = 4;