@doc-kit/core 1.2.1 → 1.3.1

package/src/utils/configuration/index.mjs

@@ -1,8 +1,6 @@
 import { readFileSync } from 'node:fs';
-import { createRequire } from 'node:module';
 import { cpus } from 'node:os';
-import { dirname, isAbsolute, resolve } from 'node:path';
-import { pathToFileURL } from 'node:url';
+import { fileURLToPath } from 'node:url';
 import { isMainThread } from 'node:worker_threads';
 
 import { cosmiconfig } from 'cosmiconfig';
@@ -16,9 +14,16 @@ import logger from '#logger/index.mjs';
 import { parseChangelog, parseIndex } from '#parsers/markdown.mjs';
 import { enforceArray } from '#utils/array.mjs';
 import { leftHandAssign } from '#utils/generators.mjs';
+import { resolveSpecifier } from '#utils/loaders.mjs';
+import {
+  CONFIGURED_PLUGINS,
+  PLUGIN_LISTS,
+} from '#utils/markdown/constants.mjs';
+import { resolveMarkdown } from '#utils/markdown/plugins.mjs';
 import { deepMerge } from '#utils/misc.mjs';
 
-import { DEFAULT_CHUNK_SIZE, DEFAULT_MAX_THREADS } from './constants.mjs';
+import { DEFAULT_MAX_THREADS } from '../../threading/constants.mjs';
+import { DEFAULT_CHUNK_SIZE, DEFAULT_OUTPUT_DIRECTORY } from './constants.mjs';
 
 const configExplorer = cosmiconfig('doc-kit');
 
@@ -58,12 +63,14 @@ export const getDefaultConfig = (generators, config) =>
       global: {
         project: detectProject() ?? 'API Docs',
         version: process.version,
-        minify: true,
+        minify: false,
+        output: DEFAULT_OUTPUT_DIRECTORY,
         ref: 'HEAD',
         // Without release history there is nothing to build a version picker
         // from, so generators render single-version output.
         changelog: [],
         pathsToCopy: ['assets', 'public', 'static'],
+        markdown: { remarkPlugins: [], rehypePlugins: [], recmaPlugins: [] },
       },
 
       // The number of wasm memory instances is severely limited on
@@ -79,21 +86,26 @@ export const getDefaultConfig = (generators, config) =>
   );
 
 /**
- * Resolves an `extends` entry of a configuration file into an importable
- * URL: relative paths resolve against the configuration file, anything else
- * resolves as a package import specifier (e.g. `@node-core/doc-kit/config`).
+ * Resolves the Markdown plugins of a configuration (in `global`, and each
+ * generator's section) from the file declaring them.
  *
- * @param {string} specifier - The `extends` entry
- * @param {string} configFilePath - The configuration file it appears in
- * @returns {string} A `file:` URL to import
+ * @param {Partial<import('./types').Configuration>} config - The configuration
+ * @param {string} filePath - The file declaring it
+ * @returns {Partial<import('./types').Configuration>}
  */
-const resolveConfigExtends = (specifier, configFilePath) => {
-  if (specifier.startsWith('.') || isAbsolute(specifier)) {
-    return pathToFileURL(resolve(dirname(configFilePath), specifier)).href;
-  }
+const resolveMarkdownPlugins = (config, filePath) =>
+  Object.fromEntries(
+    Object.entries(config).map(([name, value]) => {
+      if (value?.markdown) {
+        const label = `${name}.markdown`;
+        const markdown = resolveMarkdown(value.markdown, label, filePath);
 
-  return pathToFileURL(createRequire(configFilePath).resolve(specifier)).href;
-};
+        return [name, { ...value, markdown }];
+      }
+
+      return [name, value];
+    })
+  );
 
 /**
  * Loads an explicit configuration file or searches for one using cosmiconfig.
@@ -112,15 +124,63 @@ export const loadConfigFile = async filePath => {
 
   let { extends: presets, ...config } = result.config ?? {};
 
+  config = resolveMarkdownPlugins(config, result.filepath);
+
   for (const preset of enforceArray(presets ?? []).toReversed()) {
-    const module = await import(resolveConfigExtends(preset, result.filepath));
+    const url = resolveSpecifier(preset, result.filepath);
+    const module = await import(url);
 
-    config = deepMerge(module.default ?? module, config);
+    // A preset's Markdown plugins resolve from the preset
+    config = deepMerge(
+      resolveMarkdownPlugins(module.default ?? module, fileURLToPath(url)),
+      config
+    );
   }
 
   return config;
 };
 
+/**
+ * Returns the Markdown plugins a generator takes: for each list of its
+ * pipeline with a `'...'`, the global ones, then its own. Generators rendering
+ * Markdown (with rehype or recma plugins) skip the global remark plugins,
+ * which already ran in `ast`. Its own plugins it doesn't take are ignored with
+ * a warning.
+ *
+ * @param {GeneratorMetadata} generator - The generator
+ * @param {Partial<import('./types').MarkdownConfiguration>} [markdown] - Its own plugins
+ * @param {import('./types').GlobalConfiguration} global - The global configuration
+ * @returns {Partial<import('./types').MarkdownConfiguration> | undefined}
+ */
+const configureMarkdown = (generator, markdown = {}, global) => {
+  const pipeline = generator.markdown ?? {};
+  const renders = Boolean(pipeline.rehypePlugins || pipeline.recmaPlugins);
+  const configured = {};
+
+  for (const list of PLUGIN_LISTS) {
+    if (!pipeline[list]?.includes(CONFIGURED_PLUGINS)) {
+      if (markdown[list]?.length > 0) {
+        logger.warn(
+          `Ignoring \`${generator.name}.markdown.${list}\`: ` +
+            `\`${generator.name}\` does not take them.`
+        );
+      }
+
+      continue;
+    }
+
+    const plugins = markdown[list] ?? [];
+
+    if (list === 'remarkPlugins' && renders) {
+      configured[list] = plugins;
+    } else {
+      configured[list] = [...global.markdown[list], ...plugins];
+    }
+  }
+
+  return generator.markdown && configured;
+};
+
 /**
  * Transforms configuration values that need async processing or coercion.
  * Only processes values that haven't been transformed yet (strings for changelog/index, non-coerced versions).
@@ -232,12 +292,20 @@ export const createRunConfiguration = async options => {
 
   // Now assign to each generator config (they inherit from global)
   await Promise.all(
-    [...generators.values()].map(async ({ name }) => {
-      const value = merged[name];
+    [...generators.values()].map(async generator => {
+      const value = merged[generator.name];
 
       // Transform generator-specific overrides
       await transformConfig(value);
 
+      // A generator's own Markdown plugins add to the global ones. Set even
+      // when undefined, so the assignment below doesn't copy the global ones
+      value.markdown = configureMarkdown(
+        generator,
+        value.markdown,
+        merged.global
+      );
+
       // Assign from global (this populates missing values from global)
       leftHandAssign(value, merged.global);
     })

package/src/utils/configuration/types.d.ts

@@ -1,4 +1,5 @@
 import type { SemVer } from 'semver';
+
 import type { ReleaseEntry } from '#parsers/types';
 
 export type Configuration = {
@@ -61,4 +62,30 @@ export type GlobalConfiguration = {
 
   // Git ref (i.e. HEAD)
   ref: string;
+
+  // Markdown plugins added to the pipeline of each generator processing
+  // Markdown. A generator's own `markdown` adds to them, rather than
+  // replacing them.
+  markdown: MarkdownConfiguration;
+};
+
+// A plugin's module specifier (a package name, or a path relative to the file
+// declaring it), alone or with its options
+export type PluginEntry = string | [specifier: string, options?: unknown];
+
+export type MarkdownConfiguration = {
+  // remark plugins, run on Markdown syntax trees
+  remarkPlugins: Array<PluginEntry>;
+
+  // rehype plugins, run on HTML syntax trees
+  rehypePlugins: Array<PluginEntry>;
+
+  // recma plugins, run on JavaScript syntax trees
+  recmaPlugins: Array<PluginEntry>;
+};
+
+// A generator's Markdown pipeline, where each list takes the configured
+// plugins in place of its `'...'`
+export type MarkdownPipeline = {
+  [List in keyof MarkdownConfiguration]?: Array<PluginEntry | '...'>;
 };

package/src/utils/glob.mjs

@@ -0,0 +1,42 @@
+import { globSync } from 'node:fs';
+import { relative, resolve, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+// A path segment holding any glob syntax (wildcards, classes, braces or extglobs)
+const GLOB_SEGMENT = /[*?[{]|[!+@]\(/;
+
+/**
+ * Finds the files matching glob patterns, as POSIX paths relative to `cwd`
+ * (absolute patterns included). Directories are left out.
+ *
+ * @param {string | string[]} patterns - Glob patterns to match
+ * @param {{ cwd?: string | URL, ignore?: string[] }} [options]
+ * @returns {string[]}
+ */
+export const findFiles = (patterns, { cwd = process.cwd(), ignore } = {}) => {
+  const root = cwd instanceof URL ? fileURLToPath(cwd) : cwd;
+
+  return globSync(patterns, { cwd: root, exclude: ignore, withFileTypes: true })
+    .filter(dirent => !dirent.isDirectory())
+    .map(dirent =>
+      relative(root, resolve(root, dirent.parentPath, dirent.name))
+        .split(sep)
+        .join('/')
+    );
+};
+
+/**
+ * Returns the directory part of a glob pattern before its first glob segment,
+ * e.g. `doc/api` for `doc/api/*.md`, or the parent directory of a plain path.
+ * Like `fs.globSync`, it reads backslashes as separators, never as escapes.
+ *
+ * @param {string} pattern - A glob pattern
+ * @returns {string}
+ */
+export const globParent = pattern => {
+  const segments = pattern.replaceAll('\\', '/').split('/');
+  const index = segments.findIndex(segment => GLOB_SEGMENT.test(segment));
+  const parent = segments.slice(0, index === -1 ? -1 : index).join('/');
+
+  return parent || (segments[0] === '' ? '/' : '.');
+};

package/src/utils/loaders.mjs

@@ -1,5 +1,6 @@
 import { readFile } from 'node:fs/promises';
-import { extname, isAbsolute, join } from 'node:path';
+import { createRequire } from 'node:module';
+import { dirname, extname, isAbsolute, join, resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 
 /**
@@ -28,6 +29,27 @@ export const loadFromURL = async url => {
   }
 };
 
+/**
+ * Resolves an import specifier into a `file:` URL: relative paths resolve
+ * against the file it appears in, anything else as a package import
+ * specifier (e.g. `@node-core/doc-kit/config`).
+ *
+ * @param {string} specifier - A path, a package specifier, or a `file:` URL
+ * @param {string} filePath - The file it appears in
+ * @returns {string} A `file:` URL to import
+ */
+export const resolveSpecifier = (specifier, filePath) => {
+  if (specifier.startsWith('file:')) {
+    return specifier;
+  }
+
+  if (specifier.startsWith('.') || isAbsolute(specifier)) {
+    return pathToFileURL(resolve(dirname(filePath), specifier)).href;
+  }
+
+  return pathToFileURL(createRequire(filePath).resolve(specifier)).href;
+};
+
 /**
  * Dynamically imports a module from a URL, using JSON import assertion if applicable.
  *

package/src/utils/markdown/constants.mjs

@@ -0,0 +1,7 @@
+'use strict';
+
+// The plugin lists of a Markdown pipeline, one for each syntax tree
+export const PLUGIN_LISTS = ['remarkPlugins', 'rehypePlugins', 'recmaPlugins'];
+
+// Marks where a list of a generator's pipeline takes the configured plugins
+export const CONFIGURED_PLUGINS = '...';

package/src/utils/markdown/plugins.mjs

@@ -0,0 +1,232 @@
+'use strict';
+
+import { fileURLToPath } from 'node:url';
+
+import { getGeneratorModule } from '#generators/loader.mjs';
+import { enforceArray } from '#utils/array.mjs';
+import { resolveSpecifier } from '#utils/loaders.mjs';
+import { isPlainObject } from '#utils/misc.mjs';
+
+import { CONFIGURED_PLUGINS, PLUGIN_LISTS } from './constants.mjs';
+
+/**
+ * The plugins of a Markdown pipeline, as `processor.use()` takes them.
+ *
+ * @typedef {Record<'remarkPlugins' | 'rehypePlugins' | 'recmaPlugins', import('unified').PluggableList>} MarkdownPlugins
+ */
+
+// The pipeline each generator runs on this thread, by its name
+const loadedPipelines = new Map();
+
+/**
+ * Resolves the module specifiers of Markdown plugins (in the `markdown`
+ * option, or a generator's pipeline) from the file declaring them into
+ * `file:` URLs, which any thread can import.
+ *
+ * @param {import('../configuration/types').MarkdownPipeline} markdown - The plugins
+ * @param {string} label - Where they are declared, for errors (e.g. `global.markdown`)
+ * @param {string} filePath - The file declaring them
+ * @returns {import('../configuration/types').MarkdownPipeline}
+ */
+export const resolveMarkdown = (markdown, label, filePath) => {
+  const resolved = { ...markdown };
+
+  for (const list of PLUGIN_LISTS) {
+    resolved[list] &&= enforceArray(markdown[list]).map((entry, index) => {
+      if (entry === CONFIGURED_PLUGINS) {
+        return entry;
+      }
+
+      const [specifier, ...options] = enforceArray(entry);
+
+      if (typeof specifier !== 'string') {
+        throw new TypeError(
+          `${label}.${list}[${index}] in ${filePath} must be a module ` +
+            'specifier (a package name or a path), as Markdown is processed ' +
+            'in worker threads, which import the plugins themselves.'
+        );
+      }
+
+      const url = resolveSpecifier(specifier, filePath);
+
+      return options.length === 0 ? url : [url, ...options];
+    });
+  }
+
+  return resolved;
+};
+
+/**
+ * Resolves the module specifiers of a generator's Markdown pipeline from the
+ * module it was loaded from.
+ *
+ * @param {GeneratorMetadata} generator - The generator
+ * @returns {import('../configuration/types').MarkdownPipeline}
+ */
+const resolveMarkdownPipeline = generator => {
+  const module = getGeneratorModule(generator);
+
+  return resolveMarkdown(
+    generator.markdown,
+    `${generator.name}.markdown`,
+    module && fileURLToPath(module)
+  );
+};
+
+/**
+ * Merges options into others: arrays add up, plain objects merge, and other
+ * values replace the ones they are merged into.
+ *
+ * @param {unknown} options - The options merged into
+ * @param {unknown} added - The options merged
+ * @returns {unknown}
+ */
+const mergeOptions = (options, added) => {
+  if (Array.isArray(options) && Array.isArray(added)) {
+    return [...options, ...added];
+  }
+
+  if (!isPlainObject(options) || !isPlainObject(added)) {
+    return added;
+  }
+
+  const merged = { ...options };
+
+  for (const [key, value] of Object.entries(added)) {
+    merged[key] = mergeOptions(options[key], value);
+  }
+
+  return merged;
+};
+
+/**
+ * Builds a list of a pipeline, with the configured plugins in place of its
+ * `'...'`. A configured plugin the list already has, or added earlier, isn't
+ * added again: its options merge into the ones it has.
+ *
+ * @param {Array<import('../configuration/types').PluginEntry | '...'>} [own] - The list
+ * @param {Array<import('../configuration/types').PluginEntry>} [configured] - The configured plugins
+ * @returns {Array<{ entry: import('../configuration/types').PluginEntry, added: boolean }>}
+ */
+const configureList = (own = [], configured = []) => {
+  const plugins = own.map(entry => ({ entry, added: false }));
+  const added = [];
+
+  // Each plugin by its specifier: the first listing of it takes the options
+  const listed = new Map();
+
+  for (const plugin of plugins) {
+    const [specifier] = enforceArray(plugin.entry);
+
+    if (!listed.has(specifier)) {
+      listed.set(specifier, plugin);
+    }
+  }
+
+  for (const entry of configured) {
+    const [specifier, options] = enforceArray(entry);
+    const plugin = listed.get(specifier);
+
+    if (!plugin) {
+      const addedPlugin = { entry, added: true };
+
+      added.push(addedPlugin);
+      listed.set(specifier, addedPlugin);
+    } else if (options !== undefined) {
+      const [, listedOptions] = enforceArray(plugin.entry);
+
+      plugin.entry = [specifier, mergeOptions(listedOptions, options)];
+    }
+  }
+
+  return plugins.flatMap(plugin =>
+    plugin.entry === CONFIGURED_PLUGINS ? added : [plugin]
+  );
+};
+
+/**
+ * Imports a plugin as `processor.use()` takes it. Its module default-exports
+ * the plugin, a list of plugins, or a preset, or exports an async `load`,
+ * taking the plugin's options and returning it, for setup unified plugins
+ * can't do themselves.
+ *
+ * @param {import('../configuration/types').PluginEntry} entry - The plugin
+ * @returns {Promise<import('unified').Pluggable>}
+ */
+const importPlugin = async entry => {
+  const [specifier, ...options] = enforceArray(entry);
+  const { default: plugin, load } = await import(specifier);
+
+  if (typeof load === 'function') {
+    return [await load(...options)];
+  }
+
+  if (typeof plugin === 'function') {
+    return [plugin, ...options];
+  }
+
+  // A list is used as a preset, not to be taken for a [plugin, options] pair
+  return Array.isArray(plugin) ? { plugins: plugin } : plugin;
+};
+
+/**
+ * Loads a generator's Markdown pipeline on the current thread, with the
+ * plugins configured for it. Loading the same configuration again does
+ * nothing.
+ *
+ * @param {GeneratorMetadata} generator - The generator
+ * @param {Partial<import('../configuration/types').MarkdownConfiguration>} [markdown] - The plugins configured for it
+ * @returns {Promise<void>}
+ */
+export const loadMarkdownPlugins = async (generator, markdown = {}) => {
+  if (!generator.markdown) {
+    return;
+  }
+
+  const key = JSON.stringify(markdown);
+  const loaded = loadedPipelines.get(generator.name);
+
+  if (loaded?.generator === generator && loaded.key === key) {
+    return;
+  }
+
+  const pipeline = resolveMarkdownPipeline(generator);
+  const configured = {};
+  const own = {};
+
+  // The lists import at once, so a slow plugin (Shiki) doesn't hold the rest
+  const loading = PLUGIN_LISTS.map(async list => {
+    const plugins = configureList(pipeline[list], markdown[list]);
+    const imports = plugins.map(({ entry }) => importPlugin(entry));
+
+    const imported = await Promise.all(imports);
+
+    configured[list] = imported;
+    own[list] = imported.filter((_, index) => !plugins[index].added);
+  });
+
+  await Promise.all(loading);
+
+  loadedPipelines.set(generator.name, { generator, key, configured, own });
+};
+
+/**
+ * Gets the plugins of a generator's Markdown pipeline, as loaded on the
+ * current thread: with the configured plugins, or only its own (with their
+ * configured options).
+ *
+ * @param {string} generator - The name of the generator
+ * @param {boolean} [configured] - Whether with the configured plugins
+ * @returns {MarkdownPlugins}
+ */
+export const getMarkdownPlugins = (generator, configured = true) => {
+  const loaded = loadedPipelines.get(generator);
+
+  if (!loaded) {
+    throw new Error(
+      `The Markdown pipeline of "${generator}" is not loaded on this thread`
+    );
+  }
+
+  return configured ? loaded.configured : loaded.own;
+};

package/src/utils/markdown/processor.mjs

@@ -0,0 +1,60 @@
+'use strict';
+
+import remarkMdx from 'remark-mdx';
+import { unified } from 'unified';
+
+import { PLUGIN_LISTS } from './constants.mjs';
+import { getMarkdownPlugins } from './plugins.mjs';
+
+// The processors of each loaded pipeline, of Markdown and of MDX
+const processors = new WeakMap();
+
+/**
+ * Creates the processor of a pipeline: its remark plugins, then its rehype
+ * ones, then its recma ones. A processor of MDX knows its syntax too, and
+ * tells plugins it processes MDX through `this.data('mdx')`.
+ *
+ * @param {import('./plugins.mjs').MarkdownPlugins} plugins - The pipeline
+ * @param {boolean} mdx - Whether it processes MDX
+ * @returns {import('unified').Processor}
+ */
+const createProcessor = (plugins, mdx) => {
+  const processor = unified().data('mdx', mdx);
+
+  if (mdx) {
+    processor.use(remarkMdx);
+  }
+
+  for (const list of PLUGIN_LISTS) {
+    processor.use(plugins[list]);
+  }
+
+  return processor;
+};
+
+/**
+ * Gets the processor of a generator's Markdown pipeline, as loaded on the
+ * current thread (see `loadMarkdownPlugins`).
+ *
+ * @param {string} generator - The name of the generator
+ * @param {Object} [options]
+ * @param {boolean} [options.mdx] - Whether it processes MDX
+ * @param {boolean} [options.configured] - Whether it runs the configured
+ * plugins, rather than only the generator's own
+ * @returns {import('unified').Processor}
+ */
+export const getProcessor = (
+  generator,
+  { mdx = false, configured = true } = {}
+) => {
+  const plugins = getMarkdownPlugins(generator, configured);
+
+  if (!processors.has(plugins)) {
+    processors.set(plugins, {});
+  }
+
+  const created = processors.get(plugins);
+  const syntax = mdx ? 'mdx' : 'markdown';
+
+  return (created[syntax] ??= createProcessor(plugins, mdx));
+};