@doc-kit/core 1.2.0 → 1.3.0

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @doc-kit/core
 
+## 1.3.0
+
+### Minor Changes
+
+- [#1156](https://github.com/nodejs/doc-kit/pull/1156) [`6d5f9d1`](https://github.com/nodejs/doc-kit/commit/6d5f9d17d9f265c62db0f878c4e03903881b11c9) Thanks [@ovflowd](https://github.com/ovflowd)! - Builds take about half the memory and a third of the time (measured on the Node.js API docs): Shiki registers each language once code in it is highlighted, highlighted code reaches pages as static markup rather than a tree per token, `all.html` is rendered by the worker pool, output is no longer minified by default (serve it compressed instead), idle workers end after 250ms, and each worker's heap is limited to 512MB, which Node.js's `--max-old-space-size` raises.
+
+- [#1127](https://github.com/nodejs/doc-kit/pull/1127) [`c50e75f`](https://github.com/nodejs/doc-kit/commit/c50e75f8f50c84b6ff3b16b84059bd0f53ca5d1c) Thanks [@ovflowd](https://github.com/ovflowd)! - Add the `llms-txt-full` generator, writing a `llms-full.txt` file holding the Markdown of every page, each preceded by its URL.
+
+- [#1157](https://github.com/nodejs/doc-kit/pull/1157) [`a4c9684`](https://github.com/nodejs/doc-kit/commit/a4c9684fd46f14b3bcbaf2e8f01a246a47ee27cd) Thanks [@ovflowd](https://github.com/ovflowd)! - Add a `markdown` option to add remark, rehype, and recma plugins to the generators processing Markdown, or configure the ones they use, such as Shiki. The `@doc-kit/core` modules the generators' pipelines replace are removed (`utils/remark.mjs`, `utils/remark-shiki.mjs`, and `utils/highlighter.mjs`), and `utils/type-annotations` moves to `plugins/type-annotations`
+
+## 1.2.1
+
+### Patch Changes
+
+- [#1105](https://github.com/nodejs/doc-kit/pull/1105) [`615aabe`](https://github.com/nodejs/doc-kit/commit/615aabeb1955a8028146e26fd4589c1172f58367) Thanks [@ryzrr](https://github.com/ryzrr)! - fix(metadata): link class types whose name starts with an acronym
+
 ## 1.2.0
 
 ### Minor Changes

package/README.md

@@ -15,7 +15,7 @@ debugging-only `json-simple`); the rest come from companion packages:
 
 | Package                                                                                | Generators                                                                            |
 | -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- |
-| [`@doc-kit/generator-react`](https://www.npmjs.com/package/@doc-kit/generator-react)   | `html` (the modern site), `orama-db`, `llms-txt`, `sitemap`                           |
+| [`@doc-kit/generator-react`](https://www.npmjs.com/package/@doc-kit/generator-react)   | `html` (the modern site), `orama-db`, `llms-txt`, `llms-txt-full`, `sitemap`          |
 | [`@node-core/doc-kit-legacy`](https://www.npmjs.com/package/@node-core/doc-kit-legacy) | `legacy-html`, `legacy-html-all`, `legacy-json`, `legacy-json-all` (Node.js-specific) |
 | [`@node-core/doc-kit`](https://www.npmjs.com/package/@node-core/doc-kit)               | `man-page`, `api-links`, `addon-verify` (Node.js-specific)                            |
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@doc-kit/core",
   "type": "module",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "The doc-kit engine: the pipeline and shared APIs that power the doc-kit generators and CLI",
   "repository": {
     "type": "git",
@@ -33,6 +33,7 @@
       "./src/parsers/*",
       "./src/parsers/*.d.ts"
     ],
+    "#plugins/*": "./src/plugins/*",
     "#utils/*": "./src/utils/*"
   },
   "files": [
@@ -45,19 +46,17 @@
     "README.md"
   ],
   "dependencies": {
-    "@actions/core": "^3.0.0",
-    "@node-core/rehype-shiki": "^1.5.0",
-    "@swc/html-wasm": "^1.15.46",
+    "@node-core/rehype-shiki": "^1.5.1",
+    "@swc/html-wasm": "^1.16.2",
     "@swc/wasm": "^1.15.46",
     "acorn": "^8.17.0",
     "cosmiconfig": "^9.0.2",
     "dedent": "^1.7.2",
     "github-slugger": "^2.0.0",
     "glob-parent": "^6.0.2",
-    "hastscript": "^9.0.1",
     "mdast-util-slice-markdown": "^2.0.1",
     "mdast-util-to-string": "^4.0.0",
-    "piscina": "^5.3.1",
+    "piscina": "^5.3.2",
     "rehype-stringify": "^10.0.1",
     "remark-gfm": "^4.0.1",
     "remark-mdx": "^3.1.1",
@@ -74,7 +73,7 @@
     "unist-util-remove": "^4.0.0",
     "unist-util-select": "^5.1.0",
     "unist-util-visit": "^5.1.0",
-    "yaml": "^2.9.0"
+    "yaml": "^2.9.1"
   },
   "devDependencies": {
     "ajv": "^8.20.0",

package/src/generators/ast/generate.mjs

@@ -9,8 +9,8 @@ import { parse as parseYaml } from 'yaml';
 
 import getConfig from '#utils/configuration/index.mjs';
 import { withExt } from '#utils/file.mjs';
+import { getProcessor } from '#utils/markdown/processor.mjs';
 import { QUERIES } from '#utils/queries/index.mjs';
-import { getRemark as remark, getRemarkMdx } from '#utils/remark.mjs';
 
 import { STABILITY_INDEX_URL } from './constants.mjs';
 
@@ -67,6 +67,8 @@ export async function processChunk(inputSlice, itemIndices) {
     // The path is the relative path minus the extension
     const relativePath = sep + withExt(relative(parent, path));
 
+    const processor = getProcessor('ast', { mdx });
+
     let tree;
 
     if (mdx) {
@@ -79,7 +81,7 @@ export async function processChunk(inputSlice, itemIndices) {
         ''
       );
 
-      tree = getRemarkMdx().parse(source);
+      tree = processor.parse(source);
 
       if (frontmatter) {
         tree.children.unshift({
@@ -93,9 +95,11 @@ export async function processChunk(inputSlice, itemIndices) {
         (_, yaml) => `<!-- YAML\n${yaml}\n-->`
       );
 
-      tree = remark().parse(value);
+      tree = processor.parse(value);
     }
 
+    tree = await processor.run(tree, { path });
+
     results.push({ tree, path: relativePath, mdx });
   }
 

package/src/generators/ast/index.mjs

@@ -15,6 +15,15 @@ export default {
 
   hasParallelProcessor: true,
 
+  markdown: {
+    remarkPlugins: [
+      'remark-parse',
+      '#plugins/type-annotations/remark.mjs',
+      'remark-gfm',
+      '...',
+    ],
+  },
+
   generate,
   processChunk,
 };

package/src/generators/index.mjs

@@ -22,6 +22,7 @@ export const publicGenerators = {
   'api-links': '@node-core/doc-kit/api-links',
   'orama-db': '@doc-kit/generator-react/orama-db',
   'llms-txt': '@doc-kit/generator-react/llms-txt',
+  'llms-txt-full': '@doc-kit/generator-react/llms-txt-full',
   sitemap: '@doc-kit/generator-react/sitemap',
   html: '@doc-kit/generator-react/html',
   'section-pages': '@doc-kit/generator-react/section-pages',

package/src/generators/json/README.md

@@ -18,7 +18,7 @@ Files keep the input's directory layout: `doc/api/some/fs.md` becomes `out/some/
 
 - `output` {string} The directory to write the documents to.
 - `minify` {boolean} Whether to minify the output. Inherited from `global`.
-  **Default:** `true`.
+  **Default:** `false`.
 - `repository` {string} The `owner/name` repository source links resolve
   against. Inherited from `global`; without one, `sourceLink.url` is `null`.
 - `schemaURL` {string} Where the schema is published. `{schemaVersion}` is

package/src/generators/json/generated/schema.d.ts

@@ -1,4 +1,4 @@
-/* eslint-disable */
+/* oxlint-disable */
 /**
  * Generated from `schema.json` by `scripts/generate-json-types.mjs`.
  * Do not edit: change the schema and regenerate instead.

package/src/generators/json/index.mjs

@@ -22,6 +22,15 @@ export default {
 
   hasParallelProcessor: true,
 
+  markdown: {
+    remarkPlugins: [
+      '#plugins/type-annotations/remark.mjs',
+      'remark-gfm',
+      'remark-stringify',
+      '...',
+    ],
+  },
+
   generate,
   processChunk,
 };

package/src/generators/json/utils/markdown.mjs

@@ -3,7 +3,7 @@
 import { u as createTree } from 'unist-builder';
 import { visit } from 'unist-util-visit';
 
-import { getRemark, getRemarkMdx } from '#utils/remark.mjs';
+import { getProcessor } from '#utils/markdown/processor.mjs';
 
 import { DISPLAY_NAME } from '../constants.mjs';
 
@@ -19,7 +19,7 @@ export const blocksToMarkdown = (nodes, mdx = false) => {
     return '';
   }
 
-  const processor = mdx ? getRemarkMdx() : getRemark();
+  const processor = getProcessor('json', { mdx });
 
   return processor.stringify(createTree('root', nodes)).trim();
 };

package/src/generators/json-all/README.md

@@ -23,7 +23,7 @@ generator's schema for the documents.
 
 - `output` {string} The directory where `all.json` will be written.
 - `minify` {boolean} Whether to minify the output. Inherited from `global`.
-  **Default:** `true`.
+  **Default:** `false`.
 - `index` {Array} The `{ api }` objects defining the document order. Inherited
   from `global`.
 - `schemaURL` {string} Where the bundle's schema is published.

package/src/generators/json-simple/README.md

@@ -6,4 +6,4 @@ The `json-simple` generator creates a simplified JSON version of the API documen
 
 - `output` {string} The directory where `api-docs.json` will be written.
 - `minify` {boolean} Whether to minify the output JSON. Inherited from
-  `global`. **Default:** `true`.
+  `global`. **Default:** `false`.

package/src/generators/loader.mjs

@@ -7,6 +7,9 @@ import { enforceArray } from '#utils/array.mjs';
 
 import { allGenerators } from './index.mjs';
 
+// The module each loaded generator comes from
+const generatorModules = new WeakMap();
+
 /**
  * Resolves a CLI/configuration target into an import specifier. Shorthand
  * names map through the alias table; filesystem paths become file URLs;
@@ -64,9 +67,20 @@ export const loadGenerator = async specifier => {
     );
   }
 
+  generatorModules.set(generator, import.meta.resolve(resolved));
+
   return generator;
 };
 
+/**
+ * The URL of the module a generator was loaded from, which the specifiers of
+ * its Markdown pipeline resolve against.
+ *
+ * @param {GeneratorMetadata} generator - A generator loaded by `loadGenerator`
+ * @returns {string | undefined}
+ */
+export const getGeneratorModule = generator => generatorModules.get(generator);
+
 /**
  * Loads the given generators plus the transitive closure of their
  * dependencies (via `dependsOn`) and dependents (via `dependent`).

package/src/generators/metadata/generate.mjs

@@ -3,8 +3,6 @@
 import getConfig from '#utils/configuration/index.mjs';
 import { loadFromURL } from '#utils/loaders.mjs';
 
-import { parseApiDoc } from './utils/parse.mjs';
-
 /**
  * Process a chunk of API doc files in a worker thread.
  * Called by chunk-worker.mjs for parallel processing.
@@ -12,6 +10,10 @@ import { parseApiDoc } from './utils/parse.mjs';
  * @type {import('./types').Generator['processChunk']}
  */
 export async function processChunk(fullInput, itemIndices, typeMap) {
+  // Loaded on first use rather than with the generator, so the main thread,
+  // which never parses, does not load the TypeScript parser (~40MB of WASM)
+  const { parseApiDoc } = await import('./utils/parse.mjs');
+
   const results = [];
 
   for (const idx of itemIndices) {

package/src/generators/metadata/index.mjs

@@ -16,6 +16,15 @@ export default {
 
   hasParallelProcessor: true,
 
+  markdown: {
+    remarkPlugins: [
+      'remark-parse',
+      '#plugins/type-annotations/remark.mjs',
+      'remark-gfm',
+      '...',
+    ],
+  },
+
   generate,
   processChunk,
 };

package/src/generators/metadata/maps/mdn.json

@@ -475,6 +475,7 @@
   "magnetometer": "https://developer.mozilla.org/docs/Web/API/Magnetometer",
   "managedmediasource": "https://developer.mozilla.org/docs/Web/API/ManagedMediaSource",
   "managedsourcebuffer": "https://developer.mozilla.org/docs/Web/API/ManagedSourceBuffer",
+  "mathmlanchorelement": "https://developer.mozilla.org/docs/Web/API/MathMLAnchorElement",
   "mathmlelement": "https://developer.mozilla.org/docs/Web/API/MathMLElement",
   "mediacapabilities": "https://developer.mozilla.org/docs/Web/API/MediaCapabilities",
   "mediadeviceinfo": "https://developer.mozilla.org/docs/Web/API/MediaDeviceInfo",
@@ -553,6 +554,7 @@
   "oes_vertex_array_object": "https://developer.mozilla.org/docs/Web/API/OES_vertex_array_object",
   "otpcredential": "https://developer.mozilla.org/docs/Web/API/OTPCredential",
   "ovr_multiview2": "https://developer.mozilla.org/docs/Web/API/OVR_multiview2",
+  "observable": "https://developer.mozilla.org/docs/Web/API/Observable",
   "offlineaudiocompletionevent": "https://developer.mozilla.org/docs/Web/API/OfflineAudioCompletionEvent",
   "offlineaudiocontext": "https://developer.mozilla.org/docs/Web/API/OfflineAudioContext",
   "offscreencanvas": "https://developer.mozilla.org/docs/Web/API/OffscreenCanvas",
@@ -826,6 +828,7 @@
   "stylesheet": "https://developer.mozilla.org/docs/Web/API/StyleSheet",
   "stylesheetlist": "https://developer.mozilla.org/docs/Web/API/StyleSheetList",
   "submitevent": "https://developer.mozilla.org/docs/Web/API/SubmitEvent",
+  "subscriber": "https://developer.mozilla.org/docs/Web/API/Subscriber",
   "subtlecrypto": "https://developer.mozilla.org/docs/Web/API/SubtleCrypto",
   "summarizer": "https://developer.mozilla.org/docs/Web/API/Summarizer",
   "syncevent": "https://developer.mozilla.org/docs/Web/API/SyncEvent",

package/src/generators/metadata/types.d.ts

@@ -1,5 +1,5 @@
-import type { Position } from 'unist';
 import type { Root, Node, Data, Blockquote, Heading } from 'mdast';
+import type { Position } from 'unist';
 
 export type Generator = GeneratorMetadata<
   {},

package/src/generators/metadata/utils/parse.mjs

@@ -14,7 +14,6 @@ import {
   IGNORE_STABILITY_STEMS,
 } from '#generators/metadata/constants.mjs';
 import { UNIST } from '#utils/queries/index.mjs';
-import { getRemark as remark } from '#utils/remark.mjs';
 import { relative } from '#utils/url.mjs';
 
 import { resolveTypeAnnotations } from './resolveTypes.mjs';
@@ -147,9 +146,8 @@ export const parseApiDoc = ({ path, tree, mdx = false }, typeMap) => {
     // Remove processed YAML nodes from the content
     remove(subTree, [UNIST.isYamlNode]);
 
-    // Apply AST transformations
-    const parsedSubTree = remark().runSync(subTree);
-    metadata.content = parsedSubTree;
+    // The remark plugins already ran on the whole document, in `ast`
+    metadata.content = subTree;
 
     // Add to collection
     metadataCollection.push(metadata);

package/src/generators/metadata/utils/transformers.mjs

@@ -72,7 +72,7 @@ export const resolveTypeReference = (name, typeMap) => {
   // Transform Node.js types like 'vm.Something'.
   if (MODULE_QUALIFIED_NAME.test(name)) {
     const [mod, ...pieces] = name.split('.');
-    const isClass = pieces.at(-1).match(/^[A-Z][a-z]/);
+    const isClass = pieces.at(-1).match(/^[A-Z](?=.*[a-z])/);
 
     return `${mod}.html#${isClass ? 'class-' : ''}${slug(name)}`;
   }

package/src/generators/metadata/utils/visitors.mjs

@@ -2,8 +2,8 @@
 
 import { SKIP } from 'unist-util-visit';
 
+import { getProcessor } from '#utils/markdown/processor.mjs';
 import { QUERIES } from '#utils/queries/index.mjs';
-import { getRemark as remark } from '#utils/remark.mjs';
 import { transformNodesToString } from '#utils/unist.mjs';
 
 import { transformUnixManualToLink } from './transformers.mjs';
@@ -41,7 +41,7 @@ const updateReferences = (query, transformer, node, parent) => {
   // and adding those nodes to the parent.
   const {
     children: [newNode],
-  } = remark().parse(replacedTypes);
+  } = getProcessor('metadata').parse(replacedTypes);
 
   // Find the index of the original node in the parent
   const index = parent.children.indexOf(node);

package/src/generators/types.d.ts

@@ -70,6 +70,14 @@ declare global {
 
     hasParallelProcessor?: boolean;
 
+    /**
+     * The unified plugins this generator processes Markdown with, listed as
+     * in the `markdown` option, with paths relative to its module. Each list
+     * takes the configured plugins in place of its `'...'`.
+     * `getProcessor(name)` gives its processor.
+     */
+    markdown?: import('../utils/configuration/types').MarkdownPipeline;
+
     /**
      * The immediate generator that this generator depends on.
      * For example, the `html` generator depends on the `react` generator.

package/src/generators.mjs

@@ -9,6 +9,7 @@ import { resolvePipeline } from './generators/pipeline.mjs';
 import logger from './logger/index.mjs';
 import createWorkerPool from './threading/index.mjs';
 import createParallelWorker from './threading/parallel.mjs';
+import { loadMarkdownPlugins } from './utils/markdown/plugins.mjs';
 import { isAsyncIterable } from './utils/misc.mjs';
 
 const generatorsLogger = logger.child('generators');
@@ -78,6 +79,9 @@ const createGenerator = () => {
 
         generatorsLogger.debug(`Starting "${name}"`);
 
+        // Load its Markdown pipeline, for what it processes on this thread
+        await loadMarkdownPlugins(generator, configuration[name]?.markdown);
+
         // Create parallel worker for streaming generators
         const worker = hasParallelProcessor
           ? createParallelWorker(specifier, generator, pool, configuration)

package/src/logger/constants.mjs

@@ -32,3 +32,38 @@ export const levelToColorMap = {
   [LogLevel.error]: 'magenta',
   [LogLevel.fatal]: 'red',
 };
+
+/**
+ * Maps log level numbers to the GitHub Actions workflow commands reporting
+ * them.
+ */
+export const levelToGitHubCommand = {
+  [LogLevel.debug]: 'debug',
+  [LogLevel.info]: 'notice',
+  [LogLevel.warn]: 'warning',
+  [LogLevel.error]: 'error',
+  [LogLevel.fatal]: 'error',
+};
+
+/**
+ * Maps the characters a workflow command's message can't hold to their
+ * escapes. A command is a line, `::command key=value,key=value::message`,
+ * so its message, which runs to the end of the line, can't break it, nor hold
+ * the `%` an escape starts with.
+ */
+export const commandDataEscapes = {
+  '%': '%25',
+  '\r': '%0D',
+  '\n': '%0A',
+};
+
+/**
+ * Maps the characters a workflow command's property can't hold to their
+ * escapes: those its message can't, and the `,` separating properties and
+ * the `:` of the `::` ending them.
+ */
+export const commandPropertyEscapes = {
+  ...commandDataEscapes,
+  ':': '%3A',
+  ',': '%2C',
+};

package/src/logger/transports/github.mjs

@@ -1,23 +1,35 @@
 'use strict';
 
-import { debug, notice, warning, error } from '@actions/core';
+import { EOL } from 'node:os';
 
-import { LogLevel } from '#logger/constants.mjs';
+import {
+  commandDataEscapes,
+  commandPropertyEscapes,
+  levelToGitHubCommand,
+} from '#logger/constants.mjs';
 import { prettifyLevel } from '#logger/utils/colors.mjs';
 import { prettifyTimestamp } from '#logger/utils/time.mjs';
 
-const actions = {
-  [LogLevel.debug]: debug,
-  [LogLevel.info]: notice,
-  [LogLevel.warn]: warning,
-  [LogLevel.error]: error,
-  [LogLevel.fatal]: error,
-};
+/**
+ * Escapes a value of a workflow command: its message, with
+ * `commandDataEscapes`, or one of its properties, with
+ * `commandPropertyEscapes`.
+ *
+ * @param {string | number} value - The value
+ * @param {Record<string, string>} escapes - The escapes of the characters it can't hold
+ * @returns {string}
+ */
+const escapeCommandValue = (value, escapes) =>
+  String(value).replace(
+    /[%\r\n:,]/g,
+    character => escapes[character] ?? character
+  );
 
 /**
- * Logs messages to GitHub Actions with formatted output and file info with
- * appropriate gh actions based on level.
+ * Logs messages to GitHub Actions as workflow commands, with the file and
+ * lines they're about, for them to annotate.
  *
+ * @see https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands
  * @param {import('../types').TransportContext} context
  * @returns {void}
  */
@@ -30,13 +42,34 @@ const github = ({ level, message, timestamp, metadata = {}, module }) => {
 
   const logMessage = `[${time}] ${prettyLevel}${module ? ` (${module})` : ''}: ${message}`;
 
-  const logFn = actions[level] ?? notice;
+  const command = levelToGitHubCommand[level] ?? 'notice';
+
+  const data = escapeCommandValue(logMessage, commandDataEscapes);
 
-  logFn(logMessage, {
+  // Debug messages take no properties
+  if (command === 'debug') {
+    process.stdout.write(`::debug::${data}${EOL}`);
+
+    return;
+  }
+
+  const properties = {
     file: file?.path,
-    startLine: file?.position?.start.line,
+    line: file?.position?.start.line,
     endLine: file?.position?.end.line,
-  });
+  };
+
+  const annotations = [];
+
+  for (const [key, value] of Object.entries(properties)) {
+    if (value) {
+      annotations.push(
+        `${key}=${escapeCommandValue(value, commandPropertyEscapes)}`
+      );
+    }
+  }
+
+  process.stdout.write(`::${command} ${annotations.join(',')}::${data}${EOL}`);
 };
 
 export default github;