@doc-kit/core 0.0.0

package/CHANGELOG.md

@@ -0,0 +1,16 @@
+# @doc-kit/core
+
+## 1.4.3
+
+### Patch Changes
+
+- [#905](https://github.com/nodejs/doc-kit/pull/905) [`f214a00`](https://github.com/nodejs/doc-kit/commit/f214a009d5ceadddc2a3b8d83b94eb070eb6e790) Thanks [@avivkeller](https://github.com/avivkeller)! - Allow for the specification of dynamically generated configuration values
+
+## 1.4.2
+
+### Patch Changes
+
+- [#855](https://github.com/nodejs/doc-kit/pull/855) [`d7bdf39`](https://github.com/nodejs/doc-kit/commit/d7bdf39cc60852aaec42b6846fbacb89aecf3eec) Thanks [@bmuenzenmeyer](https://github.com/bmuenzenmeyer)! - Adopt Changesets for releases: versioning and publishing are now driven by changeset files, which
+  produce a `CHANGELOG.md`, git tags, and GitHub Releases.
+
+- [#895](https://github.com/nodejs/doc-kit/pull/895) [`9d7ce14`](https://github.com/nodejs/doc-kit/commit/9d7ce14c0fa11ee34b475cb5a752ce7e2cd44cb2) Thanks [@avivkeller](https://github.com/avivkeller)! - Avoids directly mutating the AST in `legacy-json`, as to ensure future generators do not run with a input different than they expect.

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright Node.js Website WG contributors. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,36 @@
+# `@doc-kit/core`
+
+The core of [doc-kit](https://github.com/nodejs/doc-kit): the engine that
+turns API-shaped Markdown into documentation sites, JSON, man pages, and
+more. It powers the Node.js API reference, and it can document any project.
+
+The command-line interface lives in the companion
+[`@doc-kit/cli`](https://www.npmjs.com/package/@doc-kit/cli) package.
+
+## Generators
+
+Output formats are provided by generators. This package ships the shared
+pipeline stages and `json-simple`; the rest come from companion packages:
+
+| Package                                                                                | Generators                                                                            |
+| -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- |
+| [`@doc-kit/generator-react`](https://www.npmjs.com/package/@doc-kit/generator-react)   | `html` (the modern site), `orama-db`, `llms-txt`, `sitemap`                           |
+| [`@node-core/doc-kit-legacy`](https://www.npmjs.com/package/@node-core/doc-kit-legacy) | `legacy-html`, `legacy-html-all`, `legacy-json`, `legacy-json-all` (Node.js-specific) |
+| [`@node-core/doc-kit`](https://www.npmjs.com/package/@node-core/doc-kit)               | `man-page`, `api-links`, `addon-verify` (Node.js-specific)                            |
+
+Custom generators load by import specifier — any module whose default export
+is a generator works as a `--target`.
+
+## Contributing
+
+This package lives in the [nodejs/doc-kit](https://github.com/nodejs/doc-kit)
+monorepo. From this directory (or the repository root with
+`npm run <script> --workspace @doc-kit/core`):
+
+```sh
+# Run this package's tests
+npm test
+
+# Re-run the tests on every change
+npm run test:watch
+```

package/package.json

@@ -0,0 +1,88 @@
+{
+  "name": "@doc-kit/core",
+  "type": "module",
+  "version": "0.0.0",
+  "description": "The doc-kit engine: the pipeline and shared APIs that power the doc-kit generators and CLI",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/nodejs/doc-kit.git",
+    "directory": "packages/core"
+  },
+  "main": "./src/generators.mjs",
+  "exports": {
+    ".": "./src/generators.mjs",
+    "./ast": "./src/generators/ast/index.mjs",
+    "./ast-js": "./src/generators/ast-js/index.mjs",
+    "./json-simple": "./src/generators/json-simple/index.mjs",
+    "./metadata": "./src/generators/metadata/index.mjs",
+    "./package.json": "./package.json",
+    "./shiki.config.mjs": "./shiki.config.mjs",
+    "./src/*": "./src/*",
+    "./*": "./src/*"
+  },
+  "files": [
+    "src",
+    "!src/**/*.test.mjs",
+    "!src/**/__tests__",
+    "shiki.config.mjs",
+    "CHANGELOG.md",
+    "LICENSE",
+    "README.md"
+  ],
+  "dependencies": {
+    "@actions/core": "^3.0.0",
+    "@node-core/rehype-shiki": "^1.4.3",
+    "@swc/html-wasm": "^1.15.46",
+    "@swc/wasm": "^1.15.46",
+    "acorn": "^8.17.0",
+    "cosmiconfig": "^9.0.2",
+    "dedent": "^1.7.2",
+    "github-slugger": "^2.0.0",
+    "glob-parent": "^6.0.2",
+    "hastscript": "^9.0.1",
+    "piscina": "^5.3.0",
+    "rehype-stringify": "^10.0.1",
+    "remark-gfm": "^4.0.1",
+    "remark-mdx": "^3.1.1",
+    "remark-parse": "^11.0.0",
+    "remark-rehype": "^11.1.2",
+    "remark-stringify": "^11.0.0",
+    "semver": "^7.8.5",
+    "shiki": "^4.3.0",
+    "tinyglobby": "^0.2.17",
+    "unified": "^11.0.5",
+    "unist-builder": "^4.0.0",
+    "unist-util-find-after": "^5.0.0",
+    "unist-util-position": "^5.0.0",
+    "unist-util-remove": "^4.0.0",
+    "unist-util-select": "^5.1.0",
+    "unist-util-visit": "^5.1.0",
+    "yaml": "^2.9.0"
+  },
+  "devDependencies": {
+    "hast-util-to-html": "^9.0.5",
+    "hast-util-to-string": "^3.0.1"
+  },
+  "peerDependencies": {
+    "@node-core/doc-kit": "1.4.3",
+    "@node-core/doc-kit-legacy": "0.0.0",
+    "@doc-kit/generator-react": "0.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@node-core/doc-kit": {
+      "optional": true
+    },
+    "@node-core/doc-kit-legacy": {
+      "optional": true
+    },
+    "@doc-kit/generator-react": {
+      "optional": true
+    }
+  },
+  "scripts": {
+    "test": "node --test --experimental-test-module-mocks \"src/**/*.test.mjs\"",
+    "test:coverage": "c8 node --test --experimental-test-module-mocks \"src/**/*.test.mjs\"",
+    "test:update-snapshots": "node --test --experimental-test-module-mocks --test-update-snapshots \"src/**/*.test.mjs\"",
+    "test:watch": "node --test --experimental-test-module-mocks --watch \"src/**/*.test.mjs\""
+  }
+}

package/shiki.config.mjs

@@ -0,0 +1,50 @@
+'use strict';
+
+import cLanguage from 'shiki/langs/c.mjs';
+import cmakeLanguage from 'shiki/langs/cmake.mjs';
+import coffeeScriptLanguage from 'shiki/langs/coffeescript.mjs';
+import cPlusPlusLanguage from 'shiki/langs/cpp.mjs';
+import cssLanguage from 'shiki/langs/css.mjs';
+import diffLanguage from 'shiki/langs/diff.mjs';
+import dockerLanguage from 'shiki/langs/docker.mjs';
+import htmlLanguage from 'shiki/langs/html.mjs';
+import httpLanguage from 'shiki/langs/http.mjs';
+import javaScriptLanguage from 'shiki/langs/javascript.mjs';
+import jsonLanguage from 'shiki/langs/json.mjs';
+import powershellLanguage from 'shiki/langs/powershell.mjs';
+import pythonLanguage from 'shiki/langs/python.mjs';
+import shellScriptLanguage from 'shiki/langs/shellscript.mjs';
+import shellSessionLanguage from 'shiki/langs/shellsession.mjs';
+import typeScriptLanguage from 'shiki/langs/typescript.mjs';
+import lightTheme from 'shiki/themes/catppuccin-latte.mjs';
+import darkTheme from 'shiki/themes/catppuccin-mocha.mjs';
+
+/**
+ * Creates a Shiki configuration for the API Docs tooling
+ *
+ * @type {import('@shikijs/core').HighlighterCoreOptions}
+ */
+export default {
+  // Only register the themes we need, to support light/dark theme
+  themes: [lightTheme, darkTheme],
+  // Only register the languages that the API docs use
+  // and override the JavaScript language with the aliases
+  langs: [
+    ...httpLanguage,
+    ...jsonLanguage,
+    ...typeScriptLanguage,
+    ...shellScriptLanguage,
+    ...powershellLanguage,
+    ...shellSessionLanguage,
+    ...dockerLanguage,
+    ...diffLanguage,
+    ...cLanguage,
+    ...cPlusPlusLanguage,
+    ...coffeeScriptLanguage,
+    ...cmakeLanguage,
+    ...pythonLanguage,
+    ...htmlLanguage,
+    ...cssLanguage,
+    { ...javaScriptLanguage[0], aliases: ['mjs', 'cjs', 'js'] },
+  ],
+};

package/src/caching.mjs

@@ -0,0 +1,183 @@
+'use strict';
+
+import logger from './logger/index.mjs';
+import { isAsyncIterable } from './utils/misc.mjs';
+
+const cachingLogger = logger.child('caching');
+
+/**
+ * Collects all values from an async generator into a flat array.
+ * Each yielded chunk is spread into the results array.
+ *
+ * @template T
+ * @param {AsyncGenerator<T[]>} generator - Async generator yielding arrays
+ * @returns {Promise<T[]>} Flattened array of all yielded items
+ */
+export const collectAsyncGenerator = async generator => {
+  const results = [];
+
+  let chunkCount = 0;
+
+  for await (const chunk of generator) {
+    chunkCount++;
+
+    results.push(...chunk);
+
+    cachingLogger.debug(`Collected chunk ${chunkCount}`, {
+      itemsInChunk: chunk.length,
+    });
+  }
+
+  cachingLogger.debug(`Collection complete`, {
+    totalItems: results.length,
+    chunks: chunkCount,
+  });
+
+  return results;
+};
+
+/**
+ * Creates the cache that backs the generator pipeline. It is the single owner
+ * of every piece of caching the orchestrator needs.
+ *
+ * Generator caching: each generator's pending result, so a generator and its
+ * dependencies are only ever scheduled once.
+ *
+ * Stream caching: collecting an async generator's chunks into a single array
+ * exactly once, shared by every consumer.
+ *
+ * Dependency caching & eviction: tracking how many consumers still need each
+ * result and dropping it once the last one has read it, so intermediate results
+ * can be garbage collected while later generators keep running.
+ */
+export const createCache = () => {
+  /** @type {{ [key: string]: Promise<unknown> | AsyncGenerator }} */
+  const results = {};
+
+  /** @type {Map<string, Promise<unknown[]>>} */
+  const collections = new Map();
+
+  /**
+   * Number of consumers that still need to read each result. Once it reaches
+   * zero the result is evicted.
+   *
+   * @type {{ [key: string]: number }}
+   */
+  const remaining = {};
+
+  /**
+   * Collects an async generator's chunks, ensuring a single collection is
+   * shared across all consumers of the same key.
+   *
+   * @param {string} key - Cache key (resolved generator specifier)
+   * @param {AsyncGenerator<unknown[]>} generator - Generator to collect
+   * @returns {Promise<unknown[]>} Promise resolving to the collected items
+   */
+  const collect = (key, generator) => {
+    const hasKey = collections.has(key);
+
+    if (!hasKey) {
+      collections.set(key, collectAsyncGenerator(generator));
+    }
+
+    cachingLogger.debug(
+      hasKey
+        ? `Using cached collection for "${key}"`
+        : `Starting collection for "${key}"`
+    );
+
+    return collections.get(key);
+  };
+
+  return {
+    /**
+     * Whether a generator has already been scheduled.
+     *
+     * @param {string} name - Cache key (resolved generator specifier)
+     * @returns {boolean}
+     */
+    has: name => name in results,
+
+    /**
+     * Registers a generator's pending result.
+     *
+     * @param {string} name - Cache key (resolved generator specifier)
+     * @param {Promise<unknown> | AsyncGenerator} result - Pending result
+     */
+    store: (name, result) => {
+      results[name] = result;
+    },
+
+    /**
+     * Computes and records consumer counts across the dependency closure of the
+     * requested generators. A generator is consumed once per dependent
+     * generator, plus once more when it is a requested target (read by the
+     * final collection). This drives eviction without coupling the cache to the
+     * generator registry: callers supply how to resolve a dependency.
+     *
+     * @param {string[]} targets - Requested generator cache keys
+     * @param {(name: string) => string | undefined} dependsOn - Resolves a
+     * generator's dependency name (if any)
+     */
+    populateConsumerCounts: (targets, dependsOn) => {
+      const scheduled = new Set();
+      const stack = [...targets];
+
+      // Walk the dependency closure of the requested targets
+      while (stack.length > 0) {
+        const name = stack.pop();
+
+        if (scheduled.has(name)) {
+          continue;
+        }
+
+        scheduled.add(name);
+
+        const dependency = dependsOn(name);
+
+        if (dependency) {
+          stack.push(dependency);
+        }
+      }
+
+      // Each scheduled generator consumes its dependency exactly once
+      for (const name of scheduled) {
+        const dependency = dependsOn(name);
+
+        if (dependency) {
+          remaining[dependency] = (remaining[dependency] ?? 0) + 1;
+        }
+      }
+
+      // Each requested target is consumed once more by the final collection
+      for (const name of targets) {
+        remaining[name] = (remaining[name] ?? 0) + 1;
+      }
+    },
+
+    /**
+     * Reads a generator's result (collecting it first if it streams) and
+     * evicts it once every expected consumer has read it.
+     *
+     * @param {string} name - Cache key (resolved generator specifier) to consume
+     * @returns {Promise<unknown>}
+     */
+    consume: async name => {
+      const result = await results[name];
+
+      const value = isAsyncIterable(result)
+        ? await collect(name, result)
+        : result;
+
+      // Evict once the last consumer has retrieved the result
+      if (name in remaining && --remaining[name] <= 0) {
+        delete results[name];
+        collections.delete(name);
+
+        cachingLogger.debug(`Evicted "${name}" after final consumer`);
+      }
+
+      return value;
+    },
+  };
+};

package/src/generators/ast/README.md

@@ -0,0 +1,8 @@
+# `ast` Generator
+
+The `ast` generator parses Markdown API documentation files into AST (Abstract Syntax Tree) representations, parallelizing the parsing across worker threads for better performance.
+
+## Configuring
+
+- `input` {string|string[]} Glob pattern(s) for the Markdown files to parse.
+- `ignore` {string|string[]} Glob pattern(s) for files to exclude from parsing.

package/src/generators/ast/constants.mjs

@@ -0,0 +1 @@
+export const STABILITY_INDEX_URL = 'documentation.html#stability-index';

package/src/generators/ast/generate.mjs

@@ -0,0 +1,125 @@
+'use strict';
+
+import { readFile } from 'node:fs/promises';
+import { relative, sep } from 'node:path/posix';
+
+import globParent from 'glob-parent';
+import { globSync } from 'tinyglobby';
+import { parse as parseYaml } from 'yaml';
+
+import { STABILITY_INDEX_URL } from './constants.mjs';
+import getConfig from '../../utils/configuration/index.mjs';
+import { withExt } from '../../utils/file.mjs';
+import { QUERIES } from '../../utils/queries/index.mjs';
+import { getRemark as remark, getRemarkMdx } from '../../utils/remark.mjs';
+
+/**
+ * Determines whether a file should be parsed as MDX. A `.mdx` extension opts in
+ * by default, but an explicit `mdx` boolean in the file's `---` frontmatter
+ * always takes precedence (so a `.md` file can opt in, or a `.mdx` file out).
+ *
+ * @param {string} path - Absolute file path
+ * @param {string} content - Raw file contents
+ * @returns {boolean}
+ */
+const isMdxFile = (path, content) => {
+  const frontmatter = QUERIES.standardYamlFrontmatter.exec(content);
+
+  if (frontmatter) {
+    try {
+      const { mdx } = parseYaml(frontmatter[1]) ?? {};
+
+      if (typeof mdx === 'boolean') {
+        return mdx;
+      }
+    } catch {
+      // Node.js core docs use a non-standard frontmatter dialect that may fail
+      // to parse here; fall back to the extension check below.
+    }
+  }
+
+  return path.endsWith('.mdx');
+};
+
+/**
+ * Process a chunk of markdown files in a worker thread.
+ * Loads and parses markdown files into AST representations.
+ *
+ * @type {import('./types').Generator['processChunk']}
+ */
+export async function processChunk(inputSlice, itemIndices) {
+  const filePaths = itemIndices.map(idx => inputSlice[idx]);
+
+  const results = [];
+
+  for (const [path, parent] of filePaths) {
+    const content = await readFile(path, 'utf-8');
+
+    const mdx = isMdxFile(path, content);
+
+    // Stability indexes become links in both modes.
+    const withStabilityLinks = content.replace(
+      QUERIES.stabilityIndexPrefix,
+      match => `[${match}](${STABILITY_INDEX_URL})`
+    );
+
+    // The path is the relative path minus the extension
+    const relativePath = sep + withExt(relative(parent, path));
+
+    let tree;
+
+    if (mdx) {
+      // A `<!-- -->` HTML comment is invalid MDX, so the frontmatter can't be
+      // rewritten in-source. Strip it, parse as MDX, then re-attach it as the
+      // YAML `html` node the metadata stage expects.
+      const frontmatter = QUERIES.standardYamlFrontmatter.exec(content);
+      const source = withStabilityLinks.replace(
+        QUERIES.standardYamlFrontmatter,
+        ''
+      );
+
+      tree = getRemarkMdx().parse(source);
+
+      if (frontmatter) {
+        tree.children.unshift({
+          type: 'html',
+          value: `<!-- YAML\n${frontmatter[1]}\n-->`,
+        });
+      }
+    } else {
+      const value = withStabilityLinks.replace(
+        QUERIES.standardYamlFrontmatter,
+        (_, yaml) => `<!-- YAML\n${yaml}\n-->`
+      );
+
+      tree = remark().parse(value);
+    }
+
+    results.push({ tree, path: relativePath, mdx });
+  }
+
+  return results;
+}
+
+/**
+ * Generates AST trees from markdown input files.
+ *
+ * @type {import('./types').Generator['generate']}
+ */
+export async function* generate(_, worker) {
+  const { ast: config } = getConfig();
+
+  const files = config.input.flatMap(input => {
+    const parent = globParent(input);
+
+    return globSync(input, { ignore: config.ignore }).map(child => [
+      child,
+      parent,
+    ]);
+  });
+
+  // Parse markdown files in parallel using worker threads
+  for await (const chunkResult of worker.stream(files)) {
+    yield chunkResult;
+  }
+}

package/src/generators/ast/index.mjs

@@ -0,0 +1,20 @@
+'use strict';
+
+import { generate, processChunk } from './generate.mjs';
+
+/**
+ * This generator parses Markdown API doc files into AST trees.
+ * It parallelizes the parsing across worker threads for better performance.
+ *
+ * @type {import('./types').Generator}
+ */
+export default {
+  name: 'ast',
+
+  description: 'Parses Markdown API doc files into AST trees',
+
+  hasParallelProcessor: true,
+
+  generate,
+  processChunk,
+};

package/src/generators/ast/types.d.ts

@@ -0,0 +1,7 @@
+import type { Root } from 'mdast';
+
+export type Generator = GeneratorMetadata<
+  {},
+  Generate<undefined, AsyncGenerator<ParserOutput<Root>>>,
+  ProcessChunk<string, ParserOutput<Root>>
+>;

package/src/generators/ast-js/README.md

@@ -0,0 +1,8 @@
+# `ast-js` Generator
+
+The `ast-js` generator parses JavaScript source files into AST (Abstract Syntax Tree) representations using the Acorn parser.
+
+## Configuring
+
+- `input` {string|string[]} Glob pattern(s) for the JavaScript files to parse.
+- `ignore` {string|string[]} Glob pattern(s) for files to exclude from parsing.

package/src/generators/ast-js/generate.mjs

@@ -0,0 +1,56 @@
+'use strict';
+
+import { readFile } from 'node:fs/promises';
+import { extname } from 'node:path';
+
+import { parse } from 'acorn';
+import { globSync } from 'tinyglobby';
+
+import getConfig from '../../utils/configuration/index.mjs';
+
+/**
+ * Process a chunk of JavaScript files in a worker thread.
+ * Parses JS source files into AST representations.
+ *
+ * @type {import('./types').Generator['processChunk']}
+ */
+export async function processChunk(inputSlice, itemIndices) {
+  const filePaths = itemIndices.map(idx => inputSlice[idx]);
+
+  const results = [];
+
+  for (const path of filePaths) {
+    const value = await readFile(path, 'utf-8');
+
+    const parsed = parse(value, {
+      allowReturnOutsideFunction: true,
+      ecmaVersion: 'latest',
+      locations: true,
+    });
+
+    parsed.path = path;
+
+    results.push(parsed);
+  }
+
+  return results;
+}
+
+/**
+ * Generates a JavaScript AST from the input files.
+ *
+ * @type {import('./types').Generator['generate']}
+ */
+export async function* generate(_, worker) {
+  const config = getConfig('ast-js');
+
+  const files = globSync(config.input, { ignore: config.ignore }).filter(
+    p => extname(p) === '.js'
+  );
+
+  // Parse the Javascript sources into ASTs in parallel using worker threads
+  // source is both the items list and the fullInput since we use sliceInput
+  for await (const chunkResult of worker.stream(files)) {
+    yield chunkResult;
+  }
+}

package/src/generators/ast-js/index.mjs

@@ -0,0 +1,24 @@
+'use strict';
+
+import { generate, processChunk } from './generate.mjs';
+
+/**
+ * This generator parses Javascript sources passed into the generator's input
+ * field. This is separate from the Markdown parsing step since it's not as
+ * commonly used and can take up a significant amount of memory.
+ *
+ * Putting this with the rest of the generators allows it to be lazily loaded
+ * so we're only parsing the Javascript sources when we need to.
+ *
+ * @type {import('./types').Generator}
+ */
+export default {
+  name: 'ast-js',
+
+  description: 'Parses Javascript source files passed into the input.',
+
+  hasParallelProcessor: true,
+
+  generate,
+  processChunk,
+};

package/src/generators/ast-js/types.d.ts

@@ -0,0 +1,5 @@
+export type Generator = GeneratorMetadata<
+  {},
+  Generate<undefined, AsyncGenerator<JsProgram>>,
+  ProcessChunk<string, JsProgram>
+>;