@doc-kit/core 0.0.0

package/src/utils/configuration/index.mjs

@@ -0,0 +1,263 @@
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { cpus } from 'node:os';
+import { dirname, isAbsolute, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { isMainThread } from 'node:worker_threads';
+
+import { cosmiconfig } from 'cosmiconfig';
+import { coerce } from 'semver';
+
+import {
+  loadGenerators,
+  resolveGeneratorSpecifier,
+} from '../../generators/loader.mjs';
+import logger from '../../logger/index.mjs';
+import { parseChangelog, parseIndex } from '../../parsers/markdown.mjs';
+import { enforceArray } from '../array.mjs';
+import { leftHandAssign } from '../generators.mjs';
+import { deepMerge } from '../misc.mjs';
+
+const configExplorer = cosmiconfig('doc-kit');
+
+/**
+ * The name of the project being documented, from the manifest in the working
+ * directory. Generators use it for titles, logos, and templated text.
+ *
+ * @returns {string | undefined}
+ */
+const detectProject = () => {
+  try {
+    return JSON.parse(readFileSync('package.json', 'utf-8')).name;
+  } catch {
+    return undefined;
+  }
+};
+
+/**
+ * Get's the default configuration for the loaded generators
+ *
+ * @param {Map<string, GeneratorMetadata>} generators - Loaded generators
+ * @param {Partial<import('./types').Configuration>} config - The user configuration
+ */
+export const getDefaultConfig = (generators, config) =>
+  [...generators.values()].reduce(
+    (acc, generator) => {
+      acc[generator.name] =
+        'defaultConfiguration' in generator
+          ? typeof generator.defaultConfiguration === 'function'
+            ? generator.defaultConfiguration(config)
+            : generator.defaultConfiguration
+          : {};
+
+      return acc;
+    },
+    /** @type {import('./types').Configuration} */ ({
+      global: {
+        project: detectProject() ?? 'API Docs',
+        version: process.version,
+        minify: true,
+        ref: 'HEAD',
+        // Without release history there is nothing to build a version picker
+        // from, so generators render single-version output.
+        changelog: [],
+        pathsToCopy: ['assets', 'public', 'static'],
+      },
+
+      // The number of wasm memory instances is severely limited on
+      // riscv64 with sv39. Running multiple generators that use wasm in
+      // parallel could cause failures to allocate new wasm instance.
+      // See also https://github.com/nodejs/node/pull/60591
+      threads: process.arch === 'riscv64' ? 1 : cpus().length,
+      chunkSize: 10,
+    })
+  );
+
+/**
+ * Resolves an `extends` entry of a configuration file into an importable
+ * URL: relative paths resolve against the configuration file, anything else
+ * resolves as a package import specifier (e.g. `@node-core/doc-kit/config`).
+ *
+ * @param {string} specifier - The `extends` entry
+ * @param {string} configFilePath - The configuration file it appears in
+ * @returns {string} A `file:` URL to import
+ */
+const resolveConfigExtends = (specifier, configFilePath) => {
+  if (specifier.startsWith('.') || isAbsolute(specifier)) {
+    return pathToFileURL(resolve(dirname(configFilePath), specifier)).href;
+  }
+
+  return pathToFileURL(createRequire(configFilePath).resolve(specifier)).href;
+};
+
+/**
+ * Loads an explicit configuration file or searches for one using cosmiconfig.
+ *
+ * @param {string} [filePath] - The path to an explicit configuration file
+ * @returns {Promise<Partial<import('./types').Configuration>>} The loaded configuration object, or an empty object if none is found
+ */
+export const loadConfigFile = async filePath => {
+  const result = filePath
+    ? await configExplorer.load(filePath)
+    : await configExplorer.search();
+
+  if (!result) {
+    return {};
+  }
+
+  let { extends: presets, ...config } = result.config ?? {};
+
+  for (const preset of enforceArray(presets ?? []).toReversed()) {
+    const module = await import(resolveConfigExtends(preset, result.filepath));
+
+    config = deepMerge(module.default ?? module, config);
+  }
+
+  return config;
+};
+
+/**
+ * Transforms configuration values that need async processing or coercion.
+ * Only processes values that haven't been transformed yet (strings for changelog/index, non-coerced versions).
+ *
+ * @param {import('./types').GlobalConfiguration} value
+ * @returns {Promise<import('./types').GlobalConfiguration>}
+ */
+const transformConfig = async value => {
+  // Only coerce if it's a string (not already coerced)
+  if (value.version && typeof value.version === 'string') {
+    value.version = coerce(value.version);
+  }
+
+  // Only parse if it's not already parsed
+  if (value.changelog && !Array.isArray(value.changelog)) {
+    value.changelog = await parseChangelog(value.changelog);
+  }
+
+  // Only parse if it's not already parsed
+  if (value.index && !Array.isArray(value.index)) {
+    value.index = await parseIndex(value.index);
+  }
+
+  return value;
+};
+
+/**
+ * Converts CLI options into a config
+ * @param {import('../../../bin/commands/generate.mjs').CLIOptions} options
+ * @returns {import('./types').Configuration}
+ */
+export const createConfigFromCLIOptions = options => ({
+  global: {
+    input: options.input,
+    ignore: options.ignore,
+    output: options.output,
+    minify: options.minify,
+    ref: options.gitRef,
+    version: options.version,
+    changelog: options.changelog,
+    index: options.index,
+  },
+  metadata: {
+    typeMap: options.typeMap,
+  },
+  target: options.target,
+  threads: options.threads,
+  chunkSize: options.chunkSize,
+});
+
+/**
+ * Asserts that the resolved configuration has everything needed to run:
+ * at least one generator `target` and an `input` to read source files from.
+ * These may come from CLI flags or a config file; by this point both sources
+ * have been merged, so we validate the result rather than the raw options.
+ *
+ * @param {import('./types').Configuration} config - The merged configuration
+ */
+export const assertRunnableOptions = config => {
+  if (!config.target || !config.global?.input) {
+    throw new Error(
+      'Both a `target` and an `input` must be provided, either via ' +
+        '`--target`/`--input` or a configuration file. ' +
+        'Run `doc-kit generate --help` for usage.'
+    );
+  }
+};
+
+/**
+ * Creates a complete run configuration by merging config file, user options, and defaults.
+ * Processes and validates configuration values including version coercion, changelog parsing,
+ * and constraint enforcement for threads and chunk size.
+ *
+ * @param {import('../../../bin/commands/generate.mjs').CLIOptions} options - User-provided configuration options
+ * @returns {Promise<import('./types').Configuration>} The configuration
+ */
+export const createRunConfiguration = async options => {
+  const config = await loadConfigFile(options.configFile);
+  config.target &&= enforceArray(config.target);
+
+  // Resolve user configuration first so dynamic defaults can use it
+  const cliConfig = createConfigFromCLIOptions(options);
+  const intermediate = deepMerge(config, cliConfig);
+
+  // Resolve shorthand targets into import specifiers, then load the requested
+  // generators (and their dependency closure) so their defaults can be applied
+  intermediate.target &&= intermediate.target.map(resolveGeneratorSpecifier);
+  const generators = await loadGenerators(intermediate.target ?? []);
+
+  const merged = deepMerge(
+    getDefaultConfig(generators, intermediate),
+    intermediate
+  );
+
+  // These need to be coerced
+  merged.threads = Math.max(merged.threads, 1);
+  merged.chunkSize = Math.max(merged.chunkSize, 1);
+
+  if (process.arch === 'riscv64' && merged.threads > 1) {
+    logger.warn(
+      `Using ${merged.threads} threads might cause failures when ` +
+        'allocating wasm memory due to insufficient virtual address space ' +
+        'on riscv64 with sv39. Please consider using only a single thread.'
+    );
+  }
+
+  // Transform global config if it wasn't already done
+  await transformConfig(merged.global);
+
+  // Now assign to each generator config (they inherit from global)
+  await Promise.all(
+    [...generators.values()].map(async ({ name }) => {
+      const value = merged[name];
+
+      // Transform generator-specific overrides
+      await transformConfig(value);
+
+      // Assign from global (this populates missing values from global)
+      leftHandAssign(value, merged.global);
+    })
+  );
+
+  return merged;
+};
+
+/** @type {import('./types').Configuration} */
+let config;
+
+/**
+ * Configuration setter
+ * @param {import('./types').Configuration | import('../../../bin/commands/generate.mjs').CLIOptions} options
+ * @returns {Promise<import('./types').Configuration>}
+ */
+export const setConfig = async options =>
+  (config = isMainThread ? await createRunConfiguration(options) : options);
+
+/**
+ * Configuration getter
+ * @template {keyof import('./types').Configuration} T
+ * @param {T} generator
+ * @returns {T extends keyof import('./types').Configuration ? import('./types').Configuration[T] : import('./types').Configuration}
+ */
+const getConfig = generator => (generator ? config[generator] : config);
+
+export default getConfig;

package/src/utils/configuration/templates.mjs

@@ -0,0 +1,21 @@
+// This is the CHANGELOG to be consumed to generate a list of all major versions
+export const CHANGELOG_URL =
+  'https://raw.githubusercontent.com/{repository}/{ref}/CHANGELOG.md';
+
+// This is the Base URL for viewing a file within GitHub UI
+export const GITHUB_BLOB_URL = 'https://github.com/{repository}/blob/{ref}/';
+
+// This is the API docs base URL for editing a file on GitHub UI
+export const GITHUB_EDIT_URL = 'https://github.com/{repository}/edit/{ref}';
+
+/**
+ * Populate a template string based on a configuration
+ * @param {string} template The template string
+ * @param {import("./types").GlobalConfiguration} config The configuration
+ * @returns {string} the populated string
+ */
+export const populate = (template, config) =>
+  template.replace(
+    /\{(\w+)(?:\|(\w+))?\}/g,
+    (match, path, fallback) => config[path] ?? fallback ?? match
+  );

package/src/utils/configuration/types.d.ts

@@ -0,0 +1,64 @@
+import type { SemVer } from 'semver';
+import type { ReleaseEntry } from '../../parsers/types';
+
+export type Configuration = {
+  global: GlobalConfiguration;
+
+  // A list of generators to be used in the API doc generation process;
+  // This is considered a "sorted" list of generators, in the sense that
+  // if the last entry of this list contains a generated value, we will return
+  // the value of the last generator in the list, if any.
+  target: Array<keyof AvailableGenerators>;
+
+  // The number of threads the process is allowed to use
+  threads: number;
+
+  // Number of items to process per worker thread
+  chunkSize: number;
+} & {
+  [K in keyof AllGenerators]: GlobalConfiguration &
+    AllGenerators[K]['defaultConfiguration'];
+};
+
+export type GlobalConfiguration = {
+  // The name of the project being documented, used for titles, logos, and
+  // templated text (defaults to the `name` in the working directory's
+  // `package.json`)
+  project: string;
+
+  // The repository (`owner/name`), used for source and edit links; when
+  // omitted, repository-specific UI (e.g. the GitHub link) is omitted
+  repository?: string;
+
+  // The path to the input source files. This parameter accepts globs and can
+  // be a glob when passed to a generator.
+  input: string | string[];
+
+  // The path or glob patterns used to ignore files from the input source files.
+  ignore: string | string[];
+
+  // The path used to output generated files, this is to be considered
+  // the base path that any generator will use for generating files
+  // This parameter accepts globs but when passed to generators will contain
+  // the already resolved absolute path to the output folder
+  output: string;
+
+  // Whether or not the minify the output, in whatever form it may be
+  minify: boolean;
+
+  // Target Node.js version for the generation of the API docs
+  version: SemVer;
+
+  // A list of all major versions and their respective release information
+  changelog: Array<ReleaseEntry>;
+
+  // A list of all the titles of all the documentation files
+  index: Array<{ section: string; api: string }>;
+
+  // The base URL of the published site; templates referencing `{baseURL}`
+  // (e.g. sitemap and llms-txt page URLs) need it to produce absolute URLs
+  baseURL?: string | URL;
+
+  // Git ref (i.e. HEAD)
+  ref: string;
+};

package/src/utils/file.mjs

@@ -0,0 +1,20 @@
+import fs from 'node:fs/promises';
+import { dirname } from 'node:path';
+
+/**
+ * Returns the input string with the `ext` extension, replacing any pre-existing extension
+ * @param {string} str
+ * @param {string} ext
+ */
+export const withExt = (str, ext) =>
+  `${str.replace(/\.[0-9a-z]+$/i, '')}${ext ? `.${ext}` : ''}`;
+
+/**
+ * Writes a file, recursively
+ *
+ * @type {typeof fs.writeFile}
+ */
+export const writeFile = (file, ...args) =>
+  fs
+    .mkdir(dirname(file), { recursive: true })
+    .then(() => fs.writeFile(file, ...args));

package/src/utils/generators.mjs

@@ -0,0 +1,71 @@
+'use strict';
+
+import { coerce, major } from 'semver';
+
+/**
+ * Groups all the API metadata nodes by module (`api` property) so that we can process each different file
+ * based on the module it belongs to.
+ *
+ * @param {Array<import('../generators/metadata/types').MetadataEntry>} nodes The API metadata Nodes to be grouped
+ */
+export const groupNodesByModule = nodes => {
+  return Map.groupBy(nodes, node => node.api);
+};
+
+/**
+ * Parses the SemVer string into a Node.js-alike version
+ *
+ * @param {import('semver').SemVer} version The version to be parsed
+ */
+export const getVersionFromSemVer = version =>
+  version.minor === 0
+    ? `${version.major}.x`
+    : `${version.major}.${version.minor}.x`;
+
+/**
+ * @TODO: This should not be necessary, and indicates errors within the API docs
+ * @TODO: Hookup into a future Validation/Linting API
+ *
+ * This is a safe fallback to ensure that we always have a SemVer compatible version
+ * even if the input is not a valid SemVer string
+ *
+ * @param {string|import('semver').SemVer} version SemVer compatible version (maybe)
+ * @returns {import('semver').SemVer} SemVer compatible version
+ */
+export const coerceSemVer = version => {
+  const coercedVersion = coerce(version);
+
+  if (coercedVersion === null) {
+    // @TODO: Linter to complain about invalid SemVer strings
+    return coerce('0.0.0-REPLACEME');
+  }
+
+  return coercedVersion;
+};
+
+/**
+ * Gets compatible versions for an entry
+ *
+ * @param {string | import('semver').SemVer} introduced
+ * @param {Array<import('../parsers/types').ReleaseEntry>} releases
+ * @returns {Array<import('../parsers/types').ReleaseEntry>}
+ */
+export const getCompatibleVersions = (introduced, releases) => {
+  const coercedMajor = major(coerceSemVer(introduced));
+  // All Node.js versions that support the current API; If there's no "introduced_at" field,
+  // we simply show all versions, as we cannot pinpoint the exact version
+  return releases.filter(release => release.version.major >= coercedMajor);
+};
+
+/**
+ * Assigns properties from one or more source objects to the target object
+ * **without overwriting existing keys** in the target.
+ *
+ * Similar to `Object.assign`, but preserves the target's existing keys.
+ * The target object is mutated in place.
+ *
+ * @param {Object} target - The object to assign properties to.
+ * @param {Object} source - The source object
+ */
+export const leftHandAssign = (target, source) =>
+  Object.keys(source).forEach(k => k in target || (target[k] = source[k]));

package/src/utils/highlighter.mjs

@@ -0,0 +1,198 @@
+'use strict';
+
+import { endianness } from 'node:os';
+
+import createHighlighter from '@node-core/rehype-shiki';
+import { h as createElement } from 'hastscript';
+import { SKIP, visit } from 'unist-util-visit';
+
+import shikiConfig from '../../shiki.config.mjs';
+
+// This is what Remark will use as prefix within a <pre> className
+// to attribute the current language of the <pre> element
+const languagePrefix = 'language-';
+
+/**
+ * Creates a toolbar element with a language label and copy button.
+ *
+ * @param {string} languageId - The language identifier for the code block.
+ * @returns {import('hast').Element} The toolbar element.
+ */
+function createToolbarElement(languageId) {
+  return createElement('div', { class: 'code-toolbar' }, [
+    createElement('span', { class: 'code-language' }, languageId),
+    createElement('button', { class: 'copy-button' }, 'copy'),
+  ]);
+}
+
+/**
+ * Checks if the given node is a valid code element.
+ *
+ * @param {import('unist').Node} node - The node to be verified.
+ *
+ * @return {boolean} - True when it is a valid code element, false otherwise.
+ */
+function isCodeBlock(node) {
+  return Boolean(
+    node?.tagName === 'pre' && node?.children[0].tagName === 'code'
+  );
+}
+
+export const highlighter = await createHighlighter({
+  // riscv64 with sv39 has limited virtual memory space, where creating
+  // too many (>20) wasm memory instances fails.
+  // https://github.com/nodejs/node/pull/60591
+  //
+  // The wasm highlighter is currently not compatible with big endian.
+  // https://github.com/nodejs/node/pull/62512#issuecomment-4243469950
+  wasm: process.arch !== 'riscv64' && endianness() === 'LE',
+});
+
+/**
+ * Creates a HAST transformer for Shiki which is used for transforming our codeboxes
+ *
+ * @deprecated This is used only for the legacy-html generator, please use `@node-core/rehype-shiki` directly instead.
+ *
+ * @type {import('unified').Plugin}
+ */
+export default function rehypeShikiji() {
+  /**
+   * @param {import('hast').Root} tree - The HAST tree to be transformed.
+   */
+  return function (tree) {
+    visit(tree, 'element', (node, index, parent) => {
+      // We only want to process <pre>...</pre> elements
+      if (!parent || index == null || node.tagName !== 'pre') {
+        return;
+      }
+
+      // We want the contents of the <pre> element, hence we attempt to get the first child
+      const preElement = node.children[0];
+
+      // If thereÄs nothing inside the <pre> element... What are we doing here?
+      if (!preElement || !preElement.properties) {
+        return;
+      }
+
+      // Ensure that we're not visiting a <code> element but it's inner contents
+      // (keep iterating further down until we reach where we want)
+      if (preElement.type !== 'element' || preElement.tagName !== 'code') {
+        return;
+      }
+
+      // Get the <pre> element class names
+      const preClassNames = preElement.properties.className;
+
+      // The current classnames should be an array and it should have a length
+      if (typeof preClassNames !== 'object' || preClassNames.length === 0) {
+        return;
+      }
+
+      // We want to retrieve the language class name from the class names
+      const codeLanguage = preClassNames.find(
+        c => typeof c === 'string' && c.startsWith(languagePrefix)
+      );
+
+      // If we didn't find any `language-` classname then we shouldn't highlight
+      if (typeof codeLanguage !== 'string') {
+        return;
+      }
+
+      // Grabs the relevant alias/name of the language
+      const languageId = codeLanguage.slice(languagePrefix.length);
+
+      // Parses the <pre> contents and returns a HAST tree with the highlighted code
+      const { children } = highlighter.shiki.codeToHast(
+        preElement.children[0].value,
+        {
+          lang: languageId,
+          // Allows support for dual themes (light, dark) for Shiki
+          themes: { light: shikiConfig.themes[0], dark: shikiConfig.themes[1] },
+        }
+      );
+
+      // Adds the original language back to the <pre> element
+      children[0].properties.class = `${children[0].properties.class} ${codeLanguage}`;
+
+      // Adds the toolbar (language label + copy button) to the <pre> element
+      children[0].children.push(createToolbarElement(languageId));
+
+      // Replaces the <pre> element with the updated one
+      parent.children.splice(index, 1, ...children);
+    });
+
+    visit(tree, 'element', (_, index, parent) => {
+      const codeElements = [];
+
+      let currentIndex = index;
+
+      while (isCodeBlock(parent?.children[currentIndex])) {
+        const preElement = parent?.children[currentIndex];
+        const codeElement = preElement.children[0];
+
+        // We should get the language name from the class name
+        if (preElement.properties.class?.length) {
+          const className = preElement.properties.class;
+          const matches = className.match(/language-(?<language>.*)/);
+          const language = matches?.groups.language ?? 'text';
+
+          // For this iteration of our code, we only support multi-code tab for
+          // JavaScript languages for Node.js (CJS / MJS)
+          if (language === 'cjs' || language === 'mjs') {
+            // We patch up Shiki's `pre` element properties into the `code` element
+            // since we want to keep the original properties
+            codeElement.properties.class = `${className} ${language}`;
+            codeElement.properties.style = preElement.properties.style;
+            codeElement.properties.language = language;
+
+            // Add the code element to the pre children
+            codeElements.push(codeElement);
+          }
+        }
+
+        currentIndex += 1;
+
+        // Since we only support CJS/MJS switch, we should have exactly 2 elements
+        // with different languages in order to create a switchable code tab
+        if (
+          codeElements.length === 2 &&
+          codeElements[0].properties?.language !==
+            codeElements[1].properties?.language
+        ) {
+          const switchablePreElement = createElement(
+            'pre',
+            {
+              // We grab Shiki's styling from the code tag
+              // back to the pre element tag to ensure consistency
+              style: codeElements[0].properties.style,
+              class: 'shiki',
+            },
+            [
+              createElement('input', {
+                class: 'js-flavor-toggle',
+                type: 'checkbox',
+                // If the CJS code block is the first one, then we should keep
+                // the checkbox checked so that it highglit the CJS by default
+                checked: codeElements[0].properties.language === 'cjs',
+              }),
+              ...codeElements,
+              createToolbarElement('javascript'),
+            ]
+          );
+
+          // This removes all the original code Elements and adds a new CodeTab Element
+          // at the original start of the first Code Element
+          parent.children.splice(
+            index,
+            currentIndex - index,
+            switchablePreElement
+          );
+
+          // Prevent visiting the code block children and for the next N Elements
+          // since all of them belong to this CodeTabs Element
+          return [SKIP];
+        }
+      }
+    });
+  };
+}

package/src/utils/html-minifier.mjs

@@ -0,0 +1,12 @@
+import { minify } from '@swc/html-wasm';
+
+/**
+ * Minifies HTML with project defaults and optional overrides. At the moment,
+ * swc's defaults are suitable for our needs, but in the event that this changes,
+ * allowing project defaults is beneficial.
+ *
+ * @param {string} html
+ * @param {import('@swc/html-wasm').Options} [options]
+ */
+export const minifyHTML = async (html, options = {}) =>
+  minify(html, options).then(({ code }) => code);