@doc-kit/core 0.0.0

package/src/generators/index.mjs

@@ -0,0 +1,48 @@
+'use strict';
+
+/**
+ * Maps the shorthand names accepted by the CLI and configuration files
+ * (e.g. `--target html`) to the import specifiers they resolve to.
+ *
+ * Generators are loaded dynamically by specifier (see `./loader.mjs`), so this
+ * module must not import any generator code — it is purely a lookup table.
+ * Anything that is not listed here is treated as a raw import specifier,
+ * which is how third-party generator packages are loaded.
+ */
+export const publicGenerators = {
+  'json-simple': '@doc-kit/core/json-simple',
+  'legacy-html': '@node-core/doc-kit-legacy/legacy-html',
+  'legacy-html-all': '@node-core/doc-kit-legacy/legacy-html-all',
+  'man-page': '@node-core/doc-kit/man-page',
+  'legacy-json': '@node-core/doc-kit-legacy/legacy-json',
+  'legacy-json-all': '@node-core/doc-kit-legacy/legacy-json-all',
+  'addon-verify': '@node-core/doc-kit/addon-verify',
+  'api-links': '@node-core/doc-kit/api-links',
+  'orama-db': '@doc-kit/generator-react/orama-db',
+  'llms-txt': '@doc-kit/generator-react/llms-txt',
+  sitemap: '@doc-kit/generator-react/sitemap',
+  html: '@doc-kit/generator-react/html',
+};
+
+// These ones are special since they don't produce standard output,
+// and hence, we don't expose them to the CLI. They can still be referenced
+// by shorthand (e.g. as configuration keys or generator dependencies).
+const internalGenerators = {
+  ast: '@doc-kit/core/ast',
+  metadata: '@doc-kit/core/metadata',
+  'jsx-ast': '@doc-kit/generator-react/jsx-ast',
+  'ast-js': '@doc-kit/core/ast-js',
+};
+
+// Former names kept resolvable for existing invocations and config files.
+// Unlike the maps above, keys here intentionally differ from the generator's
+// `name` property.
+export const deprecatedGenerators = {
+  web: '@doc-kit/generator-react/html',
+};
+
+export const allGenerators = {
+  ...publicGenerators,
+  ...internalGenerators,
+  ...deprecatedGenerators,
+};

package/src/generators/json-simple/README.md

@@ -0,0 +1,9 @@
+# `json-simple` Generator
+
+The `json-simple` generator creates a simplified JSON version of the API documentation, primarily for debugging and testing purposes.
+
+## Configuring
+
+- `output` {string} The directory where `api-docs.json` will be written.
+- `minify` {boolean} Whether to minify the output JSON. Inherited from
+  `global`. **Default:** `true`.

package/src/generators/json-simple/generate.mjs

@@ -0,0 +1,33 @@
+'use strict';
+
+import { join } from 'node:path';
+
+import { remove } from 'unist-util-remove';
+
+import getConfig from '../../utils/configuration/index.mjs';
+import { writeFile } from '../../utils/file.mjs';
+import { UNIST } from '../../utils/queries/index.mjs';
+
+/**
+ * Generates the simplified JSON version of the API docs
+ *
+ * @type {import('./types').Generator['generate']}
+ */
+export async function generate(input) {
+  const config = getConfig('json-simple');
+
+  input.forEach(node =>
+    remove(node.content, [UNIST.isStabilityNode, UNIST.isHeading])
+  );
+
+  if (config.output) {
+    // Writes all the API docs stringified content into one file
+    // Note: The full JSON generator in the future will create one JSON file per top-level API doc file
+    await writeFile(
+      join(config.output, 'api-docs.json'),
+      config.minify ? JSON.stringify(input) : JSON.stringify(input, null, 2)
+    );
+  }
+
+  return input;
+}

package/src/generators/json-simple/index.mjs

@@ -0,0 +1,23 @@
+'use strict';
+
+import { generate } from './generate.mjs';
+
+/**
+ * This generator generates a simplified JSON version of the API docs and returns it as a string
+ * this is not meant to be used for the final API docs, but for debugging and testing purposes
+ *
+ * This generator is a top-level generator, and it takes the raw AST tree of the API doc files
+ * and returns a stringified JSON version of the API docs.
+ *
+ * @type {import('./types').Generator}
+ */
+export default {
+  name: 'json-simple',
+
+  description:
+    'Generates the simple JSON version of the API docs, and returns it as a string',
+
+  dependsOn: '@doc-kit/core/metadata',
+
+  generate,
+};

package/src/generators/json-simple/types.d.ts

@@ -0,0 +1,6 @@
+import type { MetadataEntry } from '../metadata/types';
+
+export type Generator = GeneratorMetadata<
+  {},
+  Generate<Array<MetadataEntry>, Promise<Array<MetadataEntry>>>
+>;

package/src/generators/loader.mjs

@@ -0,0 +1,96 @@
+'use strict';
+
+import { isAbsolute } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import { allGenerators } from './index.mjs';
+
+/**
+ * Resolves a CLI/configuration target into an import specifier. Shorthand
+ * names map through the alias table; filesystem paths become file URLs;
+ * anything else is treated as an import specifier already.
+ *
+ * Resolution is idempotent, so already-resolved specifiers pass through.
+ *
+ * @param {string} target - Shorthand name, path, or import specifier
+ * @returns {string} The import specifier to load the generator from
+ */
+export const resolveGeneratorSpecifier = target => {
+  if (target in allGenerators) {
+    return allGenerators[target];
+  }
+
+  if (target.startsWith('.') || isAbsolute(target)) {
+    return pathToFileURL(target).href;
+  }
+
+  return target;
+};
+
+/**
+ * Imports a generator by specifier and returns its default export.
+ *
+ * @param {string} specifier - Shorthand name, path, or import specifier
+ * @returns {Promise<GeneratorMetadata>}
+ */
+export const loadGenerator = async specifier => {
+  const resolved = resolveGeneratorSpecifier(specifier);
+
+  /** @type {{ default?: GeneratorMetadata }} */
+  let module;
+
+  try {
+    module = await import(resolved);
+  } catch (error) {
+    if (error.code === 'ERR_MODULE_NOT_FOUND') {
+      throw new Error(
+        `Could not load generator "${specifier}" (resolved to "${resolved}"). ` +
+          'If it lives in a separate package, make sure that package is installed.',
+        { cause: error }
+      );
+    }
+
+    throw error;
+  }
+
+  const generator = module.default;
+
+  if (!generator?.name || typeof generator.generate !== 'function') {
+    throw new Error(
+      `"${specifier}" is not a generator: expected a default export ` +
+        'with a `name` and a `generate` function.'
+    );
+  }
+
+  return generator;
+};
+
+/**
+ * Loads the given generators plus the transitive closure of their
+ * dependencies (via `dependsOn`).
+ *
+ * @param {string[]} targets - Shorthand names, paths, or import specifiers
+ * @returns {Promise<Map<string, GeneratorMetadata>>} Loaded generators keyed
+ * by their resolved specifier
+ */
+export const loadGenerators = async targets => {
+  const generators = new Map();
+  const queue = targets.map(resolveGeneratorSpecifier);
+
+  while (queue.length > 0) {
+    const specifier = queue.shift();
+
+    if (generators.has(specifier)) {
+      continue;
+    }
+
+    const generator = await loadGenerator(specifier);
+    generators.set(specifier, generator);
+
+    if (generator.dependsOn) {
+      queue.push(resolveGeneratorSpecifier(generator.dependsOn));
+    }
+  }
+
+  return generators;
+};

package/src/generators/metadata/README.md

@@ -0,0 +1,8 @@
+# `metadata` Generator
+
+The `metadata` generator creates a flattened list of metadata entries from API documentation, extracting structured information about functions, classes, methods, and other API elements.
+
+## Configuring
+
+- `typeMap` {string} URL or path to the type mapping configuration file.
+  **Default:** `'typeMap.json'`.

package/src/generators/metadata/constants.mjs

@@ -0,0 +1,68 @@
+// On "About this Documentation", we define the stability indices, and thus
+// we don't need to check it for stability references
+export const IGNORE_STABILITY_STEMS = ['documentation'];
+
+// These are string replacements specific to Node.js API docs for anchor IDs
+export const DOC_API_SLUGS_REPLACEMENTS = [
+  { from: /node.js/i, to: 'nodejs' }, // Replace Node.js
+  { from: /&/, to: '-and-' }, // Replace &
+  { from: /[/,:;\\ ]/g, to: '-' }, // Replace /,:;\. and whitespace
+  { from: /^-(?!-|$)/g, to: '' }, // Remove a single leading hyphen
+  { from: /(?<!^-*)-+$/g, to: '' }, // Remove any trailing hyphens
+  { from: /^(?!-)(?!-+$).*?(--+)/g, to: '-' }, // Replace multiple hyphens
+];
+
+// Matches Node.js deprecation headings such as "DEP0001: Description".
+export const DEPRECATION_HEADING_REGEX = /^(DEP\d+):/;
+
+// A dotted identifier path, e.g. `vm.Module` or `os.constants.dlopen`. Names
+// that merely contain a dot (prose, `Object.<string, string>`, `1.5`) must not
+// be turned into a module link.
+export const MODULE_QUALIFIED_NAME = /^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+$/;
+
+// These are regular expressions used to determine if a given Markdown heading
+// is a specific type of API Doc entry (e.g., Event, Class, Method, etc)
+// and to extract the inner content of said Heading to be used as the API doc entry name
+const CAMEL_CASE = '\\w+(?:\\.\\w+)*';
+const FUNCTION_CALL = '\\([^)]*\\)';
+
+// Matches "bar":
+// Group 1: foo[bar]
+// Group 2: foo.bar
+const PROPERTY = `${CAMEL_CASE}(?:(\\[[^\\]]+\\])|\\.(\\w+))`;
+
+// An array of objects defining the different types of API doc headings we want to
+// capture and their respective regex to match against the heading text.
+// The regex are case-insensitive.
+export const DOC_API_HEADING_TYPES = [
+  {
+    type: 'method',
+    regex: new RegExp(
+      `^\`(?:${PROPERTY}|(${CAMEL_CASE}))${FUNCTION_CALL}\`$`,
+      'i'
+    ),
+  },
+  { type: 'event', regex: /^Event: +`'?([^`]*?)'?`$/i },
+  {
+    type: 'class',
+    regex: new RegExp(
+      `^Class: +\`(${CAMEL_CASE}(?: extends +${CAMEL_CASE})?)\`$`,
+      'i'
+    ),
+  },
+  {
+    type: 'ctor',
+    regex: new RegExp(`^\`new +(${CAMEL_CASE})${FUNCTION_CALL}\`$`, 'i'),
+  },
+  {
+    type: 'classMethod',
+    regex: new RegExp(`^Static method: +\`${PROPERTY}${FUNCTION_CALL}\`$`, 'i'),
+  },
+  {
+    type: 'property',
+    regex: new RegExp(`^\`${PROPERTY}\`$`, 'i'),
+  },
+];
+
+// This is the base URL of the Man7 documentation
+export const DOC_MAN_BASE_URL = 'http://man7.org/linux/man-pages/man';

package/src/generators/metadata/generate.mjs

@@ -0,0 +1,40 @@
+'use strict';
+
+import { parseApiDoc } from './utils/parse.mjs';
+import getConfig from '../../utils/configuration/index.mjs';
+import { loadFromURL } from '../../utils/loaders.mjs';
+
+/**
+ * Process a chunk of API doc files in a worker thread.
+ * Called by chunk-worker.mjs for parallel processing.
+ *
+ * @type {import('./types').Generator['processChunk']}
+ */
+export async function processChunk(fullInput, itemIndices, typeMap) {
+  const results = [];
+
+  for (const idx of itemIndices) {
+    results.push(...parseApiDoc(fullInput[idx], typeMap));
+  }
+
+  return results;
+}
+
+/**
+ * Generates a flattened list of metadata entries from API docs.
+ *
+ * @type {import('./types').Generator['generate']}
+ */
+export async function* generate(inputs, worker) {
+  const { metadata: config } = getConfig();
+
+  const typeMap = config.typeMap
+    ? JSON.parse(await loadFromURL(config.typeMap))
+    : {};
+
+  // Stream chunks as they complete - allows dependent generators
+  // to start collecting/preparing while we're still processing
+  for await (const chunkResult of worker.stream(inputs, typeMap)) {
+    yield chunkResult.flat();
+  }
+}

package/src/generators/metadata/index.mjs

@@ -0,0 +1,21 @@
+'use strict';
+
+import { generate, processChunk } from './generate.mjs';
+
+/**
+ * This generator generates a flattened list of metadata entries from a API doc
+ *
+ * @type {import('./types').Generator}
+ */
+export default {
+  name: 'metadata',
+
+  description: 'generates a flattened list of API doc metadata entries',
+
+  dependsOn: '@doc-kit/core/ast',
+
+  hasParallelProcessor: true,
+
+  generate,
+  processChunk,
+};

package/src/generators/metadata/maps/builtin.json

@@ -0,0 +1,16 @@
+{
+  "asynciterable": "https://tc39.github.io/ecma262/#sec-asynciterable-interface",
+  "iterable": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols#the_iterable_protocol",
+  "void": "https://developer.mozilla.org/docs/Web/JavaScript/Reference/Operators/void",
+  "module namespace object": "https://tc39.github.io/ecma262/#sec-module-namespace-exotic-objects",
+  "null": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#null_type",
+  "undefined": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#undefined_type",
+  "boolean": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#boolean_type",
+  "number": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#number_type",
+  "bigint": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#bigint_type",
+  "string": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#string_type",
+  "symbol": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#symbol_type",
+  "integer": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#number_type",
+  "any": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Data_types",
+  "this": "https://developer.mozilla.org/docs/Web/JavaScript/Reference/Operators/this"
+}