@doc-kit/core 0.0.0

package/src/generators/metadata/utils/visitors.mjs

@@ -0,0 +1,134 @@
+'use strict';
+
+import { SKIP } from 'unist-util-visit';
+
+import { transformUnixManualToLink } from './transformers.mjs';
+import { extractYamlContent, parseYAMLIntoMetadata } from './yaml.mjs';
+import { QUERIES } from '../../../utils/queries/index.mjs';
+import { getRemark as remark } from '../../../utils/remark.mjs';
+import { transformNodesToString } from '../../../utils/unist.mjs';
+
+/**
+ * Updates a Markdown link into a HTML link for API docs
+ * @param {import('@types/mdast').Link} node A Markdown link node
+ */
+export const visitMarkdownLink = node => {
+  node.url = node.url.replace(
+    QUERIES.markdownUrl,
+    (_, filename, hash = '') => `${filename}.html${hash}`
+  );
+
+  return [SKIP];
+};
+
+/**
+ * Updates a reference
+ * @param {import('@types/mdast').Text} node The current node
+ * @param {import('@types/mdast').Parent} parent The parent node
+ * @param {string|RegExp} query The search query
+ * @param {Function} transformer The function to transform the reference
+ */
+const updateReferences = (query, transformer, node, parent) => {
+  const replacedTypes = node.value
+    .replace(query, transformer)
+    // Remark doesn't handle leading / trailing spaces, so replace them with
+    // HTML entities.
+    .replace(/^\s/, ' ')
+    .replace(/\s$/, ' ');
+
+  // This changes the type into a link by splitting it up into several nodes,
+  // and adding those nodes to the parent.
+  const {
+    children: [newNode],
+  } = remark().parse(replacedTypes);
+
+  // Find the index of the original node in the parent
+  const index = parent.children.indexOf(node);
+
+  // Replace the original node with the new node(s)
+  parent.children.splice(index, 1, ...newNode.children);
+
+  return [SKIP];
+};
+
+/**
+ * Updates unix manual references to be markdown links
+ * @param {import('@types/mdast').Text} node The text node
+ * @param {import('@types/mdast').Parent} parent The parent node
+ */
+export const visitTextWithUnixManualNode = (node, parent) =>
+  updateReferences(
+    QUERIES.unixManualPage,
+    transformUnixManualToLink,
+    node,
+    parent
+  );
+
+/**
+ * Updates a Markdown Link Reference into an actual Link to the Definition
+ * @param {import('@types/mdast').LinkReference} node A link reference node
+ * @param {Array<import('@types/mdast').Definition>} definitions The Definitions of the API Doc
+ */
+export const visitLinkReference = (node, definitions) => {
+  const definition = definitions.find(
+    ({ identifier }) => identifier === node.identifier
+  );
+
+  node.type = 'link';
+  node.url = definition.url;
+
+  return [SKIP];
+};
+
+/**
+ * Parses a Stability Index Entry and updates the current Metadata
+ * @param {import('@types/mdast').Blockquote} node The blockquote node
+ * @param {import('../types').MetadataEntry} metadata The metadata object to update
+ */
+export const visitStability = (node, metadata) => {
+  // `node` is a `blockquote` node, and the first child will always be
+  // a `paragraph` node, so we can safely access the children of the first child
+  // which we use as the prefix and description of the Stability Index
+  const stabilityPrefix = transformNodesToString(node.children[0].children);
+
+  // Attempts to grab the Stability index and description from the prefix
+  const matches = QUERIES.stabilityIndex.exec(stabilityPrefix);
+
+  // Ensures that the matches are valid and that we have at least 3 entries
+  if (matches && matches.length === 3) {
+    // Updates the `data` property of the Stability Index node
+    // so that the original node data can also be inferred
+    node.data = {
+      // The 2nd match should be the group that matches the Stability Index
+      index: matches[1],
+      // The 3rd match should be the group containing all the remaining text
+      // which is used as a description (we trim it to an one liner)
+      description: matches[2].replace(/\n/g, ' ').trim(),
+    };
+
+    // Adds the Stability Index metadata to the current Metadata entry
+    if (metadata) {
+      metadata.stability = node;
+    }
+  }
+
+  return [SKIP];
+};
+
+/**
+ * Helper function to parse YAML from a node
+ * @param {import('@types/mdast').Blockquote} node The blockquote node
+ * @param {Object} metadata The metadata object to update
+ */
+export const visitYAML = (node, metadata) => {
+  const yaml = extractYamlContent(node);
+  const meta = parseYAMLIntoMetadata(yaml);
+
+  Object.entries(meta).forEach(([key, value]) => {
+    if (Array.isArray(metadata[key])) {
+      return metadata[key].push(...value);
+    }
+
+    metadata[key] = value;
+  });
+};

package/src/generators/metadata/utils/yaml.mjs

@@ -0,0 +1,58 @@
+'use strict';
+
+import yaml from 'yaml';
+
+import { QUERIES } from '../../../utils/queries/index.mjs';
+
+/**
+ * Extracts raw YAML content from a node
+ *
+ * @param {import('mdast').Node} node A HTML node containing the YAML content
+ * @returns {string} The extracted raw YAML content
+ */
+export const extractYamlContent = node => {
+  return node.value.replace(
+    QUERIES.yamlInnerContent,
+    // Either capture a YAML multinline block, or a simple single-line YAML block
+    (_, simple, yaml) => simple || yaml
+  );
+};
+
+/**
+ * Normalizes YAML syntax by fixing some non-cool formatted properties of the
+ * docs schema
+ *
+ * @param {string} yamlContent The raw YAML content to normalize
+ * @returns {string} The normalized YAML content
+ */
+export const normalizeYamlSyntax = yamlContent => {
+  return yamlContent
+    .replace('introduced_in=', 'introduced_in: ')
+    .replace('source_link=', 'source_link: ')
+    .replace('type=', 'type: ')
+    .replace('name=', 'name: ')
+    .replace('llm_description=', 'llm_description: ')
+    .replace(/^[\r\n]+|[\r\n]+$/g, ''); // Remove initial and final line breaks
+};
+
+/**
+ * Parses Markdown YAML source into a JavaScript object containing all the metadata
+ * (this is forwarded to the parser so it knows what to do with said metadata)
+ *
+ * @param {string} yamlString The YAML string to be parsed
+ * @returns {import('../types').YAMLProperties} The parsed YAML metadata
+ */
+export const parseYAMLIntoMetadata = yamlString => {
+  const normalizedYaml = normalizeYamlSyntax(yamlString);
+
+  // Ensures that the parsed YAML is an object, because even if it is not
+  // i.e. a plain string or an array, it will simply not result into anything
+  let parsedYaml = yaml.parse(normalizedYaml);
+  // Ensure that only Objects get parsed on Object.keys(), since some `<!--`
+  // comments, might be just plain strings and not even a valid YAML metadata
+  if (typeof parsedYaml === 'string') {
+    return { tags: [parsedYaml] };
+  }
+
+  return parsedYaml;
+};

package/src/generators/types.d.ts

@@ -0,0 +1,129 @@
+import type { publicGenerators, allGenerators } from './index.mjs';
+
+declare global {
+  // Public generators exposed to the CLI
+  export type AvailableGenerators = typeof publicGenerators;
+
+  // All generators including internal ones (metadata, jsx-ast, ast-js)
+  export type AllGenerators = typeof allGenerators;
+
+  /**
+   * ParallelWorker interface for distributing work across Node.js worker threads.
+   * Streams results as chunks complete, enabling pipeline parallelism.
+   */
+  export interface ParallelWorker {
+    /**
+     * Processes items in parallel across worker threads and yields results
+     * as each chunk completes. Enables downstream processing to begin
+     * while upstream chunks are still being processed.
+     *
+     * @param items - Items to process (determines chunk distribution)
+     * @param opts - Additional options to pass to workers
+     * @yields Each chunk's results as they complete
+     */
+    stream<T, R>(
+      items: T[],
+      opts?: Record<string, unknown>
+    ): AsyncGenerator<R[], void, unknown>;
+  }
+
+  export interface ParallelTaskOptions {
+    /** Resolved import specifier the worker loads the generator from */
+    generatorSpecifier: string;
+    input: unknown[];
+    itemIndices: number[];
+  }
+
+  /**
+   * Type for the generate function of a generator
+   * @template I - Input type
+   * @template O - Output type (can be AsyncGenerator or Promise)
+   */
+  export type Generate<I, O> = (
+    input: I,
+    worker: ParallelWorker
+  ) => O extends AsyncGenerator ? O : Promise<O>;
+
+  /**
+   * Type for the optional processChunk function of a generator
+   * @template I - Input type
+   * @template O - Output type
+   * @template D - Dependencies type
+   */
+  export type ProcessChunk<I, O, D = {}> = (
+    slicedInput: I[],
+    itemIndices: number[],
+    dependencies: D
+  ) => Promise<O>;
+
+  export type GeneratorMetadata<
+    C extends any,
+    G extends Generate<any, any>,
+    P extends ProcessChunk<any, any, any> | undefined = undefined,
+  > = {
+    readonly defaultConfiguration: C;
+
+    // The name of the Generator. Must match the Key in AllGenerators
+    name: keyof AllGenerators;
+
+    description: string;
+
+    hasParallelProcessor?: boolean;
+
+    /**
+     * The immediate generator that this generator depends on.
+     * For example, the `html` generator depends on the `react` generator.
+     *
+     * If a given generator has no "before" generator, it will be considered a top-level
+     * generator, and run in parallel.
+     *
+     * Assume you pass to the `createGenerator`: ['json', 'html'] as the generators,
+     * this means both the 'json' and the 'html' generators will be executed and generate their
+     * own outputs in parallel. If the 'html' generator depends on the 'react' generator, then
+     * the 'react' generator will be executed first, then the 'html' generator.
+     *
+     * But both 'json' and 'html' generators will be executed in parallel.
+     *
+     * If you pass `createGenerator` with ['react', 'html'], the 'react' generator will be executed first,
+     * as it is a top level generator and then the 'html' generator would be executed after the 'react' generator.
+     *
+     * The 'ast' generator is the top-level parser for markdown files. It has no dependencies.
+     *
+     * The `ast-js` generator is the top-level parser for JavaScript files. It
+     * passes the ASTs for any JavaScript files given in the input.
+     *
+     * Declared as an import specifier (e.g. `@doc-kit/core/metadata`),
+     * so a dependency may live in any installed package.
+     */
+    dependsOn: string | undefined;
+
+    /**
+     * Generators are abstract and the different generators have different sort of inputs and outputs.
+     * For example, a MDX generator would take the raw AST and output MDX with React Components;
+     * Whereas a JSON generator would take the raw AST and output JSON;
+     * Then a React generator could receive either the raw AST or the MDX output and output React Components.
+     * (depending if they support such I/O)
+     *
+     * Hence you can combine different generators to achieve different outputs.
+     */
+    generate: G;
+
+    /**
+     * Optional method for chunk-level parallelization using real worker threads.
+     * Called by chunk-worker.mjs when processing items in parallel.
+     *
+     * Generators that implement this method can have their work distributed
+     * across multiple worker threads for true parallel processing.
+     *
+     * Input is automatically sliced to only include items at the specified indices,
+     * reducing serialization overhead. The itemIndices are remapped to 0-based
+     * indices into the sliced array.
+     *
+     * @param slicedInput - Sliced input containing only items for this chunk
+     * @param itemIndices - Array of 0-based indices into slicedInput
+     * @param dependencies - Generator options (without worker, which isn't serializable)
+     * @returns Array of results for the processed items
+     */
+    processChunk?: P;
+  };
+}

package/src/generators.mjs

@@ -0,0 +1,145 @@
+'use strict';
+
+import { createCache } from './caching.mjs';
+import {
+  loadGenerators,
+  resolveGeneratorSpecifier,
+} from './generators/loader.mjs';
+import logger from './logger/index.mjs';
+import createWorkerPool from './threading/index.mjs';
+import createParallelWorker from './threading/parallel.mjs';
+import { isAsyncIterable } from './utils/misc.mjs';
+
+const generatorsLogger = logger.child('generators');
+
+/**
+ * Creates a generator orchestration system that manages the execution of
+ * documentation generators in dependency order, with support for parallel
+ * processing and streaming results.
+ */
+const createGenerator = () => {
+  // Owns all caching: generator results, stream collection, and the
+  // consumer-count bookkeeping that drives eviction.
+  const cache = createCache();
+
+  /** @type {import('piscina').Piscina} */
+  let pool;
+
+  /**
+   * Gets the collected input from a dependency generator.
+   *
+   * @param {string | undefined} dependsOn - Resolved dependency specifier
+   * @returns {Promise<unknown>}
+   */
+  const getDependencyInput = async dependsOn => {
+    if (!dependsOn) {
+      return undefined;
+    }
+
+    return cache.consume(dependsOn);
+  };
+
+  /**
+   * Schedules a generator and its dependencies for execution.
+   *
+   * @param {string} specifier - Resolved generator specifier to schedule
+   * @param {Map<string, GeneratorMetadata>} generators - Loaded generators
+   * @param {import('./utils/configuration/types').Configuration} configuration - Runtime options
+   */
+  const scheduleGenerator = (specifier, generators, configuration) => {
+    if (cache.has(specifier)) {
+      return;
+    }
+
+    const generator = generators.get(specifier);
+    const { name, generate, hasParallelProcessor } = generator;
+
+    const dependsOn =
+      generator.dependsOn && resolveGeneratorSpecifier(generator.dependsOn);
+
+    // Schedule dependency first
+    if (dependsOn && !cache.has(dependsOn)) {
+      scheduleGenerator(dependsOn, generators, configuration);
+    }
+
+    generatorsLogger.debug(`Scheduling "${name}"`, {
+      dependsOn: dependsOn || 'none',
+      streaming: hasParallelProcessor,
+    });
+
+    // Schedule the generator
+    cache.store(
+      specifier,
+      (async () => {
+        const dependencyInput = await getDependencyInput(dependsOn);
+
+        generatorsLogger.debug(`Starting "${name}"`);
+
+        // Create parallel worker for streaming generators
+        const worker = hasParallelProcessor
+          ? createParallelWorker(specifier, generator, pool, configuration)
+          : Promise.resolve(null);
+
+        const result = await generate(dependencyInput, await worker);
+
+        // For streaming generators, "Completed" is logged when the cache
+        // finishes collecting, not here when the generator returns
+        if (!isAsyncIterable(result)) {
+          generatorsLogger.debug(`Completed "${name}"`);
+        }
+
+        return result;
+      })()
+    );
+  };
+
+  /**
+   * Runs all requested generators with their dependencies.
+   *
+   * @param {import('./utils/configuration/types').Configuration} configuration - Runtime options
+   * @returns {Promise<unknown[]>} Results of all requested generators
+   */
+  const runGenerators = async configuration => {
+    const { target, threads } = configuration;
+
+    // Resolve shorthand names and load the full dependency closure up front,
+    // so scheduling below is fully synchronous.
+    const targets = target.map(resolveGeneratorSpecifier);
+    const generators = await loadGenerators(targets);
+
+    generatorsLogger.debug(`Starting pipeline`, {
+      generators: targets.join(', '),
+      threads,
+    });
+
+    // Compute consumer counts up front so dependencies can be evicted as soon
+    // as their last consumer runs (must be ready before any generator starts).
+    cache.populateConsumerCounts(targets, specifier => {
+      const { dependsOn } = generators.get(specifier);
+
+      return dependsOn && resolveGeneratorSpecifier(dependsOn);
+    });
+
+    // Create worker pool
+    pool = createWorkerPool(threads);
+
+    // Schedule all generators
+    for (const specifier of targets) {
+      scheduleGenerator(specifier, generators, configuration);
+    }
+
+    // Start all collections in parallel (don't await sequentially). Consuming
+    // through the shared path lets the final read also trigger eviction.
+    const results = await Promise.all(
+      targets.map(specifier => cache.consume(specifier))
+    );
+
+    await pool.destroy();
+
+    return results;
+  };
+
+  return { runGenerators };
+};
+
+export default createGenerator;

package/src/logger/constants.mjs

@@ -0,0 +1,34 @@
+'use strict';
+
+/**
+ * Numeric log level definitions.
+ */
+export const LogLevel = {
+  debug: 10,
+  info: 20,
+  warn: 30,
+  error: 40,
+  fatal: 50,
+};
+
+/**
+ * Maps log level numbers to their string tags.
+ */
+export const levelTags = {
+  [LogLevel.debug]: 'DEBUG',
+  [LogLevel.info]: 'INFO',
+  [LogLevel.warn]: 'WARN',
+  [LogLevel.error]: 'ERROR',
+  [LogLevel.fatal]: 'FATAL',
+};
+
+/**
+ * Maps log level numbers to CLI color names.
+ */
+export const levelToColorMap = {
+  [LogLevel.debug]: 'blue',
+  [LogLevel.info]: 'green',
+  [LogLevel.warn]: 'yellow',
+  [LogLevel.error]: 'magenta',
+  [LogLevel.fatal]: 'red',
+};

package/src/logger/index.mjs

@@ -0,0 +1,27 @@
+'use strict';
+
+import { createLogger } from './logger.mjs';
+import { transports } from './transports/index.mjs';
+
+/**
+ * @typedef {ReturnType<typeof createLogger>} LoggerInstance
+ */
+
+/**
+ * Creates a new logger instance with the specified transport.
+ *
+ * @param {string} [transportName='console'] - Name of the transport to use.
+ * @returns {LoggerInstance}
+ */
+export const Logger = (transportName = 'console') => {
+  const transport = transports[transportName];
+
+  if (!transport) {
+    throw new Error(`Transport '${transportName}' not found.`);
+  }
+
+  return createLogger(transport);
+};
+
+// Default logger instance using console transport
+export default Logger();