@doc-kit/core 0.0.0

package/src/generators/metadata/types.d.ts

@@ -0,0 +1,145 @@
+import type { Position } from 'unist';
+import type { Root, Node, Data, Blockquote, Heading } from 'mdast';
+
+export type Generator = GeneratorMetadata<
+  {},
+  Generate<Array<Root>, AsyncGenerator<MetadataEntry>>,
+  ProcessChunk<Root, MetadataEntry, Record<string, string>>
+>;
+
+/**
+ * Utility type to augment AST nodes with typed data properties.
+ *
+ * This ensures type safety when working with nodes that have been
+ * enhanced with additional metadata during processing.
+ *
+ * @template T - The base AST node type
+ * @template J - The data structure to attach
+ */
+type NodeWithData<T extends Node, J extends Data> = T & {
+  data: J;
+};
+
+export interface ChangeEntry {
+  // The Node.js version or versions where said change was introduced simultaneously
+  version: string | Array<string>;
+  // The GitHub PR URL of said change
+  'pr-url': string | undefined;
+  // The description of said change
+  description: string;
+}
+
+/**
+ * YAML frontmatter properties commonly found in API documentation.
+ *
+ * These properties provide metadata about API elements including
+ * versioning, deprecation status, stability indicators, and
+ * additional documentation context.
+ */
+export interface YAMLProperties {
+  /** Source position information for the YAML block */
+  yamlPosition?: Position;
+
+  // === Core Metadata ===
+  /** Type classification of the API element */
+  type?: string;
+  /** Link to the source code implementation */
+  source_link?: string;
+  /** Human-readable description for LLM processing */
+  llm_description?: string;
+  /** Associated tags for categorization */
+  tags?: Array<string>;
+
+  /** Version when this API was first added */
+  added?: string;
+  /** Version when this API was first added */ // TODO(@avivkeller): Merge this with `added`
+  introduced_in?: string;
+  /** Version when this API was deprecated */
+  deprecated?: string;
+  /** Version when this API was removed */
+  removed?: string;
+
+  /** N-API version requirement (legacy format) */
+  napiVersion?: string;
+
+  /** Array of change records with version info */
+  changes?: Array<ChangeEntry>;
+
+  /** Allow for additional YAML properties not explicitly defined */
+  [key: string]: unknown;
+}
+
+export type HeadingType =
+  | 'event'
+  | 'global'
+  | 'method'
+  | 'property'
+  | 'class'
+  | 'module'
+  | 'classMethod'
+  | 'ctor';
+
+/**
+ * Data structure for enhanced heading nodes.
+ * Extracted as a separate interface for reusability and clarity.
+ */
+export interface HeadingData extends Data {
+  /** Extracted plain text content of the heading */
+  text: string;
+  /** The name of the module that this heading refers to */
+  name: string;
+  /** URL-safe slug derived from heading text */
+  slug: string;
+  /** Optional type classification */
+  type?: HeadingType;
+  /**
+   * Marks the 2nd..nth heading of an overloaded function so it can be dropped
+   * from the ToC while still rendering on the page.
+   */
+  isOverload?: boolean;
+}
+
+/**
+ * Represents a processed heading that includes both the original
+ * AST structure and computed metadata like text content and URL slug.
+ */
+export type HeadingNode = NodeWithData<Heading, HeadingData>;
+
+/**
+ * Data structure for stability index entries.
+ */
+export interface StabilityData extends Data {
+  /** Numeric stability index (0-3) */
+  index: string;
+  /** Human-readable stability description */
+  description: string;
+}
+
+/**
+ * Individual stability entry with metadata.
+ */
+export type StabilityNode = NodeWithData<Blockquote, StabilityData>;
+
+/**
+ * Complete metadata entry for a single API documentation element.
+ *
+ * This represents the processed result of parsing a section of API
+ * documentation, containing all the structured information needed
+ * for documentation generation, search indexing, and validation.
+ */
+export interface MetadataEntry extends YAMLProperties {
+  /** Path + API identification */
+  api: string;
+  path: string; // Note: this is extensionless
+  basename: string;
+  /** Whether this page was generated by a downstream generator. */
+  synthetic?: boolean;
+  /** Whether this page was authored as MDX (JSX-in-Markdown). */
+  mdx?: boolean;
+  /** Processed heading with metadata */
+  heading: HeadingNode;
+  /** Stability classification information */
+  stability: StabilityNode;
+  /** Main content as markdown AST */
+  content: Root;
+}

package/src/generators/metadata/utils/parse.mjs

@@ -0,0 +1,160 @@
+'use strict';
+
+import { basename, sep } from 'node:path/posix';
+
+import { slug } from 'github-slugger';
+import { u as createTree } from 'unist-builder';
+import { findAfter } from 'unist-util-find-after';
+import { remove } from 'unist-util-remove';
+import { selectAll } from 'unist-util-select';
+import { SKIP, visit } from 'unist-util-visit';
+
+import createNodeSlugger from './slugger.mjs';
+import { transformNodeToHeading } from './transformers.mjs';
+import {
+  visitLinkReference,
+  visitMarkdownLink,
+  visitStability,
+  visitTextWithUnixManualNode,
+  visitYAML,
+} from './visitors.mjs';
+import { UNIST } from '../../../utils/queries/index.mjs';
+import { getRemark as remark } from '../../../utils/remark.mjs';
+import { relative } from '../../../utils/url.mjs';
+import {
+  DEPRECATION_HEADING_REGEX,
+  IGNORE_STABILITY_STEMS,
+} from '../constants.mjs';
+import { resolveTypeAnnotations } from './resolveTypes.mjs';
+
+/**
+ * This generator generates a flattened list of metadata entries from a API doc
+ *
+ * @param {{ tree: import('mdast.Root'), mdx?: boolean } & import('../types').MetadataEntry} input
+ * @param {Record<string, string>} typeMap
+ * @returns {Array<import('../types').MetadataEntry>}
+ */
+export const parseApiDoc = ({ path, tree, mdx = false }, typeMap) => {
+  /**
+   * Collection of metadata entries for the file
+   * @type {Array<import('../types').MetadataEntry>}
+   */
+  const metadataCollection = [];
+
+  // Creates a new Slugger instance for the current API doc file
+  const nodeSlugger = createNodeSlugger();
+
+  // Slug the API (We use a non-class slugger, since we are fairly certain that `path` is unique)
+  const api = slug(path.slice(1).replace(sep, '-'));
+
+  /**
+   * Creates a stable slug for a heading in the current API document.
+   * @param {string} text Heading text to slug.
+   * @returns {string} The generated heading slug.
+   */
+  const getHeadingSlug = text => {
+    const deprecationHeading = DEPRECATION_HEADING_REGEX.exec(text);
+
+    return deprecationHeading ? deprecationHeading[1] : nodeSlugger.slug(text);
+  };
+
+  // Get all Markdown Footnote definitions from the tree
+  const markdownDefinitions = selectAll('definition', tree);
+
+  // Get all Markdown Heading entries from the tree
+  const headingNodes = selectAll('heading', tree);
+
+  // Handles Markdown link references and updates them to be plain links
+  visit(tree, UNIST.isLinkReference, node =>
+    visitLinkReference(node, markdownDefinitions)
+  );
+
+  // Removes all the original definitions from the tree as they are not needed anymore
+  remove(tree, markdownDefinitions);
+
+  // Make all the typeMap links relative to us
+  const relativeTypeMap = Object.fromEntries(
+    Object.entries(typeMap).map(([type, url]) => [type, relative(url, path)])
+  );
+
+  // Resolve every type annotation in the file at once (one TypeScript parse
+  // per file). MDX trees never contain `typeAnnotation` nodes — there,
+  // `{...}` is a real expression — so this is skipped.
+  if (!mdx) {
+    resolveTypeAnnotations(tree, relativeTypeMap, path);
+  }
+
+  // Handles the normalisation URLs that reference to API doc files with .md extension
+  visit(tree, UNIST.isMarkdownUrl, node => visitMarkdownLink(node));
+
+  // If the document has no headings but it has content, we add a fake heading to the top
+  if (headingNodes.length === 0 && tree.children.length > 0) {
+    tree.children.unshift(createTree('heading', { depth: 1 }, []));
+  }
+
+  // On "About this Documentation", we define the stability indices, and thus
+  // we don't need to check it for stability references
+  const ignoreStability = IGNORE_STABILITY_STEMS.includes(api);
+
+  // Process each heading and create metadata entries
+  visit(tree, UNIST.isHeading, (headingNode, index) => {
+    // Initialize heading
+    headingNode.data = transformNodeToHeading(headingNode);
+    // Initialize the metadata
+    const metadata = /** @type {import('../types').MetadataEntry} */ ({
+      api,
+      path,
+      mdx,
+      basename: basename(path),
+      heading: headingNode,
+    });
+
+    // Generate slug and update heading data
+    metadata.heading.data.slug = getHeadingSlug(metadata.heading.data.text);
+
+    // Find the next heading to determine section boundaries
+    const nextHeadingNode =
+      findAfter(tree, index, UNIST.isHeading) ?? headingNode;
+
+    const stop =
+      headingNode === nextHeadingNode
+        ? tree.children.length
+        : tree.children.indexOf(nextHeadingNode);
+
+    // Create subtree for this section. If it's the first entry, we start from the
+    // beginning of the tree to ensure top-level nodes (like YAML) are captured.
+    const startIndex = metadataCollection.length === 0 ? 0 : index;
+    const subTree = createTree('root', tree.children.slice(startIndex, stop));
+
+    visit(subTree, UNIST.isStabilityNode, node =>
+      visitStability(node, ignoreStability ? undefined : metadata)
+    );
+
+    // Process YAML nodes directly - merge all YAML data
+    visit(subTree, UNIST.isYamlNode, node => visitYAML(node, metadata));
+
+    // If YAML data contains a 'type', use it to override heading type
+    if (metadata.type) {
+      metadata.heading.data.type = metadata.type;
+    }
+
+    // Process Unix manual references
+    visit(subTree, UNIST.isTextWithUnixManual, (node, _, parent) =>
+      visitTextWithUnixManualNode(node, parent)
+    );
+
+    // Remove processed YAML nodes from the content
+    remove(subTree, [UNIST.isYamlNode]);
+
+    // Apply AST transformations
+    const parsedSubTree = remark().runSync(subTree);
+    metadata.content = parsedSubTree;
+
+    // Add to collection
+    metadataCollection.push(metadata);
+
+    return SKIP;
+  });
+
+  return metadataCollection;
+};

package/src/generators/metadata/utils/resolveTypes.mjs

@@ -0,0 +1,204 @@
+import { parseSync } from '@swc/wasm';
+import { visit } from 'unist-util-visit';
+
+import { lookupTypeName, resolveTypeReference } from './transformers.mjs';
+import logger from '../../../logger/index.mjs';
+import { walk } from '../../../utils/swc.mjs';
+
+// `null` is a keyword kind in SWC, unlike most parsers which treat it as a literal type
+const KEYWORDS = new Set([
+  'string',
+  'number',
+  'boolean',
+  'any',
+  'unknown',
+  'never',
+  'void',
+  'undefined',
+  'object',
+  'bigint',
+  'symbol',
+  'null',
+]);
+
+/**
+ * Parses a wrapped `type $Ti = <value>;` statement
+ */
+const parseAliases = values => {
+  const prefixes = [];
+  let offset = 0;
+
+  const source = values
+    .map((value, i) => {
+      const decl = `type $T${i} = `;
+      prefixes.push(offset + decl.length);
+      const line = `${decl}${value};\n`;
+      offset += line.length;
+      return line;
+    })
+    .join('');
+
+  let program;
+
+  try {
+    program = parseSync(source, { syntax: 'typescript' });
+  } catch {
+    return undefined;
+  }
+
+  // SWC spans are absolute and keep growing across parse calls; the module's
+  // span begins at the first token, which is the start of our source
+  const spanOffset = program.span.start;
+
+  const identifiersPerAlias = values.map(() => []);
+
+  /**
+   *
+   */
+  const record = node => {
+    const start = node.span.start - spanOffset;
+    const end = node.span.end - spanOffset;
+
+    const index = Math.max(
+      0,
+      prefixes.findLastIndex(prefix => prefix <= start)
+    );
+
+    const base = prefixes[index];
+    const text = source.slice(start, end);
+
+    identifiersPerAlias[index].push({
+      start: start - base,
+      end: end - base,
+      text,
+      lookup: text,
+    });
+  };
+
+  walk(program, {
+    /**
+     *
+     */
+    TsTypeReference: node => record(node.typeName), // Promise<T>, vm.Module
+    /**
+     *
+     */
+    TsTypeQuery: node => record(node.exprName), // typeof Foo
+    /**
+     *
+     */
+    TsImportType: node => node.qualifier && record(node.qualifier), // import('fs').Stats
+    /**
+     *
+     */
+    TsKeywordType: node => KEYWORDS.has(node.kind) && record(node), // string, null, ...
+    TsThisType: record,
+  });
+
+  return identifiersPerAlias.map(identifiers => ({ identifiers }));
+};
+
+/**
+ * Parses type values
+ */
+export const parseTypeValues = values => {
+  const batch = parseAliases(values);
+
+  if (batch) {
+    return batch;
+  }
+
+  return values.map(value => parseAliases([value])?.[0] ?? { error: true });
+};
+
+/**
+ * Resolves a type value as a `|`-separated union of display names (e.g.
+ * `HTTP/2 Headers Object | HTTP/2 Raw Headers`). Map keys may contain spaces
+ * and slashes, so such values may not parse as TypeScript, but each part is
+ * still resolvable on its own.
+ *
+ * @param {string} value The type value
+ * @param {Record<string, string>} typeMap The toolchain mapping of types to links
+ * @returns {Array<{ start: number, end: number, text: string, href: string }>} The resolved links
+ */
+const resolveDisplayNames = (value, typeMap) => {
+  const links = [];
+
+  for (const { 0: part, index } of value.matchAll(/(?:[^|]|\|\|)+/g)) {
+    // An array of a type links to the type itself
+    const name = part.trim().replace(/(\[\])+$/, '');
+    const href = name && resolveTypeReference(name, typeMap);
+
+    if (href) {
+      const start = index + part.indexOf(name);
+
+      links.push({ start, end: start + name.length, text: name, href });
+    }
+  }
+
+  return links;
+};
+
+/**
+ * Resolves a type value's links: the whole value when it is a map key in its
+ * own right, then its parsed identifiers, then its display names
+ *
+ * @param {string} value The type value
+ * @param {Record<string, string>} typeMap The toolchain mapping of types to links
+ * @param {{ error?: boolean, identifiers?: Array<object> }} result The parse of `value`
+ * @returns {Array<{ start: number, end: number, text: string, href: string }>} The resolved links
+ */
+const resolveLinks = (value, typeMap, result) => {
+  const url = lookupTypeName(value, typeMap);
+
+  if (url) {
+    return [{ start: 0, end: value.length, text: value, href: url }];
+  }
+
+  if (result.error) {
+    return resolveDisplayNames(value, typeMap);
+  }
+
+  return (
+    result.identifiers
+      .map(({ start, end, text, lookup }) => ({
+        start,
+        end,
+        text,
+        href: resolveTypeReference(lookup, typeMap),
+      }))
+      .filter(({ href }) => href)
+      // The hast handlers slice the value by these ranges in order
+      .sort((a, b) => a.start - b.start)
+  );
+};
+
+/**
+ * Resolves every `typeAnnotation` node in a file's tree
+ */
+export const resolveTypeAnnotations = (tree, typeMap, path) => {
+  const nodes = [];
+
+  // A visitor must not return a value: `visit` reads a number as the index to
+  // continue from, which would skip siblings
+  visit(tree, 'typeAnnotation', node => {
+    nodes.push(node);
+  });
+
+  parseTypeValues(nodes.map(({ value }) => value)).forEach((result, index) => {
+    const node = nodes[index];
+    const links = resolveLinks(node.value, typeMap, result);
+
+    // Unparseable display names must not be highlighted as the TypeScript
+    // they are not (which colours the `/` and `2` of `HTTP/2`)
+    node.data = { links, typescript: !result.error };
+
+    if (result.error && !links.length) {
+      node.data.parseError = true;
+
+      logger.warn(`Invalid type annotation: {${node.value}}`, {
+        file: { path, position: node.position },
+      });
+    }
+  });
+};

package/src/generators/metadata/utils/slugger.mjs

@@ -0,0 +1,39 @@
+'use strict';
+
+import GitHubSlugger, { slug as defaultSlugFn } from 'github-slugger';
+
+import { DOC_API_SLUGS_REPLACEMENTS } from '../constants.mjs';
+
+/**
+ * Creates a modified version of the GitHub Slugger
+ *
+ * @returns {InstanceType<typeof import('github-slugger').default>} The modified GitHub Slugger
+ */
+const createNodeSlugger = () => {
+  const slugger = new GitHubSlugger();
+  const slugFn = slugger.slug.bind(slugger);
+
+  return {
+    ...slugger,
+    /**
+     * Creates a new slug based on the provided string
+     *
+     * @param {string} title The title to be parsed into a slug
+     */
+    // Applies certain string replacements that are specific
+    // to the way how Node.js generates slugs/anchor IDs
+    slug: title => slug(title, slugFn),
+  };
+};
+
+/**
+ * @param {string} title
+ * @param {typeof defaultSlugFn} slugFn
+ */
+export const slug = (title, slugFn = defaultSlugFn) =>
+  DOC_API_SLUGS_REPLACEMENTS.reduce(
+    (piece, { from, to }) => piece.replace(from, to),
+    slugFn(title)
+  );
+
+export default createNodeSlugger;

package/src/generators/metadata/utils/transformers.mjs

@@ -0,0 +1,109 @@
+import {
+  DOC_MAN_BASE_URL,
+  DOC_API_HEADING_TYPES,
+  MODULE_QUALIFIED_NAME,
+} from '../constants.mjs';
+import { slug } from './slugger.mjs';
+import { transformNodesToString } from '../../../utils/unist.mjs';
+import BUILTIN_TYPE_MAP from '../maps/builtin.json' with { type: 'json' };
+import MDN_TYPE_MAP from '../maps/mdn.json' with { type: 'json' };
+
+/**
+ * @param {string} text The inner text
+ * @param {string} command The manual page
+ * @param {string} sectionNumber The manual section
+ * @param {string} sectionLetter The manual section number
+ */
+export const transformUnixManualToLink = (
+  text,
+  command,
+  sectionNumber,
+  sectionLetter = ''
+) => {
+  return `[\`${text}\`](${DOC_MAN_BASE_URL}${sectionNumber}/${command}.${sectionNumber}${sectionLetter}.html)`;
+};
+
+/**
+ * Looks a type name up in the type maps: the toolchain-provided map first
+ * (Node.js types/modules), then the built-in (TC39) map, then MDN. Map keys
+ * may be arbitrary display names (e.g. `zlib options`), not just identifiers.
+ *
+ * @param {string} name The type name to look up
+ * @param {Record<string, string>} typeMap The toolchain mapping of types to links
+ * @returns {string} The reference URL or empty string if no match
+ */
+export const lookupTypeName = (name, typeMap) => {
+  if (typeMap && name in typeMap) {
+    return typeMap[name];
+  }
+
+  const key = name.toLowerCase();
+
+  // Check in our built-in map (i.e. TC39 objects)
+  if (key in BUILTIN_TYPE_MAP) {
+    return BUILTIN_TYPE_MAP[key];
+  }
+
+  // Check in MDN
+  if (key in MDN_TYPE_MAP) {
+    return MDN_TYPE_MAP[key];
+  }
+
+  return '';
+};
+
+/**
+ * Resolves a type identifier to a documentation URL: map lookups first, then
+ * the dotted-name heuristic for Node.js types like `vm.Module` (which links
+ * to the module's page).
+ *
+ * @param {string} name The type identifier to resolve
+ * @param {Record<string, string>} typeMap The toolchain mapping of types to links
+ * @returns {string} The reference URL or empty string if no match
+ */
+export const resolveTypeReference = (name, typeMap) => {
+  const url = lookupTypeName(name, typeMap);
+
+  if (url) {
+    return url;
+  }
+
+  // Transform Node.js types like 'vm.Something'.
+  if (MODULE_QUALIFIED_NAME.test(name)) {
+    const [mod, ...pieces] = name.split('.');
+    const isClass = pieces.at(-1).match(/^[A-Z][a-z]/);
+
+    return `${mod}.html#${isClass ? 'class-' : ''}${slug(name)}`;
+  }
+
+  return '';
+};
+
+/**
+ * Parses a raw Heading string into Heading metadata
+ *
+ * @param {import('mdast').Heading} heading The raw Heading text
+ * @param {number} depth The depth of the heading
+ * @returns {HeadingMetadataEntry} Parsed Heading entry
+ */
+export const transformNodeToHeading = node => {
+  const text = transformNodesToString(node.children);
+
+  for (const { type, regex } of DOC_API_HEADING_TYPES) {
+    // Attempts to get a match from one of the heading types, if a match is found
+    // we use that type as the heading type, and extract the regex expression match group
+    // which should be the inner "plain" heading content (or the title of the heading for navigation)
+    const [, ...matches] = text.match(regex) ?? [];
+
+    if (matches?.length) {
+      return {
+        text,
+        type,
+        // The highest match group should be used.
+        name: matches.filter(Boolean).at(-1),
+      };
+    }
+  }
+
+  return { text, name: text, depth: node.depth };
+};