@doc-kit/core 1.1.0 → 1.2.1

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @doc-kit/core
 
+## 1.2.1
+
+### Patch Changes
+
+- [#1105](https://github.com/nodejs/doc-kit/pull/1105) [`615aabe`](https://github.com/nodejs/doc-kit/commit/615aabeb1955a8028146e26fd4589c1172f58367) Thanks [@ryzrr](https://github.com/ryzrr)! - fix(metadata): link class types whose name starts with an acronym
+
+## 1.2.0
+
+### Minor Changes
+
+- [#1079](https://github.com/nodejs/doc-kit/pull/1079) [`c4ec767`](https://github.com/nodejs/doc-kit/commit/c4ec76753c523cccb636a7f294669ec81d77dd47) Thanks [@avivkeller](https://github.com/avivkeller)! - feat: the `json` and `json-all` generators
+
+### Patch Changes
+
+- [#1058](https://github.com/nodejs/doc-kit/pull/1058) [`2a356c3`](https://github.com/nodejs/doc-kit/commit/2a356c356858c631c45f1826edfa42a24b435d51) Thanks [@zeexzeex](https://github.com/zeexzeex)! - fix: respect `--log-level` in worker threads
+
 ## 1.1.0
 
 ### Minor Changes

package/README.md

@@ -10,7 +10,8 @@ The command-line interface lives in the companion
 ## Generators
 
 Output formats are provided by generators. This package ships the shared
-pipeline stages and `json-simple`; the rest come from companion packages:
+pipeline stages and the JSON generators (`json`, `json-all`, and the
+debugging-only `json-simple`); the rest come from companion packages:
 
 | Package                                                                                | Generators                                                                            |
 | -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- |

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@doc-kit/core",
   "type": "module",
-  "version": "1.1.0",
+  "version": "1.2.1",
   "description": "The doc-kit engine: the pipeline and shared APIs that power the doc-kit generators and CLI",
   "repository": {
     "type": "git",
@@ -14,6 +14,8 @@
     ".": "./src/generators.mjs",
     "./ast": "./src/generators/ast/index.mjs",
     "./ast-js": "./src/generators/ast-js/index.mjs",
+    "./json": "./src/generators/json/index.mjs",
+    "./json-all": "./src/generators/json-all/index.mjs",
     "./json-simple": "./src/generators/json-simple/index.mjs",
     "./metadata": "./src/generators/metadata/index.mjs",
     "./package.json": "./package.json",
@@ -45,7 +47,7 @@
   "dependencies": {
     "@actions/core": "^3.0.0",
     "@node-core/rehype-shiki": "^1.5.0",
-    "@swc/html-wasm": "^1.15.46",
+    "@swc/html-wasm": "^1.16.1",
     "@swc/wasm": "^1.15.46",
     "acorn": "^8.17.0",
     "cosmiconfig": "^9.0.2",
@@ -53,6 +55,8 @@
     "github-slugger": "^2.0.0",
     "glob-parent": "^6.0.2",
     "hastscript": "^9.0.1",
+    "mdast-util-slice-markdown": "^2.0.1",
+    "mdast-util-to-string": "^4.0.0",
     "piscina": "^5.3.1",
     "rehype-stringify": "^10.0.1",
     "remark-gfm": "^4.0.1",
@@ -73,13 +77,14 @@
     "yaml": "^2.9.0"
   },
   "devDependencies": {
+    "ajv": "^8.20.0",
     "hast-util-to-html": "^9.0.5",
     "hast-util-to-string": "^3.0.1"
   },
   "peerDependencies": {
+    "@doc-kit/generator-react": ">=0.1.0",
     "@node-core/doc-kit": ">=2.0.0",
-    "@node-core/doc-kit-legacy": ">=1.0.0",
-    "@doc-kit/generator-react": ">=0.1.0"
+    "@node-core/doc-kit-legacy": ">=1.0.0"
   },
   "peerDependenciesMeta": {
     "@node-core/doc-kit": {

package/src/generators/index.mjs

@@ -10,6 +10,8 @@
  * which is how third-party generator packages are loaded.
  */
 export const publicGenerators = {
+  json: '@doc-kit/core/json',
+  'json-all': '@doc-kit/core/json-all',
   'json-simple': '@doc-kit/core/json-simple',
   'legacy-html': '@node-core/doc-kit-legacy/legacy-html',
   'legacy-html-all': '@node-core/doc-kit-legacy/legacy-html-all',

package/src/generators/json/README.md

@@ -0,0 +1,151 @@
+# `json` Generator
+
+The `json` generator writes one JSON document per source file. Each document
+is a tree of the file's headings, in document order, with the metadata,
+signature or type, Markdown body, and code examples of every one of them.
+
+The output is described by a JSON schema, published at the URL every document
+carries in `$schema`, and shipped with the package as
+`@doc-kit/core/generators/json/schema.json`.
+
+```sh
+npx @doc-kit/cli generate -t json -i "doc/api/*.md" -o out
+```
+
+Files keep the input's directory layout: `doc/api/some/fs.md` becomes `out/some/fs.json`.
+
+## Configuring
+
+- `output` {string} The directory to write the documents to.
+- `minify` {boolean} Whether to minify the output. Inherited from `global`.
+  **Default:** `true`.
+- `repository` {string} The `owner/name` repository source links resolve
+  against. Inherited from `global`; without one, `sourceLink.url` is `null`.
+- `schemaURL` {string} Where the schema is published. `{schemaVersion}` is
+  filled in. **Default:**
+  `'https://doc-kit.nodejs.org/schemas/api-doc/{schemaVersion}.json'`.
+
+## The document
+
+```json
+{
+  "$schema": "https://doc-kit.nodejs.org/schemas/api-doc/1.0.0.json",
+  "id": "fs",
+  "path": "/fs",
+  "type": "module",
+  "module": "fs",
+  "title": "File system",
+  "introducedIn": "v0.10.0",
+  "sourceLink": {
+    "path": "lib/fs.js",
+    "url": "https://github.com/nodejs/node/blob/HEAD/lib/fs.js"
+  },
+  "stability": { "index": "2", "description": "Stable" },
+  "added": [],
+  "deprecated": [],
+  "removed": [],
+  "napiVersion": [],
+  "changes": [],
+  "description": "The `node:fs` module enables interacting with the file system in a\nway modeled on standard POSIX functions.\n\n…",
+  "summary": "The `node:fs` module enables interacting with the file system in a way modeled on standard POSIX functions.",
+  "examples": [],
+  "children": []
+}
+```
+
+- `id` is the file's path, slugged; `path` is that path without extension.
+  Cross-document links in Markdown target `<path>.html`.
+- `type` is the file's `type=` directive: `module` (the default), `misc`, or
+  `global`. `module` is the module's name, from the `name=` directive.
+- `introducedIn` and `sourceLink` are the `introduced_in=` and `source_link=`
+  directives.
+- Everything from `title` on is what every heading carries, described below.
+- `children` are the file's headings, nested by depth.
+
+## Every entry
+
+The document and every node carry:
+
+- `title` The heading text as authored, inline Markdown included.
+- `stability` The stability index, or `null`: `{ index, description }`,
+  where `index` is the text as authored (`"1.1"`) and `description` the
+  Markdown after it.
+- `added`, `deprecated`, `removed` Arrays of version strings, as authored.
+- `napiVersion` An array of numbers.
+- `changes` The change history: `{ versions, prUrl, commit, description }`.
+- `description` The body as Markdown: everything under the heading except
+  its metadata, its stability index, and the typed list a signature or type
+  was taken from. Links are rewritten as they are for HTML output.
+- `summary` A plain-text paragraph: the `llm_description` when there is one,
+  else the first paragraph.
+- `examples` The fenced code blocks of the body, `{ language, displayName, code }`.
+  They stay in the description too.
+
+Every key is always present. What is missing is `null` or an empty array.
+
+## Nodes
+
+A node is a heading below the title. Its `kind` says what the heading
+documents, and decides which further properties it has:
+
+| `kind`         | Heading                                        | Further properties           |
+| -------------- | ---------------------------------------------- | ---------------------------- |
+| `section`      | Anything else: prose, `DEP0005: …`, `--flag`   | none                         |
+| `class`        | `` Class: `net.Server` ``                      | `extends` (`Type` or `null`) |
+| `constructor`  | `` `new Agent([options])` ``                   | `signature`                  |
+| `method`       | `` `fs.readFile(path[, options], callback)` `` | `signature`                  |
+| `staticMethod` | `` Static method: `Buffer.from(string)` ``     | `signature`                  |
+| `property`     | `` `buf.length` ``                             | `type`, `default`            |
+| `event`        | `` Event: `'close'` ``                         | `parameters`                 |
+
+Every node also has:
+
+- `id` The heading's slug, and its anchor in HTML output.
+- `name` The bare identifier: `readFile`, `Server`, `close`. A section's
+  plain heading text. A `name=` directive overrides it.
+- `scope` `module`, or `global` for entries typed `global`.
+- `overloadOf` When several sibling headings document one callable, the `id`
+  of the first on the second and later ones; otherwise `null`.
+- `children` The headings nested under it.
+
+## Signatures and types
+
+```json
+"signature": {
+  "parameters": [
+    {
+      "name": "options",
+      "type": { "text": "Object | string", "links": [{ "name": "Object", "href": "https://developer.mozilla.org/…", "start": 0, "end": 6 }] },
+      "description": "",
+      "default": null,
+      "optional": true,
+      "rest": false,
+      "properties": [
+        { "name": "encoding", "type": { "text": "string | null", "links": [] }, "description": "", "default": "null", "optional": true, "rest": false, "properties": [] }
+      ]
+    }
+  ],
+  "returns": { "type": { "text": "Promise", "links": [] }, "description": "Fulfills upon success." }
+}
+```
+
+- A signature's `parameters` are the ones the heading declares,
+  described by the typed list under it. `optional` is set for parameters
+  bracketed in the heading or documented with a default; `rest` for
+  `...args`. `properties` are the nested list items: the properties of an
+  options object, or the arguments of a callback.
+- `returns` is the `Returns:` item, or `null`.
+- A `Type` is the annotation's TypeScript text, normalised to one line with
+  `|` between union members, plus the names in it that resolved to
+  documentation, with their character offsets. A type that was not
+  documented is `null`, never guessed.
+- An event's `parameters` are the arguments its listeners receive.
+
+## The schema
+
+`schema.json` is the source of truth. After changing it, bump its `$id` and
+`SCHEMA_VERSION` in `constants.mjs` together, and regenerate the types:
+
+```sh
+node scripts/generate-json-types.mjs
+```

package/src/generators/json/constants.mjs

@@ -0,0 +1,57 @@
+'use strict';
+
+// See the `version` in schema.json. These values should be in sync.
+export const SCHEMA_VERSION = '1.0.1';
+
+// Where a schema version is published.
+export const SCHEMA_URL =
+  'https://doc-kit.nodejs.org/schemas/api-doc/{schemaVersion}.json';
+
+// The heading classifications the metadata generator assigns, and the node
+// kind each one becomes.
+export const KINDS = {
+  class: 'class',
+  ctor: 'constructor',
+  method: 'method',
+  classMethod: 'staticMethod',
+  property: 'property',
+  event: 'event',
+};
+
+// The kind of a heading that documents no API entry
+export const SECTION_KIND = 'section';
+
+// The kinds whose leading typed list is lifted out of the body as data
+export const KINDS_WITH_TYPED_LIST = new Set([
+  'class',
+  'constructor',
+  'method',
+  'staticMethod',
+  'property',
+  'event',
+]);
+
+// The kinds whose heading declares the parameters of a signature
+export const CALLABLE_KINDS = new Set([
+  'constructor',
+  'method',
+  'staticMethod',
+]);
+
+// The document-level `type` classifications
+export const DOCUMENT_TYPES = new Set(['module', 'misc', 'global']);
+
+// The `displayName="..."` attribute of a fenced code block's info string
+export const DISPLAY_NAME = /displayName="([^"]*)"/;
+
+// A rest parameter's marker
+export const REST_MARKER = /^\.\.\./;
+
+// The code span around a default value
+export const CODE_SPAN = /^`([^]*)`$/;
+
+// The `=` a default value declared in a heading starts with
+export const DECLARED_DEFAULT = /^=\s*/;
+
+// The `extends` clause a class heading may carry: `Class: \`Foo extends Bar\``
+export const EXTENDS_CLAUSE = / extends +/;

package/src/generators/json/generate.mjs

@@ -0,0 +1,65 @@
+'use strict';
+
+import { join } from 'node:path';
+
+import getConfig from '#utils/configuration/index.mjs';
+import { GITHUB_BLOB_URL, populate } from '#utils/configuration/templates.mjs';
+import { withExt, writeJSON } from '#utils/file.mjs';
+import { groupNodesByModule } from '#utils/generators.mjs';
+
+import { buildDocument } from './utils/document.mjs';
+import { resolveSchemaURL } from './utils/schema.mjs';
+
+/**
+ * Builds the documents of a chunk of modules in a worker thread.
+ *
+ * @type {import('./types').Generator['processChunk']}
+ */
+export async function processChunk(slicedInput, itemIndices, dependencies) {
+  return itemIndices.map(idx => {
+    const { head, entries } = slicedInput[idx];
+
+    return buildDocument(head, entries, dependencies);
+  });
+}
+
+/**
+ * Generates one JSON document per input file.
+ *
+ * @type {import('./types').Generator['generate']}
+ */
+export async function* generate(input, worker) {
+  const config = getConfig('json');
+
+  /** @type {import('./types').Dependencies} */
+  const dependencies = {
+    schemaURL: resolveSchemaURL(config),
+    // Source links resolve against the repository, when one is configured
+    sourceURL: config.repository ? populate(GITHUB_BLOB_URL, config) : null,
+  };
+
+  // Pages other generators added to the pipeline are theirs to render
+  const entries = input.filter(entry => !entry.synthetic && !entry.chunk);
+
+  // One item per module, so a worker gets everything a document needs
+  const modules = [...groupNodesByModule(entries).values()].map(nodes => ({
+    head: nodes.find(({ heading }) => heading.depth === 1) ?? nodes[0],
+    entries: nodes,
+  }));
+
+  for await (const chunk of worker.stream(modules, dependencies)) {
+    if (config.output) {
+      await Promise.all(
+        chunk.map(document =>
+          writeJSON(
+            join(config.output, withExt(document.path, 'json')),
+            document,
+            config.minify
+          )
+        )
+      );
+    }
+
+    yield chunk;
+  }
+}