@doc-kit/core 1.3.0 → 1.3.1

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @doc-kit/core
 
+## 1.3.1
+
+### Patch Changes
+
+- [#1187](https://github.com/nodejs/doc-kit/pull/1187) [`e49eb52`](https://github.com/nodejs/doc-kit/commit/e49eb52472dfc2707a5966ee25c4dc7201360850) Thanks [@dxnnystyles](https://github.com/dxnnystyles)! - Keep json-simple output generation from mutating shared metadata entries.
+
+- [#1179](https://github.com/nodejs/doc-kit/pull/1179) [`b8d7b77`](https://github.com/nodejs/doc-kit/commit/b8d7b7725f0dbe085d9f32af60fddd7459555af9) Thanks [@ovflowd](https://github.com/ovflowd)! - Builds use less memory, most of all with few threads: each worker's young generation is limited to 96MB, a single thread processes one chunk at a time and lets go of the pages' code before rendering them, and Vite's native bundler gets as many threads as doc-kit and returns the memory it frees. On the Node.js API docs, `--threads 1` now builds with Node.js's `--max-old-space-size=512`, and no longer crashes when `legacy-html` is built alongside other targets.
+
+- [#1180](https://github.com/nodejs/doc-kit/pull/1180) [`92efbbe`](https://github.com/nodejs/doc-kit/commit/92efbbeb727a38b178c9ccb63a3a4f9cfbc110a4) Thanks [@ovflowd](https://github.com/ovflowd)! - Find input files with Node.js's `fs.globSync` instead of `tinyglobby` and `glob-parent`.
+
+- [#1082](https://github.com/nodejs/doc-kit/pull/1082) [`9e21017`](https://github.com/nodejs/doc-kit/commit/9e21017d9597e3f8b5a18a41f3db8e5de216213a) Thanks [@btea](https://github.com/btea)! - Add a `doc-kit preview` command that serves the generated docs locally, building first when any `generate` option is passed. The default output directory (`./out`) is now defined in the shared configuration, so `generate` also writes there when no `--output` is given.
+
+- [#1193](https://github.com/nodejs/doc-kit/pull/1193) [`854a944`](https://github.com/nodejs/doc-kit/commit/854a944699b3a996cb3a8544deaec3fe916fd449) Thanks [@OKAN-SOFTWARE](https://github.com/OKAN-SOFTWARE)! - Keep worker results and discovered source files in deterministic order so repeated documentation builds produce stable output.
+
+- [#1178](https://github.com/nodejs/doc-kit/pull/1178) [`0321911`](https://github.com/nodejs/doc-kit/commit/03219116c78e7f4a721b4e39bc71ba54cb6f0ba8) Thanks [@ovflowd](https://github.com/ovflowd)! - Install fewer packages: `@doc-kit/core` updates `cosmiconfig` to v10 (TypeScript config files now load through Node's type stripping) and no longer installs `dedent`, and `@doc-kit/generator-react` estimates reading time with `Intl.Segmenter` instead of the `reading-time` package and updates `@node-core/ui-components` to 1.8.1, which no longer installs `@vcarl/remark-headings`.
+
 ## 1.3.0
 
 ### Minor Changes

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@doc-kit/core",
   "type": "module",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "description": "The doc-kit engine: the pipeline and shared APIs that power the doc-kit generators and CLI",
   "repository": {
     "type": "git",
@@ -50,10 +50,8 @@
     "@swc/html-wasm": "^1.16.2",
     "@swc/wasm": "^1.15.46",
     "acorn": "^8.17.0",
-    "cosmiconfig": "^9.0.2",
-    "dedent": "^1.7.2",
+    "cosmiconfig": "^10.0.1",
     "github-slugger": "^2.0.0",
-    "glob-parent": "^6.0.2",
     "mdast-util-slice-markdown": "^2.0.1",
     "mdast-util-to-string": "^4.0.0",
     "piscina": "^5.3.2",
@@ -65,7 +63,6 @@
     "remark-stringify": "^11.0.0",
     "semver": "^7.8.5",
     "shiki": "^4.4.3",
-    "tinyglobby": "^0.2.17",
     "unified": "^11.0.5",
     "unist-builder": "^4.0.0",
     "unist-util-find-after": "^5.0.0",
@@ -77,6 +74,7 @@
   },
   "devDependencies": {
     "ajv": "^8.20.0",
+    "dedent": "^1.7.2",
     "hast-util-to-html": "^9.0.5",
     "hast-util-to-string": "^3.0.1"
   },

package/src/generators/ast/generate.mjs

@@ -3,12 +3,11 @@
 import { readFile } from 'node:fs/promises';
 import { relative, sep } from 'node:path/posix';
 
-import globParent from 'glob-parent';
-import { globSync } from 'tinyglobby';
 import { parse as parseYaml } from 'yaml';
 
 import getConfig from '#utils/configuration/index.mjs';
 import { withExt } from '#utils/file.mjs';
+import { findFiles, globParent } from '#utils/glob.mjs';
 import { getProcessor } from '#utils/markdown/processor.mjs';
 import { QUERIES } from '#utils/queries/index.mjs';
 
@@ -117,10 +116,9 @@ export async function* generate(_, worker) {
   const files = config.input.flatMap(input => {
     const parent = globParent(input);
 
-    return globSync(input, { ignore: config.ignore }).map(child => [
-      child,
-      parent,
-    ]);
+    return findFiles(input, { ignore: config.ignore })
+      .sort()
+      .map(child => [child, parent]);
   });
 
   // Parse markdown files in parallel using worker threads

package/src/generators/ast-js/generate.mjs

@@ -4,9 +4,9 @@ import { readFile } from 'node:fs/promises';
 import { extname } from 'node:path';
 
 import { parse } from 'acorn';
-import { globSync } from 'tinyglobby';
 
 import getConfig from '#utils/configuration/index.mjs';
+import { findFiles } from '#utils/glob.mjs';
 
 /**
  * Process a chunk of JavaScript files in a worker thread.
@@ -44,9 +44,9 @@ export async function processChunk(inputSlice, itemIndices) {
 export async function* generate(_, worker) {
   const config = getConfig('ast-js');
 
-  const files = globSync(config.input, { ignore: config.ignore }).filter(
-    p => extname(p) === '.js'
-  );
+  const files = findFiles(config.input, { ignore: config.ignore })
+    .filter(p => extname(p) === '.js')
+    .sort();
 
   // Parse the Javascript sources into ASTs in parallel using worker threads
   // source is both the items list and the fullInput since we use sliceInput

package/src/generators/json-simple/generate.mjs

@@ -16,13 +16,21 @@ import { UNIST } from '#utils/queries/index.mjs';
 export async function generate(input) {
   const config = getConfig('json-simple');
 
-  input.forEach(node =>
-    remove(node.content, [UNIST.isStabilityNode, UNIST.isHeading])
-  );
+  const output = input.map(entry => {
+    const copy = { ...entry, content: structuredClone(entry.content) };
+
+    remove(copy.content, [UNIST.isStabilityNode, UNIST.isHeading]);
+
+    return copy;
+  });
 
   if (config.output) {
-    await writeJSON(join(config.output, 'api-docs.json'), input, config.minify);
+    await writeJSON(
+      join(config.output, 'api-docs.json'),
+      output,
+      config.minify
+    );
   }
 
-  return input;
+  return output;
 }

package/src/generators.mjs

@@ -87,8 +87,10 @@ const createGenerator = () => {
           ? createParallelWorker(specifier, generator, pool, configuration)
           : Promise.resolve(null);
 
-        const result = await generate(dependencyInput, await worker);
-
+        // Returned rather than awaited, so this function doesn't hold on to the
+        // input while the generator runs, and the generator can let it go
+        return generate(dependencyInput, await worker);
+      })().then(result => {
         // For streaming generators, "Completed" is logged when the cache
         // finishes collecting, not here when the generator returns
         if (!isAsyncIterable(result)) {
@@ -96,7 +98,7 @@ const createGenerator = () => {
         }
 
         return result;
-      })()
+      })
     );
   };
 

package/src/threading/constants.mjs

@@ -1,5 +1,12 @@
 'use strict';
 
+/**
+ * The default `threads` ceiling. Each worker holds a heap of its own, with the
+ * libraries and the pages it is working on, so past a few threads memory, not
+ * CPU, is what runs out. `--threads` raises it.
+ */
+export const DEFAULT_MAX_THREADS = 4;
+
 /**
  * The heap size limit of each worker thread (V8's old space), in MB. V8 lets a
  * heap grow to several times its live data before collecting it, the more the
@@ -8,3 +15,10 @@
  * `--max-old-space-size` replaces it.
  */
 export const WORKER_HEAP_SIZE = 512;
+
+/**
+ * The young generation size limit of each worker thread, in MB. V8 sizes it
+ * from the machine's memory, as if each worker were the only one, so it made up
+ * over half of a worker's heap. Node.js's `--max-semi-space-size` replaces it.
+ */
+export const WORKER_GEN_SIZE = 96;

package/src/threading/index.mjs

@@ -2,7 +2,7 @@ import Piscina from 'piscina';
 
 import logger from '#logger/index.mjs';
 
-import { WORKER_HEAP_SIZE } from './constants.mjs';
+import { WORKER_HEAP_SIZE, WORKER_GEN_SIZE } from './constants.mjs';
 
 const poolLogger = logger.child('WorkerPool');
 
@@ -27,7 +27,11 @@ export default function createWorkerPool(threads) {
     // A worker idle for 250ms ends, so its heap is gone during a long stretch
     // of work on the main thread, such as bundling the site
     idleTimeout: 250,
-    resourceLimits: { maxOldGenerationSizeMb: WORKER_HEAP_SIZE },
+    resourceLimits: {
+      maxOldGenerationSizeMb: WORKER_HEAP_SIZE,
+      maxYoungGenerationSizeMb: WORKER_GEN_SIZE,
+    },
+    recordTiming: false,
     workerData: { logLevel: logger.getLogLevel() },
   });
 }

package/src/threading/parallel.mjs

@@ -77,12 +77,12 @@ export default function createParallelWorker(
 
   return {
     /**
-     * Processes items in parallel, yielding results as chunks complete.
+     * Processes items in parallel, yielding results in submission order.
      *
      * @template T
      * @param {T[]} items - Items to process
      * @param {object} extra - Extra options
-     * @yields {R[]} Chunk results as they complete
+     * @yields {R[]} Chunk results in input order
      */
     async *stream(items, extra) {
       if (items.length === 0) {
@@ -96,44 +96,41 @@ export default function createParallelWorker(
         { generator: name, chunks: chunks.length, chunkSize, threads }
       );
 
-      const runInOneGo = threads <= 1 || items.length <= 2;
+      // On this thread, one chunk at a time, so their work isn't all held at once
+      if (threads <= 1 || items.length <= 2) {
+        for (const indices of chunks) {
+          yield await generator.processChunk(items, indices, extra);
+        }
 
-      // Submit all tasks to Piscina - each promise resolves to itself for removal
-      const pending = new Set(
-        chunks.map(indices => {
-          if (runInOneGo) {
-            const promise = generator
-              .processChunk(items, indices, extra)
-              .then(result => ({ promise, result }));
-
-            return promise;
-          }
-
-          const promise = pool
-            .run(
-              createTask(items, indices, extra, configuration, specifier, name)
-            )
-            .then(result => ({ promise, result }));
-
-          return promise;
-        })
-      );
+        return;
+      }
 
-      // Yield results as they complete (true parallel collection)
-      let completed = 0;
+      // Start every chunk immediately, but emit results in submission order.
+      // Handle errors as each task settles so a later failure cannot become an
+      // unhandled rejection while earlier chunks are still running.
+      const tasks = chunks.map(indices => {
+        const task = pool.run(
+          createTask(items, indices, extra, configuration, specifier, name)
+        );
 
-      while (pending.size > 0) {
-        const { promise, result } = await Promise.race(pending);
+        return task.then(
+          result => ({ result }),
+          error => ({ error })
+        );
+      });
 
-        pending.delete(promise);
+      for (const [index, task] of tasks.entries()) {
+        const outcome = await task;
 
-        completed++;
+        if ('error' in outcome) {
+          throw outcome.error;
+        }
 
-        parallelLogger.debug(`Chunk ${completed}/${chunks.length} completed`, {
+        parallelLogger.debug(`Chunk ${index + 1}/${chunks.length} completed`, {
           generator: name,
         });
 
-        yield result;
+        yield outcome.result;
       }
     },
   };

package/src/utils/configuration/constants.mjs

@@ -1,13 +1,11 @@
 'use strict';
 
 /**
- * The default `threads` ceiling. Each worker holds a heap of its own, with the
- * libraries and the pages it is working on, so past a few threads memory, not
- * CPU, is what runs out. `--threads` raises it.
+ * The default number of items each worker task processes.
  */
-export const DEFAULT_MAX_THREADS = 4;
+export const DEFAULT_CHUNK_SIZE = 10;
 
 /**
- * The default number of items each worker task processes.
+ * The default directory generators write to when no `output` is configured.
  */
-export const DEFAULT_CHUNK_SIZE = 10;
+export const DEFAULT_OUTPUT_DIRECTORY = './out';

package/src/utils/configuration/index.mjs

@@ -22,7 +22,8 @@ import {
 import { resolveMarkdown } from '#utils/markdown/plugins.mjs';
 import { deepMerge } from '#utils/misc.mjs';
 
-import { DEFAULT_CHUNK_SIZE, DEFAULT_MAX_THREADS } from './constants.mjs';
+import { DEFAULT_MAX_THREADS } from '../../threading/constants.mjs';
+import { DEFAULT_CHUNK_SIZE, DEFAULT_OUTPUT_DIRECTORY } from './constants.mjs';
 
 const configExplorer = cosmiconfig('doc-kit');
 
@@ -63,6 +64,7 @@ export const getDefaultConfig = (generators, config) =>
         project: detectProject() ?? 'API Docs',
         version: process.version,
         minify: false,
+        output: DEFAULT_OUTPUT_DIRECTORY,
         ref: 'HEAD',
         // Without release history there is nothing to build a version picker
         // from, so generators render single-version output.

package/src/utils/glob.mjs

@@ -0,0 +1,42 @@
+import { globSync } from 'node:fs';
+import { relative, resolve, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+// A path segment holding any glob syntax (wildcards, classes, braces or extglobs)
+const GLOB_SEGMENT = /[*?[{]|[!+@]\(/;
+
+/**
+ * Finds the files matching glob patterns, as POSIX paths relative to `cwd`
+ * (absolute patterns included). Directories are left out.
+ *
+ * @param {string | string[]} patterns - Glob patterns to match
+ * @param {{ cwd?: string | URL, ignore?: string[] }} [options]
+ * @returns {string[]}
+ */
+export const findFiles = (patterns, { cwd = process.cwd(), ignore } = {}) => {
+  const root = cwd instanceof URL ? fileURLToPath(cwd) : cwd;
+
+  return globSync(patterns, { cwd: root, exclude: ignore, withFileTypes: true })
+    .filter(dirent => !dirent.isDirectory())
+    .map(dirent =>
+      relative(root, resolve(root, dirent.parentPath, dirent.name))
+        .split(sep)
+        .join('/')
+    );
+};
+
+/**
+ * Returns the directory part of a glob pattern before its first glob segment,
+ * e.g. `doc/api` for `doc/api/*.md`, or the parent directory of a plain path.
+ * Like `fs.globSync`, it reads backslashes as separators, never as escapes.
+ *
+ * @param {string} pattern - A glob pattern
+ * @returns {string}
+ */
+export const globParent = pattern => {
+  const segments = pattern.replaceAll('\\', '/').split('/');
+  const index = segments.findIndex(segment => GLOB_SEGMENT.test(segment));
+  const parent = segments.slice(0, index === -1 ? -1 : index).join('/');
+
+  return parent || (segments[0] === '' ? '/' : '.');
+};