@doc-kit/core 1.2.1 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/README.md +1 -1
  3. package/package.json +8 -11
  4. package/src/generators/ast/generate.mjs +11 -9
  5. package/src/generators/ast/index.mjs +9 -0
  6. package/src/generators/ast-js/generate.mjs +4 -4
  7. package/src/generators/index.mjs +1 -0
  8. package/src/generators/json/README.md +1 -1
  9. package/src/generators/json/generated/schema.d.ts +1 -1
  10. package/src/generators/json/index.mjs +9 -0
  11. package/src/generators/json/utils/markdown.mjs +2 -2
  12. package/src/generators/json-all/README.md +1 -1
  13. package/src/generators/json-simple/README.md +1 -1
  14. package/src/generators/json-simple/generate.mjs +13 -5
  15. package/src/generators/loader.mjs +14 -0
  16. package/src/generators/metadata/generate.mjs +4 -2
  17. package/src/generators/metadata/index.mjs +9 -0
  18. package/src/generators/metadata/maps/mdn.json +2 -0
  19. package/src/generators/metadata/types.d.ts +1 -1
  20. package/src/generators/metadata/utils/parse.mjs +2 -4
  21. package/src/generators/metadata/utils/visitors.mjs +2 -2
  22. package/src/generators/types.d.ts +8 -0
  23. package/src/generators.mjs +9 -3
  24. package/src/logger/constants.mjs +35 -0
  25. package/src/logger/transports/github.mjs +48 -15
  26. package/src/plugins/shiki/highlighter.mjs +373 -0
  27. package/src/plugins/shiki/rehype.mjs +38 -0
  28. package/src/{utils/type-annotations/highlighted.mjs → plugins/type-annotations/highlighter.mjs} +14 -19
  29. package/src/{utils → plugins}/type-annotations/remark.mjs +6 -2
  30. package/src/threading/chunk-worker.mjs +4 -0
  31. package/src/threading/constants.mjs +24 -0
  32. package/src/threading/index.mjs +10 -1
  33. package/src/threading/parallel.mjs +28 -31
  34. package/src/utils/configuration/constants.mjs +4 -6
  35. package/src/utils/configuration/index.mjs +89 -21
  36. package/src/utils/configuration/types.d.ts +27 -0
  37. package/src/utils/glob.mjs +42 -0
  38. package/src/utils/loaders.mjs +23 -1
  39. package/src/utils/markdown/constants.mjs +7 -0
  40. package/src/utils/markdown/plugins.mjs +232 -0
  41. package/src/utils/markdown/processor.mjs +60 -0
  42. package/src/utils/highlighter.mjs +0 -198
  43. package/src/utils/remark-shiki.mjs +0 -29
  44. package/src/utils/remark.mjs +0 -90
  45. /package/src/{utils → plugins}/type-annotations/hast.mjs +0 -0
  46. /package/src/{utils → plugins}/type-annotations/mdast.mjs +0 -0
  47. /package/src/{utils → plugins}/type-annotations/syntax.mjs +0 -0
package/CHANGELOG.md CHANGED
@@ -1,5 +1,31 @@
1
1
  # @doc-kit/core
2
2
 
3
+ ## 1.3.1
4
+
5
+ ### Patch Changes
6
+
7
+ - [#1187](https://github.com/nodejs/doc-kit/pull/1187) [`e49eb52`](https://github.com/nodejs/doc-kit/commit/e49eb52472dfc2707a5966ee25c4dc7201360850) Thanks [@dxnnystyles](https://github.com/dxnnystyles)! - Keep json-simple output generation from mutating shared metadata entries.
8
+
9
+ - [#1179](https://github.com/nodejs/doc-kit/pull/1179) [`b8d7b77`](https://github.com/nodejs/doc-kit/commit/b8d7b7725f0dbe085d9f32af60fddd7459555af9) Thanks [@ovflowd](https://github.com/ovflowd)! - Builds use less memory, most of all with few threads: each worker's young generation is limited to 96MB, a single thread processes one chunk at a time and lets go of the pages' code before rendering them, and Vite's native bundler gets as many threads as doc-kit and returns the memory it frees. On the Node.js API docs, `--threads 1` now builds with Node.js's `--max-old-space-size=512`, and no longer crashes when `legacy-html` is built alongside other targets.
10
+
11
+ - [#1180](https://github.com/nodejs/doc-kit/pull/1180) [`92efbbe`](https://github.com/nodejs/doc-kit/commit/92efbbeb727a38b178c9ccb63a3a4f9cfbc110a4) Thanks [@ovflowd](https://github.com/ovflowd)! - Find input files with Node.js's `fs.globSync` instead of `tinyglobby` and `glob-parent`.
12
+
13
+ - [#1082](https://github.com/nodejs/doc-kit/pull/1082) [`9e21017`](https://github.com/nodejs/doc-kit/commit/9e21017d9597e3f8b5a18a41f3db8e5de216213a) Thanks [@btea](https://github.com/btea)! - Add a `doc-kit preview` command that serves the generated docs locally, building first when any `generate` option is passed. The default output directory (`./out`) is now defined in the shared configuration, so `generate` also writes there when no `--output` is given.
14
+
15
+ - [#1193](https://github.com/nodejs/doc-kit/pull/1193) [`854a944`](https://github.com/nodejs/doc-kit/commit/854a944699b3a996cb3a8544deaec3fe916fd449) Thanks [@OKAN-SOFTWARE](https://github.com/OKAN-SOFTWARE)! - Keep worker results and discovered source files in deterministic order so repeated documentation builds produce stable output.
16
+
17
+ - [#1178](https://github.com/nodejs/doc-kit/pull/1178) [`0321911`](https://github.com/nodejs/doc-kit/commit/03219116c78e7f4a721b4e39bc71ba54cb6f0ba8) Thanks [@ovflowd](https://github.com/ovflowd)! - Install fewer packages: `@doc-kit/core` updates `cosmiconfig` to v10 (TypeScript config files now load through Node's type stripping) and no longer installs `dedent`, and `@doc-kit/generator-react` estimates reading time with `Intl.Segmenter` instead of the `reading-time` package and updates `@node-core/ui-components` to 1.8.1, which no longer installs `@vcarl/remark-headings`.
18
+
19
+ ## 1.3.0
20
+
21
+ ### Minor Changes
22
+
23
+ - [#1156](https://github.com/nodejs/doc-kit/pull/1156) [`6d5f9d1`](https://github.com/nodejs/doc-kit/commit/6d5f9d17d9f265c62db0f878c4e03903881b11c9) Thanks [@ovflowd](https://github.com/ovflowd)! - Builds take about half the memory and a third of the time (measured on the Node.js API docs): Shiki registers each language once code in it is highlighted, highlighted code reaches pages as static markup rather than a tree per token, `all.html` is rendered by the worker pool, output is no longer minified by default (serve it compressed instead), idle workers end after 250ms, and each worker's heap is limited to 512MB, which Node.js's `--max-old-space-size` raises.
24
+
25
+ - [#1127](https://github.com/nodejs/doc-kit/pull/1127) [`c50e75f`](https://github.com/nodejs/doc-kit/commit/c50e75f8f50c84b6ff3b16b84059bd0f53ca5d1c) Thanks [@ovflowd](https://github.com/ovflowd)! - Add the `llms-txt-full` generator, writing a `llms-full.txt` file holding the Markdown of every page, each preceded by its URL.
26
+
27
+ - [#1157](https://github.com/nodejs/doc-kit/pull/1157) [`a4c9684`](https://github.com/nodejs/doc-kit/commit/a4c9684fd46f14b3bcbaf2e8f01a246a47ee27cd) Thanks [@ovflowd](https://github.com/ovflowd)! - Add a `markdown` option to add remark, rehype, and recma plugins to the generators processing Markdown, or configure the ones they use, such as Shiki. The `@doc-kit/core` modules the generators' pipelines replace are removed (`utils/remark.mjs`, `utils/remark-shiki.mjs`, and `utils/highlighter.mjs`), and `utils/type-annotations` moves to `plugins/type-annotations`
28
+
3
29
  ## 1.2.1
4
30
 
5
31
  ### Patch Changes
package/README.md CHANGED
@@ -15,7 +15,7 @@ debugging-only `json-simple`); the rest come from companion packages:
15
15
 
16
16
  | Package | Generators |
17
17
  | -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- |
18
- | [`@doc-kit/generator-react`](https://www.npmjs.com/package/@doc-kit/generator-react) | `html` (the modern site), `orama-db`, `llms-txt`, `sitemap` |
18
+ | [`@doc-kit/generator-react`](https://www.npmjs.com/package/@doc-kit/generator-react) | `html` (the modern site), `orama-db`, `llms-txt`, `llms-txt-full`, `sitemap` |
19
19
  | [`@node-core/doc-kit-legacy`](https://www.npmjs.com/package/@node-core/doc-kit-legacy) | `legacy-html`, `legacy-html-all`, `legacy-json`, `legacy-json-all` (Node.js-specific) |
20
20
  | [`@node-core/doc-kit`](https://www.npmjs.com/package/@node-core/doc-kit) | `man-page`, `api-links`, `addon-verify` (Node.js-specific) |
21
21
 
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@doc-kit/core",
3
3
  "type": "module",
4
- "version": "1.2.1",
4
+ "version": "1.3.1",
5
5
  "description": "The doc-kit engine: the pipeline and shared APIs that power the doc-kit generators and CLI",
6
6
  "repository": {
7
7
  "type": "git",
@@ -33,6 +33,7 @@
33
33
  "./src/parsers/*",
34
34
  "./src/parsers/*.d.ts"
35
35
  ],
36
+ "#plugins/*": "./src/plugins/*",
36
37
  "#utils/*": "./src/utils/*"
37
38
  },
38
39
  "files": [
@@ -45,19 +46,15 @@
45
46
  "README.md"
46
47
  ],
47
48
  "dependencies": {
48
- "@actions/core": "^3.0.0",
49
- "@node-core/rehype-shiki": "^1.5.0",
50
- "@swc/html-wasm": "^1.16.1",
49
+ "@node-core/rehype-shiki": "^1.5.1",
50
+ "@swc/html-wasm": "^1.16.2",
51
51
  "@swc/wasm": "^1.15.46",
52
52
  "acorn": "^8.17.0",
53
- "cosmiconfig": "^9.0.2",
54
- "dedent": "^1.7.2",
53
+ "cosmiconfig": "^10.0.1",
55
54
  "github-slugger": "^2.0.0",
56
- "glob-parent": "^6.0.2",
57
- "hastscript": "^9.0.1",
58
55
  "mdast-util-slice-markdown": "^2.0.1",
59
56
  "mdast-util-to-string": "^4.0.0",
60
- "piscina": "^5.3.1",
57
+ "piscina": "^5.3.2",
61
58
  "rehype-stringify": "^10.0.1",
62
59
  "remark-gfm": "^4.0.1",
63
60
  "remark-mdx": "^3.1.1",
@@ -66,7 +63,6 @@
66
63
  "remark-stringify": "^11.0.0",
67
64
  "semver": "^7.8.5",
68
65
  "shiki": "^4.4.3",
69
- "tinyglobby": "^0.2.17",
70
66
  "unified": "^11.0.5",
71
67
  "unist-builder": "^4.0.0",
72
68
  "unist-util-find-after": "^5.0.0",
@@ -74,10 +70,11 @@
74
70
  "unist-util-remove": "^4.0.0",
75
71
  "unist-util-select": "^5.1.0",
76
72
  "unist-util-visit": "^5.1.0",
77
- "yaml": "^2.9.0"
73
+ "yaml": "^2.9.1"
78
74
  },
79
75
  "devDependencies": {
80
76
  "ajv": "^8.20.0",
77
+ "dedent": "^1.7.2",
81
78
  "hast-util-to-html": "^9.0.5",
82
79
  "hast-util-to-string": "^3.0.1"
83
80
  },
@@ -3,14 +3,13 @@
3
3
  import { readFile } from 'node:fs/promises';
4
4
  import { relative, sep } from 'node:path/posix';
5
5
 
6
- import globParent from 'glob-parent';
7
- import { globSync } from 'tinyglobby';
8
6
  import { parse as parseYaml } from 'yaml';
9
7
 
10
8
  import getConfig from '#utils/configuration/index.mjs';
11
9
  import { withExt } from '#utils/file.mjs';
10
+ import { findFiles, globParent } from '#utils/glob.mjs';
11
+ import { getProcessor } from '#utils/markdown/processor.mjs';
12
12
  import { QUERIES } from '#utils/queries/index.mjs';
13
- import { getRemark as remark, getRemarkMdx } from '#utils/remark.mjs';
14
13
 
15
14
  import { STABILITY_INDEX_URL } from './constants.mjs';
16
15
 
@@ -67,6 +66,8 @@ export async function processChunk(inputSlice, itemIndices) {
67
66
  // The path is the relative path minus the extension
68
67
  const relativePath = sep + withExt(relative(parent, path));
69
68
 
69
+ const processor = getProcessor('ast', { mdx });
70
+
70
71
  let tree;
71
72
 
72
73
  if (mdx) {
@@ -79,7 +80,7 @@ export async function processChunk(inputSlice, itemIndices) {
79
80
  ''
80
81
  );
81
82
 
82
- tree = getRemarkMdx().parse(source);
83
+ tree = processor.parse(source);
83
84
 
84
85
  if (frontmatter) {
85
86
  tree.children.unshift({
@@ -93,9 +94,11 @@ export async function processChunk(inputSlice, itemIndices) {
93
94
  (_, yaml) => `<!-- YAML\n${yaml}\n-->`
94
95
  );
95
96
 
96
- tree = remark().parse(value);
97
+ tree = processor.parse(value);
97
98
  }
98
99
 
100
+ tree = await processor.run(tree, { path });
101
+
99
102
  results.push({ tree, path: relativePath, mdx });
100
103
  }
101
104
 
@@ -113,10 +116,9 @@ export async function* generate(_, worker) {
113
116
  const files = config.input.flatMap(input => {
114
117
  const parent = globParent(input);
115
118
 
116
- return globSync(input, { ignore: config.ignore }).map(child => [
117
- child,
118
- parent,
119
- ]);
119
+ return findFiles(input, { ignore: config.ignore })
120
+ .sort()
121
+ .map(child => [child, parent]);
120
122
  });
121
123
 
122
124
  // Parse markdown files in parallel using worker threads
@@ -15,6 +15,15 @@ export default {
15
15
 
16
16
  hasParallelProcessor: true,
17
17
 
18
+ markdown: {
19
+ remarkPlugins: [
20
+ 'remark-parse',
21
+ '#plugins/type-annotations/remark.mjs',
22
+ 'remark-gfm',
23
+ '...',
24
+ ],
25
+ },
26
+
18
27
  generate,
19
28
  processChunk,
20
29
  };
@@ -4,9 +4,9 @@ import { readFile } from 'node:fs/promises';
4
4
  import { extname } from 'node:path';
5
5
 
6
6
  import { parse } from 'acorn';
7
- import { globSync } from 'tinyglobby';
8
7
 
9
8
  import getConfig from '#utils/configuration/index.mjs';
9
+ import { findFiles } from '#utils/glob.mjs';
10
10
 
11
11
  /**
12
12
  * Process a chunk of JavaScript files in a worker thread.
@@ -44,9 +44,9 @@ export async function processChunk(inputSlice, itemIndices) {
44
44
  export async function* generate(_, worker) {
45
45
  const config = getConfig('ast-js');
46
46
 
47
- const files = globSync(config.input, { ignore: config.ignore }).filter(
48
- p => extname(p) === '.js'
49
- );
47
+ const files = findFiles(config.input, { ignore: config.ignore })
48
+ .filter(p => extname(p) === '.js')
49
+ .sort();
50
50
 
51
51
  // Parse the Javascript sources into ASTs in parallel using worker threads
52
52
  // source is both the items list and the fullInput since we use sliceInput
@@ -22,6 +22,7 @@ export const publicGenerators = {
22
22
  'api-links': '@node-core/doc-kit/api-links',
23
23
  'orama-db': '@doc-kit/generator-react/orama-db',
24
24
  'llms-txt': '@doc-kit/generator-react/llms-txt',
25
+ 'llms-txt-full': '@doc-kit/generator-react/llms-txt-full',
25
26
  sitemap: '@doc-kit/generator-react/sitemap',
26
27
  html: '@doc-kit/generator-react/html',
27
28
  'section-pages': '@doc-kit/generator-react/section-pages',
@@ -18,7 +18,7 @@ Files keep the input's directory layout: `doc/api/some/fs.md` becomes `out/some/
18
18
 
19
19
  - `output` {string} The directory to write the documents to.
20
20
  - `minify` {boolean} Whether to minify the output. Inherited from `global`.
21
- **Default:** `true`.
21
+ **Default:** `false`.
22
22
  - `repository` {string} The `owner/name` repository source links resolve
23
23
  against. Inherited from `global`; without one, `sourceLink.url` is `null`.
24
24
  - `schemaURL` {string} Where the schema is published. `{schemaVersion}` is
@@ -1,4 +1,4 @@
1
- /* eslint-disable */
1
+ /* oxlint-disable */
2
2
  /**
3
3
  * Generated from `schema.json` by `scripts/generate-json-types.mjs`.
4
4
  * Do not edit: change the schema and regenerate instead.
@@ -22,6 +22,15 @@ export default {
22
22
 
23
23
  hasParallelProcessor: true,
24
24
 
25
+ markdown: {
26
+ remarkPlugins: [
27
+ '#plugins/type-annotations/remark.mjs',
28
+ 'remark-gfm',
29
+ 'remark-stringify',
30
+ '...',
31
+ ],
32
+ },
33
+
25
34
  generate,
26
35
  processChunk,
27
36
  };
@@ -3,7 +3,7 @@
3
3
  import { u as createTree } from 'unist-builder';
4
4
  import { visit } from 'unist-util-visit';
5
5
 
6
- import { getRemark, getRemarkMdx } from '#utils/remark.mjs';
6
+ import { getProcessor } from '#utils/markdown/processor.mjs';
7
7
 
8
8
  import { DISPLAY_NAME } from '../constants.mjs';
9
9
 
@@ -19,7 +19,7 @@ export const blocksToMarkdown = (nodes, mdx = false) => {
19
19
  return '';
20
20
  }
21
21
 
22
- const processor = mdx ? getRemarkMdx() : getRemark();
22
+ const processor = getProcessor('json', { mdx });
23
23
 
24
24
  return processor.stringify(createTree('root', nodes)).trim();
25
25
  };
@@ -23,7 +23,7 @@ generator's schema for the documents.
23
23
 
24
24
  - `output` {string} The directory where `all.json` will be written.
25
25
  - `minify` {boolean} Whether to minify the output. Inherited from `global`.
26
- **Default:** `true`.
26
+ **Default:** `false`.
27
27
  - `index` {Array} The `{ api }` objects defining the document order. Inherited
28
28
  from `global`.
29
29
  - `schemaURL` {string} Where the bundle's schema is published.
@@ -6,4 +6,4 @@ The `json-simple` generator creates a simplified JSON version of the API documen
6
6
 
7
7
  - `output` {string} The directory where `api-docs.json` will be written.
8
8
  - `minify` {boolean} Whether to minify the output JSON. Inherited from
9
- `global`. **Default:** `true`.
9
+ `global`. **Default:** `false`.
@@ -16,13 +16,21 @@ import { UNIST } from '#utils/queries/index.mjs';
16
16
  export async function generate(input) {
17
17
  const config = getConfig('json-simple');
18
18
 
19
- input.forEach(node =>
20
- remove(node.content, [UNIST.isStabilityNode, UNIST.isHeading])
21
- );
19
+ const output = input.map(entry => {
20
+ const copy = { ...entry, content: structuredClone(entry.content) };
21
+
22
+ remove(copy.content, [UNIST.isStabilityNode, UNIST.isHeading]);
23
+
24
+ return copy;
25
+ });
22
26
 
23
27
  if (config.output) {
24
- await writeJSON(join(config.output, 'api-docs.json'), input, config.minify);
28
+ await writeJSON(
29
+ join(config.output, 'api-docs.json'),
30
+ output,
31
+ config.minify
32
+ );
25
33
  }
26
34
 
27
- return input;
35
+ return output;
28
36
  }
@@ -7,6 +7,9 @@ import { enforceArray } from '#utils/array.mjs';
7
7
 
8
8
  import { allGenerators } from './index.mjs';
9
9
 
10
+ // The module each loaded generator comes from
11
+ const generatorModules = new WeakMap();
12
+
10
13
  /**
11
14
  * Resolves a CLI/configuration target into an import specifier. Shorthand
12
15
  * names map through the alias table; filesystem paths become file URLs;
@@ -64,9 +67,20 @@ export const loadGenerator = async specifier => {
64
67
  );
65
68
  }
66
69
 
70
+ generatorModules.set(generator, import.meta.resolve(resolved));
71
+
67
72
  return generator;
68
73
  };
69
74
 
75
+ /**
76
+ * The URL of the module a generator was loaded from, which the specifiers of
77
+ * its Markdown pipeline resolve against.
78
+ *
79
+ * @param {GeneratorMetadata} generator - A generator loaded by `loadGenerator`
80
+ * @returns {string | undefined}
81
+ */
82
+ export const getGeneratorModule = generator => generatorModules.get(generator);
83
+
70
84
  /**
71
85
  * Loads the given generators plus the transitive closure of their
72
86
  * dependencies (via `dependsOn`) and dependents (via `dependent`).
@@ -3,8 +3,6 @@
3
3
  import getConfig from '#utils/configuration/index.mjs';
4
4
  import { loadFromURL } from '#utils/loaders.mjs';
5
5
 
6
- import { parseApiDoc } from './utils/parse.mjs';
7
-
8
6
  /**
9
7
  * Process a chunk of API doc files in a worker thread.
10
8
  * Called by chunk-worker.mjs for parallel processing.
@@ -12,6 +10,10 @@ import { parseApiDoc } from './utils/parse.mjs';
12
10
  * @type {import('./types').Generator['processChunk']}
13
11
  */
14
12
  export async function processChunk(fullInput, itemIndices, typeMap) {
13
+ // Loaded on first use rather than with the generator, so the main thread,
14
+ // which never parses, does not load the TypeScript parser (~40MB of WASM)
15
+ const { parseApiDoc } = await import('./utils/parse.mjs');
16
+
15
17
  const results = [];
16
18
 
17
19
  for (const idx of itemIndices) {
@@ -16,6 +16,15 @@ export default {
16
16
 
17
17
  hasParallelProcessor: true,
18
18
 
19
+ markdown: {
20
+ remarkPlugins: [
21
+ 'remark-parse',
22
+ '#plugins/type-annotations/remark.mjs',
23
+ 'remark-gfm',
24
+ '...',
25
+ ],
26
+ },
27
+
19
28
  generate,
20
29
  processChunk,
21
30
  };
@@ -554,6 +554,7 @@
554
554
  "oes_vertex_array_object": "https://developer.mozilla.org/docs/Web/API/OES_vertex_array_object",
555
555
  "otpcredential": "https://developer.mozilla.org/docs/Web/API/OTPCredential",
556
556
  "ovr_multiview2": "https://developer.mozilla.org/docs/Web/API/OVR_multiview2",
557
+ "observable": "https://developer.mozilla.org/docs/Web/API/Observable",
557
558
  "offlineaudiocompletionevent": "https://developer.mozilla.org/docs/Web/API/OfflineAudioCompletionEvent",
558
559
  "offlineaudiocontext": "https://developer.mozilla.org/docs/Web/API/OfflineAudioContext",
559
560
  "offscreencanvas": "https://developer.mozilla.org/docs/Web/API/OffscreenCanvas",
@@ -827,6 +828,7 @@
827
828
  "stylesheet": "https://developer.mozilla.org/docs/Web/API/StyleSheet",
828
829
  "stylesheetlist": "https://developer.mozilla.org/docs/Web/API/StyleSheetList",
829
830
  "submitevent": "https://developer.mozilla.org/docs/Web/API/SubmitEvent",
831
+ "subscriber": "https://developer.mozilla.org/docs/Web/API/Subscriber",
830
832
  "subtlecrypto": "https://developer.mozilla.org/docs/Web/API/SubtleCrypto",
831
833
  "summarizer": "https://developer.mozilla.org/docs/Web/API/Summarizer",
832
834
  "syncevent": "https://developer.mozilla.org/docs/Web/API/SyncEvent",
@@ -1,5 +1,5 @@
1
- import type { Position } from 'unist';
2
1
  import type { Root, Node, Data, Blockquote, Heading } from 'mdast';
2
+ import type { Position } from 'unist';
3
3
 
4
4
  export type Generator = GeneratorMetadata<
5
5
  {},
@@ -14,7 +14,6 @@ import {
14
14
  IGNORE_STABILITY_STEMS,
15
15
  } from '#generators/metadata/constants.mjs';
16
16
  import { UNIST } from '#utils/queries/index.mjs';
17
- import { getRemark as remark } from '#utils/remark.mjs';
18
17
  import { relative } from '#utils/url.mjs';
19
18
 
20
19
  import { resolveTypeAnnotations } from './resolveTypes.mjs';
@@ -147,9 +146,8 @@ export const parseApiDoc = ({ path, tree, mdx = false }, typeMap) => {
147
146
  // Remove processed YAML nodes from the content
148
147
  remove(subTree, [UNIST.isYamlNode]);
149
148
 
150
- // Apply AST transformations
151
- const parsedSubTree = remark().runSync(subTree);
152
- metadata.content = parsedSubTree;
149
+ // The remark plugins already ran on the whole document, in `ast`
150
+ metadata.content = subTree;
153
151
 
154
152
  // Add to collection
155
153
  metadataCollection.push(metadata);
@@ -2,8 +2,8 @@
2
2
 
3
3
  import { SKIP } from 'unist-util-visit';
4
4
 
5
+ import { getProcessor } from '#utils/markdown/processor.mjs';
5
6
  import { QUERIES } from '#utils/queries/index.mjs';
6
- import { getRemark as remark } from '#utils/remark.mjs';
7
7
  import { transformNodesToString } from '#utils/unist.mjs';
8
8
 
9
9
  import { transformUnixManualToLink } from './transformers.mjs';
@@ -41,7 +41,7 @@ const updateReferences = (query, transformer, node, parent) => {
41
41
  // and adding those nodes to the parent.
42
42
  const {
43
43
  children: [newNode],
44
- } = remark().parse(replacedTypes);
44
+ } = getProcessor('metadata').parse(replacedTypes);
45
45
 
46
46
  // Find the index of the original node in the parent
47
47
  const index = parent.children.indexOf(node);
@@ -70,6 +70,14 @@ declare global {
70
70
 
71
71
  hasParallelProcessor?: boolean;
72
72
 
73
+ /**
74
+ * The unified plugins this generator processes Markdown with, listed as
75
+ * in the `markdown` option, with paths relative to its module. Each list
76
+ * takes the configured plugins in place of its `'...'`.
77
+ * `getProcessor(name)` gives its processor.
78
+ */
79
+ markdown?: import('../utils/configuration/types').MarkdownPipeline;
80
+
73
81
  /**
74
82
  * The immediate generator that this generator depends on.
75
83
  * For example, the `html` generator depends on the `react` generator.
@@ -9,6 +9,7 @@ import { resolvePipeline } from './generators/pipeline.mjs';
9
9
  import logger from './logger/index.mjs';
10
10
  import createWorkerPool from './threading/index.mjs';
11
11
  import createParallelWorker from './threading/parallel.mjs';
12
+ import { loadMarkdownPlugins } from './utils/markdown/plugins.mjs';
12
13
  import { isAsyncIterable } from './utils/misc.mjs';
13
14
 
14
15
  const generatorsLogger = logger.child('generators');
@@ -78,13 +79,18 @@ const createGenerator = () => {
78
79
 
79
80
  generatorsLogger.debug(`Starting "${name}"`);
80
81
 
82
+ // Load its Markdown pipeline, for what it processes on this thread
83
+ await loadMarkdownPlugins(generator, configuration[name]?.markdown);
84
+
81
85
  // Create parallel worker for streaming generators
82
86
  const worker = hasParallelProcessor
83
87
  ? createParallelWorker(specifier, generator, pool, configuration)
84
88
  : Promise.resolve(null);
85
89
 
86
- const result = await generate(dependencyInput, await worker);
87
-
90
+ // Returned rather than awaited, so this function doesn't hold on to the
91
+ // input while the generator runs, and the generator can let it go
92
+ return generate(dependencyInput, await worker);
93
+ })().then(result => {
88
94
  // For streaming generators, "Completed" is logged when the cache
89
95
  // finishes collecting, not here when the generator returns
90
96
  if (!isAsyncIterable(result)) {
@@ -92,7 +98,7 @@ const createGenerator = () => {
92
98
  }
93
99
 
94
100
  return result;
95
- })()
101
+ })
96
102
  );
97
103
  };
98
104
 
@@ -32,3 +32,38 @@ export const levelToColorMap = {
32
32
  [LogLevel.error]: 'magenta',
33
33
  [LogLevel.fatal]: 'red',
34
34
  };
35
+
36
+ /**
37
+ * Maps log level numbers to the GitHub Actions workflow commands reporting
38
+ * them.
39
+ */
40
+ export const levelToGitHubCommand = {
41
+ [LogLevel.debug]: 'debug',
42
+ [LogLevel.info]: 'notice',
43
+ [LogLevel.warn]: 'warning',
44
+ [LogLevel.error]: 'error',
45
+ [LogLevel.fatal]: 'error',
46
+ };
47
+
48
+ /**
49
+ * Maps the characters a workflow command's message can't hold to their
50
+ * escapes. A command is a line, `::command key=value,key=value::message`,
51
+ * so its message, which runs to the end of the line, can't break it, nor hold
52
+ * the `%` an escape starts with.
53
+ */
54
+ export const commandDataEscapes = {
55
+ '%': '%25',
56
+ '\r': '%0D',
57
+ '\n': '%0A',
58
+ };
59
+
60
+ /**
61
+ * Maps the characters a workflow command's property can't hold to their
62
+ * escapes: those its message can't, and the `,` separating properties and
63
+ * the `:` of the `::` ending them.
64
+ */
65
+ export const commandPropertyEscapes = {
66
+ ...commandDataEscapes,
67
+ ':': '%3A',
68
+ ',': '%2C',
69
+ };
@@ -1,23 +1,35 @@
1
1
  'use strict';
2
2
 
3
- import { debug, notice, warning, error } from '@actions/core';
3
+ import { EOL } from 'node:os';
4
4
 
5
- import { LogLevel } from '#logger/constants.mjs';
5
+ import {
6
+ commandDataEscapes,
7
+ commandPropertyEscapes,
8
+ levelToGitHubCommand,
9
+ } from '#logger/constants.mjs';
6
10
  import { prettifyLevel } from '#logger/utils/colors.mjs';
7
11
  import { prettifyTimestamp } from '#logger/utils/time.mjs';
8
12
 
9
- const actions = {
10
- [LogLevel.debug]: debug,
11
- [LogLevel.info]: notice,
12
- [LogLevel.warn]: warning,
13
- [LogLevel.error]: error,
14
- [LogLevel.fatal]: error,
15
- };
13
+ /**
14
+ * Escapes a value of a workflow command: its message, with
15
+ * `commandDataEscapes`, or one of its properties, with
16
+ * `commandPropertyEscapes`.
17
+ *
18
+ * @param {string | number} value - The value
19
+ * @param {Record<string, string>} escapes - The escapes of the characters it can't hold
20
+ * @returns {string}
21
+ */
22
+ const escapeCommandValue = (value, escapes) =>
23
+ String(value).replace(
24
+ /[%\r\n:,]/g,
25
+ character => escapes[character] ?? character
26
+ );
16
27
 
17
28
  /**
18
- * Logs messages to GitHub Actions with formatted output and file info with
19
- * appropriate gh actions based on level.
29
+ * Logs messages to GitHub Actions as workflow commands, with the file and
30
+ * lines they're about, for them to annotate.
20
31
  *
32
+ * @see https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands
21
33
  * @param {import('../types').TransportContext} context
22
34
  * @returns {void}
23
35
  */
@@ -30,13 +42,34 @@ const github = ({ level, message, timestamp, metadata = {}, module }) => {
30
42
 
31
43
  const logMessage = `[${time}] ${prettyLevel}${module ? ` (${module})` : ''}: ${message}`;
32
44
 
33
- const logFn = actions[level] ?? notice;
45
+ const command = levelToGitHubCommand[level] ?? 'notice';
46
+
47
+ const data = escapeCommandValue(logMessage, commandDataEscapes);
34
48
 
35
- logFn(logMessage, {
49
+ // Debug messages take no properties
50
+ if (command === 'debug') {
51
+ process.stdout.write(`::debug::${data}${EOL}`);
52
+
53
+ return;
54
+ }
55
+
56
+ const properties = {
36
57
  file: file?.path,
37
- startLine: file?.position?.start.line,
58
+ line: file?.position?.start.line,
38
59
  endLine: file?.position?.end.line,
39
- });
60
+ };
61
+
62
+ const annotations = [];
63
+
64
+ for (const [key, value] of Object.entries(properties)) {
65
+ if (value) {
66
+ annotations.push(
67
+ `${key}=${escapeCommandValue(value, commandPropertyEscapes)}`
68
+ );
69
+ }
70
+ }
71
+
72
+ process.stdout.write(`::${command} ${annotations.join(',')}::${data}${EOL}`);
40
73
  };
41
74
 
42
75
  export default github;