@doc-kit/core 1.0.1 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,13 @@
1
1
  # @doc-kit/core
2
2
 
3
+ ## 1.1.0
4
+
5
+ ### Minor Changes
6
+
7
+ - [#1090](https://github.com/nodejs/doc-kit/pull/1090) [`0a9dc37`](https://github.com/nodejs/doc-kit/commit/0a9dc37dc96f8044c76c3ca9b35e97895a87d1fd) Thanks [@avivkeller](https://github.com/avivkeller)! - perf: build the site one page at a time, and `all.html` from the module pages
8
+
9
+ - [#1057](https://github.com/nodejs/doc-kit/pull/1057) [`da5d8e6`](https://github.com/nodejs/doc-kit/commit/da5d8e63d0d33fe7a95c46afb8d5140d576827b4) Thanks [@avivkeller](https://github.com/avivkeller)! - feat: `dependent` generators and the `section-pages` generator
10
+
3
11
  ## 1.0.1
4
12
 
5
13
  ### Patch Changes
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@doc-kit/core",
3
3
  "type": "module",
4
- "version": "1.0.1",
4
+ "version": "1.1.0",
5
5
  "description": "The doc-kit engine: the pipeline and shared APIs that power the doc-kit generators and CLI",
6
6
  "repository": {
7
7
  "type": "git",
@@ -44,7 +44,7 @@
44
44
  ],
45
45
  "dependencies": {
46
46
  "@actions/core": "^3.0.0",
47
- "@node-core/rehype-shiki": "^1.4.3",
47
+ "@node-core/rehype-shiki": "^1.5.0",
48
48
  "@swc/html-wasm": "^1.15.46",
49
49
  "@swc/wasm": "^1.15.46",
50
50
  "acorn": "^8.17.0",
@@ -53,7 +53,7 @@
53
53
  "github-slugger": "^2.0.0",
54
54
  "glob-parent": "^6.0.2",
55
55
  "hastscript": "^9.0.1",
56
- "piscina": "^5.3.0",
56
+ "piscina": "^5.3.1",
57
57
  "rehype-stringify": "^10.0.1",
58
58
  "remark-gfm": "^4.0.1",
59
59
  "remark-mdx": "^3.1.1",
@@ -61,7 +61,7 @@
61
61
  "remark-rehype": "^11.1.2",
62
62
  "remark-stringify": "^11.0.0",
63
63
  "semver": "^7.8.5",
64
- "shiki": "^4.3.0",
64
+ "shiki": "^4.4.3",
65
65
  "tinyglobby": "^0.2.17",
66
66
  "unified": "^11.0.5",
67
67
  "unist-builder": "^4.0.0",
@@ -22,6 +22,7 @@ export const publicGenerators = {
22
22
  'llms-txt': '@doc-kit/generator-react/llms-txt',
23
23
  sitemap: '@doc-kit/generator-react/sitemap',
24
24
  html: '@doc-kit/generator-react/html',
25
+ 'section-pages': '@doc-kit/generator-react/section-pages',
25
26
  };
26
27
 
27
28
  // These ones are special since they don't produce standard output,
@@ -3,6 +3,8 @@
3
3
  import { isAbsolute } from 'node:path';
4
4
  import { pathToFileURL } from 'node:url';
5
5
 
6
+ import { enforceArray } from '#utils/array.mjs';
7
+
6
8
  import { allGenerators } from './index.mjs';
7
9
 
8
10
  /**
@@ -67,7 +69,7 @@ export const loadGenerator = async specifier => {
67
69
 
68
70
  /**
69
71
  * Loads the given generators plus the transitive closure of their
70
- * dependencies (via `dependsOn`).
72
+ * dependencies (via `dependsOn`) and dependents (via `dependent`).
71
73
  *
72
74
  * @param {string[]} targets - Shorthand names, paths, or import specifiers
73
75
  * @returns {Promise<Map<string, GeneratorMetadata>>} Loaded generators keyed
@@ -87,8 +89,11 @@ export const loadGenerators = async targets => {
87
89
  const generator = await loadGenerator(specifier);
88
90
  generators.set(specifier, generator);
89
91
 
90
- if (generator.dependsOn) {
91
- queue.push(resolveGeneratorSpecifier(generator.dependsOn));
92
+ for (const related of [
93
+ ...enforceArray(generator.dependsOn ?? []),
94
+ ...enforceArray(generator.dependent ?? []),
95
+ ]) {
96
+ queue.push(resolveGeneratorSpecifier(related));
92
97
  }
93
98
  }
94
99
 
@@ -120,6 +120,23 @@ export interface StabilityData extends Data {
120
120
  */
121
121
  export type StabilityNode = NodeWithData<Blockquote, StabilityData>;
122
122
 
123
+ /**
124
+ * Describes a chunk page: a section of a module rendered as a page of its own
125
+ * (see the `section-pages` generator). Present on every entry of a chunk page.
126
+ */
127
+ export interface ChunkInfo {
128
+ /** The `api` of the module the chunk was split from */
129
+ api: string;
130
+ /** The `path` of the module the chunk was split from */
131
+ path: string;
132
+ /** The section's anchor on the full module page */
133
+ slug: string;
134
+ /** The chunk's position within the module, in document order */
135
+ index: number;
136
+ /** The original depth of the chunk's heading */
137
+ depth: number;
138
+ }
139
+
123
140
  /**
124
141
  * Complete metadata entry for a single API documentation element.
125
142
  *
@@ -136,6 +153,8 @@ export interface MetadataEntry extends YAMLProperties {
136
153
  synthetic?: boolean;
137
154
  /** Whether this page was authored as MDX (JSX-in-Markdown). */
138
155
  mdx?: boolean;
156
+ /** Set when this entry belongs to a chunk page rather than a full page. */
157
+ chunk?: ChunkInfo;
139
158
  /** Processed heading with metadata */
140
159
  heading: HeadingNode;
141
160
  /** Stability classification information */
@@ -0,0 +1,169 @@
1
+ 'use strict';
2
+
3
+ import { enforceArray } from '#utils/array.mjs';
4
+
5
+ import { resolveGeneratorSpecifier } from './loader.mjs';
6
+
7
+ /**
8
+ * The resolved specifiers of the generators a generator is delivered through.
9
+ *
10
+ * @param {GeneratorMetadata | undefined} generator
11
+ * @returns {string[]}
12
+ */
13
+ const dependentsOf = generator =>
14
+ enforceArray(generator?.dependent ?? []).map(resolveGeneratorSpecifier);
15
+
16
+ /**
17
+ * Collects the generators reached from the given ones through `dependsOn`.
18
+ *
19
+ * @param {string[]} specifiers - Resolved generator specifiers
20
+ * @param {Map<string, GeneratorMetadata>} generators - Loaded generators
21
+ * @returns {Set<string>}
22
+ */
23
+ const dependencyClosure = (specifiers, generators) => {
24
+ const closure = new Set();
25
+ const stack = [...specifiers];
26
+
27
+ while (stack.length > 0) {
28
+ const specifier = stack.pop();
29
+
30
+ if (closure.has(specifier)) {
31
+ continue;
32
+ }
33
+
34
+ closure.add(specifier);
35
+
36
+ const { dependsOn } = generators.get(specifier) ?? {};
37
+
38
+ if (dependsOn) {
39
+ stack.push(resolveGeneratorSpecifier(dependsOn));
40
+ }
41
+ }
42
+
43
+ return closure;
44
+ };
45
+
46
+ /**
47
+ * Splices a generator into a dependent's pipeline: walking from the dependent
48
+ * up its dependency chain, the first generator that consumes the splicing
49
+ * generator's own input is rewired to consume the splicing generator instead.
50
+ *
51
+ * @param {string} specifier - The generator to splice in
52
+ * @param {string} dependent - The pipeline to splice it into
53
+ * @param {Map<string, string | undefined>} inputOf - Effective inputs, mutated
54
+ * @param {string} name - The splicing generator's name, for errors
55
+ */
56
+ const splice = (specifier, dependent, inputOf, name) => {
57
+ const upstream = inputOf.get(specifier);
58
+ const visited = new Set();
59
+
60
+ let current = dependent;
61
+
62
+ while (current !== undefined && current !== specifier) {
63
+ if (visited.has(current)) {
64
+ break;
65
+ }
66
+
67
+ visited.add(current);
68
+
69
+ if (inputOf.get(current) === upstream) {
70
+ inputOf.set(current, specifier);
71
+ return;
72
+ }
73
+
74
+ current = inputOf.get(current);
75
+ }
76
+
77
+ throw new Error(
78
+ `Generator "${name}" declares "${dependent}" as its dependent, but ` +
79
+ `nothing in that pipeline consumes "${upstream ?? 'no input'}", so ` +
80
+ 'there is nowhere to splice it in.'
81
+ );
82
+ };
83
+
84
+ /**
85
+ * Follows a generator's `dependent` links to the generators that finally
86
+ * deliver its output. When some of its dependents are already part of the
87
+ * run, only those deliver it; otherwise all of them do.
88
+ *
89
+ * @param {string} specifier - Resolved generator specifier
90
+ * @param {Map<string, GeneratorMetadata>} generators - Loaded generators
91
+ * @param {Set<string>} requested - Generators already part of the run
92
+ * @param {Set<string>} [seen] - Guards against dependent cycles
93
+ * @returns {string[]} Resolved specifiers of the terminal generators
94
+ */
95
+ const resolveTerminals = (
96
+ specifier,
97
+ generators,
98
+ requested,
99
+ seen = new Set()
100
+ ) => {
101
+ if (seen.has(specifier)) {
102
+ return [];
103
+ }
104
+
105
+ seen.add(specifier);
106
+
107
+ const dependents = dependentsOf(generators.get(specifier));
108
+
109
+ if (dependents.length === 0) {
110
+ return [specifier];
111
+ }
112
+
113
+ const active = dependents.filter(dependent => requested.has(dependent));
114
+
115
+ return (active.length > 0 ? active : dependents).flatMap(dependent =>
116
+ resolveTerminals(dependent, generators, requested, seen)
117
+ );
118
+ };
119
+
120
+ /**
121
+ * Resolves the effective pipeline for a run: which generator each generator
122
+ * reads its input from, and which generators the run finally collects.
123
+ *
124
+ * By default a generator reads from its `dependsOn`. A generator that declares
125
+ * one or more `dependent`s is spliced into each dependent's pipeline (see
126
+ * {@link splice}). Several generators may splice at the same point; each later
127
+ * one is inserted upstream of the earlier ones, forming a chain.
128
+ *
129
+ * A requested generator that declares dependents is delivered through them,
130
+ * so the run collects their output rather than the generator's own: through
131
+ * the dependents that are already part of the run when there are any, and
132
+ * through all of them otherwise.
133
+ *
134
+ * @param {string[]} targets - Requested targets, as resolved specifiers
135
+ * @param {Map<string, GeneratorMetadata>} generators - Loaded generators,
136
+ * including every dependency and dependent of the targets
137
+ * @returns {{ targets: string[], inputOf: Map<string, string | undefined> }}
138
+ * The generators to collect, and each generator's effective input generator
139
+ */
140
+ export const resolvePipeline = (targets, generators) => {
141
+ /** @type {Map<string, string | undefined>} */
142
+ const inputOf = new Map();
143
+
144
+ for (const [specifier, { dependsOn }] of generators) {
145
+ inputOf.set(
146
+ specifier,
147
+ dependsOn ? resolveGeneratorSpecifier(dependsOn) : undefined
148
+ );
149
+ }
150
+
151
+ for (const [specifier, generator] of generators) {
152
+ for (const dependent of dependentsOf(generator)) {
153
+ splice(specifier, dependent, inputOf, generator.name);
154
+ }
155
+ }
156
+
157
+ const requested = dependencyClosure(targets, generators);
158
+ const collected = [];
159
+
160
+ for (const target of targets) {
161
+ for (const terminal of resolveTerminals(target, generators, requested)) {
162
+ if (!collected.includes(terminal)) {
163
+ collected.push(terminal);
164
+ }
165
+ }
166
+ }
167
+
168
+ return { targets: collected, inputOf };
169
+ };
@@ -97,6 +97,26 @@ declare global {
97
97
  */
98
98
  dependsOn: string | undefined;
99
99
 
100
+ /**
101
+ * The generator(s) this generator's output is delivered through, declared
102
+ * as import specifiers. This is the inverse of `dependsOn`: rather than
103
+ * pulling a dependency in, it pushes this generator into another
104
+ * generator's pipeline.
105
+ *
106
+ * The orchestrator splices this generator in front of the first generator
107
+ * in each dependent's dependency chain that consumes the same `dependsOn`
108
+ * as this one. For example, a generator with
109
+ * `dependsOn: '@doc-kit/core/metadata'` and
110
+ * `dependent: '@doc-kit/generator-react/html'` is inserted between
111
+ * `metadata` and `jsx-ast` (the consumer of `metadata` on the way to
112
+ * `html`), so its output is what `jsx-ast` — and therefore `html` — sees.
113
+ *
114
+ * Requesting a generator that declares dependents runs them as well, and
115
+ * the run's result is their output: when some of the dependents are
116
+ * already part of the run, only those; otherwise all of them.
117
+ */
118
+ dependent?: string | string[];
119
+
100
120
  /**
101
121
  * Generators are abstract and the different generators have different sort of inputs and outputs.
102
122
  * For example, a MDX generator would take the raw AST and output MDX with React Components;
@@ -5,6 +5,7 @@ import {
5
5
  loadGenerators,
6
6
  resolveGeneratorSpecifier,
7
7
  } from './generators/loader.mjs';
8
+ import { resolvePipeline } from './generators/pipeline.mjs';
8
9
  import logger from './logger/index.mjs';
9
10
  import createWorkerPool from './threading/index.mjs';
10
11
  import createParallelWorker from './threading/parallel.mjs';
@@ -44,9 +45,12 @@ const createGenerator = () => {
44
45
  *
45
46
  * @param {string} specifier - Resolved generator specifier to schedule
46
47
  * @param {Map<string, GeneratorMetadata>} generators - Loaded generators
48
+ * @param {Map<string, string | undefined>} inputOf - Each generator's
49
+ * effective input generator (its dependency, unless another generator was
50
+ * spliced in front of it via `dependent`)
47
51
  * @param {import('./utils/configuration/types').Configuration} configuration - Runtime options
48
52
  */
49
- const scheduleGenerator = (specifier, generators, configuration) => {
53
+ const scheduleGenerator = (specifier, generators, inputOf, configuration) => {
50
54
  if (cache.has(specifier)) {
51
55
  return;
52
56
  }
@@ -54,12 +58,11 @@ const createGenerator = () => {
54
58
  const generator = generators.get(specifier);
55
59
  const { name, generate, hasParallelProcessor } = generator;
56
60
 
57
- const dependsOn =
58
- generator.dependsOn && resolveGeneratorSpecifier(generator.dependsOn);
61
+ const dependsOn = inputOf.get(specifier);
59
62
 
60
63
  // Schedule dependency first
61
64
  if (dependsOn && !cache.has(dependsOn)) {
62
- scheduleGenerator(dependsOn, generators, configuration);
65
+ scheduleGenerator(dependsOn, generators, inputOf, configuration);
63
66
  }
64
67
 
65
68
  generatorsLogger.debug(`Scheduling "${name}"`, {
@@ -104,8 +107,16 @@ const createGenerator = () => {
104
107
 
105
108
  // Resolve shorthand names and load the full dependency closure up front,
106
109
  // so scheduling below is fully synchronous.
107
- const targets = target.map(resolveGeneratorSpecifier);
108
- const generators = await loadGenerators(targets);
110
+ const generators = await loadGenerators(
111
+ target.map(resolveGeneratorSpecifier)
112
+ );
113
+
114
+ // Work out who reads from whom once generators declaring a `dependent`
115
+ // have been spliced in, and which generators the run finally collects.
116
+ const { targets, inputOf } = resolvePipeline(
117
+ target.map(resolveGeneratorSpecifier),
118
+ generators
119
+ );
109
120
 
110
121
  generatorsLogger.debug(`Starting pipeline`, {
111
122
  generators: targets.join(', '),
@@ -114,18 +125,14 @@ const createGenerator = () => {
114
125
 
115
126
  // Compute consumer counts up front so dependencies can be evicted as soon
116
127
  // as their last consumer runs (must be ready before any generator starts).
117
- cache.populateConsumerCounts(targets, specifier => {
118
- const { dependsOn } = generators.get(specifier);
119
-
120
- return dependsOn && resolveGeneratorSpecifier(dependsOn);
121
- });
128
+ cache.populateConsumerCounts(targets, specifier => inputOf.get(specifier));
122
129
 
123
130
  // Create worker pool
124
131
  pool = createWorkerPool(threads);
125
132
 
126
133
  // Schedule all generators
127
134
  for (const specifier of targets) {
128
- scheduleGenerator(specifier, generators, configuration);
135
+ scheduleGenerator(specifier, generators, inputOf, configuration);
129
136
  }
130
137
 
131
138
  // Start all collections in parallel (don't await sequentially). Consuming
@@ -22,9 +22,7 @@ const NODE_LTS_VERSION_REGEX = /Long Term Support/i;
22
22
  export const parseChangelog = async path => {
23
23
  const changelog = await loadFromURL(path);
24
24
 
25
- const nodeMajors = Array.from(changelog.matchAll(NODE_VERSIONS_REGEX));
26
-
27
- return nodeMajors.map(match => ({
25
+ return Array.from(changelog.matchAll(NODE_VERSIONS_REGEX), match => ({
28
26
  version: coerce(match[1]),
29
27
  isLts: NODE_LTS_VERSION_REGEX.test(match[2]),
30
28
  }));
@@ -43,7 +41,8 @@ export const parseIndex = async path => {
43
41
 
44
42
  const index = await loadFromURL(path);
45
43
 
46
- const items = Array.from(index.matchAll(LIST_ITEM_REGEX));
47
-
48
- return items.map(([, section, api]) => ({ section, api }));
44
+ return Array.from(index.matchAll(LIST_ITEM_REGEX), ([, section, api]) => ({
45
+ section,
46
+ api,
47
+ }));
49
48
  };
@@ -0,0 +1,13 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * The default `threads` ceiling. Each worker that highlights code holds Shiki's
5
+ * grammars and regex engine (~300MB) on top of the pages it is building, so
6
+ * past a few threads memory, not CPU, is what runs out. `--threads` raises it.
7
+ */
8
+ export const DEFAULT_MAX_THREADS = 4;
9
+
10
+ /**
11
+ * The default number of items each worker task processes.
12
+ */
13
+ export const DEFAULT_CHUNK_SIZE = 10;
@@ -18,6 +18,8 @@ import { enforceArray } from '#utils/array.mjs';
18
18
  import { leftHandAssign } from '#utils/generators.mjs';
19
19
  import { deepMerge } from '#utils/misc.mjs';
20
20
 
21
+ import { DEFAULT_CHUNK_SIZE, DEFAULT_MAX_THREADS } from './constants.mjs';
22
+
21
23
  const configExplorer = cosmiconfig('doc-kit');
22
24
 
23
25
  /**
@@ -68,8 +70,11 @@ export const getDefaultConfig = (generators, config) =>
68
70
  // riscv64 with sv39. Running multiple generators that use wasm in
69
71
  // parallel could cause failures to allocate new wasm instance.
70
72
  // See also https://github.com/nodejs/node/pull/60591
71
- threads: process.arch === 'riscv64' ? 1 : cpus().length,
72
- chunkSize: 10,
73
+ threads:
74
+ process.arch === 'riscv64'
75
+ ? 1
76
+ : Math.min(cpus().length, DEFAULT_MAX_THREADS),
77
+ chunkSize: DEFAULT_CHUNK_SIZE,
73
78
  })
74
79
  );
75
80
 
@@ -0,0 +1,29 @@
1
+ 'use strict';
2
+
3
+ import rehypeStringify from 'rehype-stringify';
4
+ import remarkParse from 'remark-parse';
5
+ import remarkRehype from 'remark-rehype';
6
+ import { unified } from 'unified';
7
+
8
+ import syntaxHighlighter from './highlighter.mjs';
9
+ import { lazy } from './misc.mjs';
10
+ import { rehypeOptions } from './remark.mjs';
11
+
12
+ /**
13
+ * Retrieves an instance of Remark configured to output stringified HTML code
14
+ * including parsing Code Boxes with syntax highlighting.
15
+ *
16
+ * This lives apart from `./remark.mjs` because importing it loads Shiki (every
17
+ * grammar plus the regex engine, ~250MB per process). Only the generators that
18
+ * highlight code — `legacy-html` here — should pay for that.
19
+ */
20
+ export const getRemarkRehypeWithShiki = lazy(() =>
21
+ unified()
22
+ .use(remarkParse)
23
+ // legacy-html gets the minimal (unhighlighted) type rendering
24
+ .use(remarkRehype, rehypeOptions)
25
+ // This is a custom ad-hoc within the Shiki Rehype plugin, used to highlight code
26
+ // and transform them into HAST nodes
27
+ .use(syntaxHighlighter)
28
+ .use(rehypeStringify, { allowDangerousHtml: true })
29
+ );
@@ -8,13 +8,13 @@ import remarkRehype from 'remark-rehype';
8
8
  import remarkStringify from 'remark-stringify';
9
9
  import { unified } from 'unified';
10
10
 
11
- import syntaxHighlighter from './highlighter.mjs';
12
11
  import { lazy } from './misc.mjs';
13
12
  import { typeAnnotationToHast } from './type-annotations/hast.mjs';
14
13
  import remarkTypeAnnotations from './type-annotations/remark.mjs';
15
14
 
16
- // Nodes the rehype pipelines pass through untouched.
17
- const passThrough = ['element'];
15
+ // Nothing in this module loads Shiki: the `ast` and `metadata` stages (and
16
+ // every worker that runs them) import it, and none of them highlight code.
17
+ // The highlighting pipeline lives in `./remark-shiki.mjs`.
18
18
 
19
19
  /**
20
20
  * Renders an MDX JSX element as just its children, so the surrounding prose
@@ -30,15 +30,28 @@ const mdxElementToChildren = (state, node) => state.all(node);
30
30
  */
31
31
  const dropNode = () => undefined;
32
32
 
33
- // The HTML-string pipelines cannot render MDX nodes (rendering those is the
34
- // React generators' job): JSX elements degrade to their children so the
35
- // surrounding prose still renders, and expressions/ESM are dropped.
36
- const mdxToHastHandlers = {
37
- mdxJsxTextElement: mdxElementToChildren,
38
- mdxJsxFlowElement: mdxElementToChildren,
39
- mdxFlowExpression: dropNode,
40
- mdxTextExpression: dropNode,
41
- mdxjsEsm: dropNode,
33
+ /**
34
+ * The `remark-rehype` options shared by the HTML-string pipelines.
35
+ *
36
+ * Existing HTML nodes pass through untouched (they were created during the
37
+ * rehype process), and dangerous HTML is allowed since the Markdown sources
38
+ * are trusted. The MDX node types cannot be rendered to an HTML string (that
39
+ * is the React generators' job): JSX elements degrade to their children so the
40
+ * surrounding prose still renders, and expressions/ESM are dropped.
41
+ *
42
+ * @type {import('remark-rehype').Options}
43
+ */
44
+ export const rehypeOptions = {
45
+ allowDangerousHtml: true,
46
+ passThrough: ['element'],
47
+ handlers: {
48
+ typeAnnotation: typeAnnotationToHast,
49
+ mdxJsxTextElement: mdxElementToChildren,
50
+ mdxJsxFlowElement: mdxElementToChildren,
51
+ mdxFlowExpression: dropNode,
52
+ mdxTextExpression: dropNode,
53
+ mdxjsEsm: dropNode,
54
+ },
42
55
  };
43
56
 
44
57
  /**
@@ -72,41 +85,6 @@ export const getRemarkMdx = lazy(() =>
72
85
  export const getRemarkRehype = lazy(() =>
73
86
  unified()
74
87
  .use(remarkParse)
75
- // We make Rehype ignore existing HTML nodes (just the node itself, not its children)
76
- // as these are nodes we manually created during the rehype process
77
- // We also allow dangerous HTML to be passed through, since we have HTML within our Markdown
78
- // and we trust the sources of the Markdown files
79
- .use(remarkRehype, {
80
- allowDangerousHtml: true,
81
- passThrough,
82
- handlers: { typeAnnotation: typeAnnotationToHast, ...mdxToHastHandlers },
83
- })
84
- // We allow dangerous HTML to be passed through, since we have HTML within our Markdown
85
- // and we trust the sources of the Markdown files
86
- .use(rehypeStringify, { allowDangerousHtml: true })
87
- );
88
-
89
- /**
90
- * Retrieves an instance of Remark configured to output stringified HTML code
91
- * including parsing Code Boxes with syntax highlighting
92
- */
93
- export const getRemarkRehypeWithShiki = lazy(() =>
94
- unified()
95
- .use(remarkParse)
96
- // We make Rehype ignore existing HTML nodes (just the node itself, not its children)
97
- // as these are nodes we manually created during the rehype process
98
- // We also allow dangerous HTML to be passed through, since we have HTML within our Markdown
99
- // and we trust the sources of the Markdown files
100
- .use(remarkRehype, {
101
- allowDangerousHtml: true,
102
- passThrough,
103
- // legacy-html gets the minimal (unhighlighted) type rendering
104
- handlers: { typeAnnotation: typeAnnotationToHast, ...mdxToHastHandlers },
105
- })
106
- // This is a custom ad-hoc within the Shiki Rehype plugin, used to highlight code
107
- // and transform them into HAST nodes
108
- .use(syntaxHighlighter)
109
- // We allow dangerous HTML to be passed through, since we have HTML within our Markdown
110
- // and we trust the sources of the Markdown files
88
+ .use(remarkRehype, rehypeOptions)
111
89
  .use(rehypeStringify, { allowDangerousHtml: true })
112
90
  );
@@ -1,9 +1,5 @@
1
1
  'use strict';
2
2
 
3
- import { highlighter } from '#utils/highlighter.mjs';
4
-
5
- const [lightTheme, darkTheme] = highlighter.shiki.getLoadedThemes();
6
-
7
3
  /**
8
4
  * Slices a type's text by its resolved link ranges into hast children —
9
5
  * plain text segments interleaved with `<a class="type-link">` anchors.
@@ -59,57 +55,3 @@ export const typeAnnotationToHast = (state, node) => {
59
55
 
60
56
  return state.applyData(node, result);
61
57
  };
62
-
63
- /**
64
- * Syntax-highlighted mdast→hast handler for `typeAnnotation` nodes, used by
65
- * the web (JSX) pipeline. The whole type is highlighted as one inline
66
- * fragment, and each resolved identifier's exact character range is wrapped
67
- * in an `<a>` via Shiki decorations. Values that are not TypeScript (display
68
- * names such as `HTTP/2 Headers Object`) are highlighted as plain text, so
69
- * their prose is not coloured as operators and numeric literals.
70
- *
71
- * Falls back to the minimal handler when the type failed to parse or nothing
72
- * resolved (no point paying for highlighting then).
73
- *
74
- * @param {import('mdast-util-to-hast').State} state
75
- * @param {import('mdast').Node} node
76
- * @returns {import('hast').Element}
77
- */
78
- export const typeAnnotationToHighlightedHast = (state, node) => {
79
- const links = node.data?.links ?? [];
80
-
81
- if (node.data?.parseError || links.length === 0) {
82
- return typeAnnotationToHast(state, node);
83
- }
84
-
85
- const root = highlighter.shiki.codeToHast(node.value, {
86
- lang: node.data?.typescript ? 'typescript' : 'text',
87
- themes: { light: lightTheme, dark: darkTheme },
88
- decorations: links.map(({ start, end, href }) => ({
89
- start,
90
- end,
91
- tagName: 'a',
92
- properties: { href, class: 'type-link' },
93
- alwaysWrap: true,
94
- })),
95
- });
96
-
97
- // codeToHast wraps the highlighted line in <pre><code>; re-shape that into
98
- // a single inline <code> element ("only the outermost type opens/closes
99
- // the code fragment")
100
- const [preElement] = root.children;
101
- const [codeElement] = preElement.children;
102
-
103
- const result = {
104
- type: 'element',
105
- tagName: 'code',
106
- properties: {
107
- class: `${preElement.properties.class} type`,
108
- },
109
- children: codeElement.children,
110
- };
111
-
112
- state.patch(node, result);
113
-
114
- return state.applyData(node, result);
115
- };
@@ -0,0 +1,64 @@
1
+ 'use strict';
2
+
3
+ import { highlighter } from '#utils/highlighter.mjs';
4
+
5
+ import { typeAnnotationToHast } from './hast.mjs';
6
+
7
+ // Kept apart from `./hast.mjs` on purpose: importing this module loads Shiki
8
+ // (every grammar plus the regex engine), which only the pipelines that
9
+ // highlight should pay for. The `ast` and `metadata` stages never do.
10
+ const [lightTheme, darkTheme] = highlighter.shiki.getLoadedThemes();
11
+
12
+ /**
13
+ * Syntax-highlighted mdast→hast handler for `typeAnnotation` nodes, used by
14
+ * the web (JSX) pipeline. The whole type is highlighted as one inline
15
+ * fragment, and each resolved identifier's exact character range is wrapped
16
+ * in an `<a>` via Shiki decorations. Values that are not TypeScript (display
17
+ * names such as `HTTP/2 Headers Object`) are highlighted as plain text, so
18
+ * their prose is not coloured as operators and numeric literals.
19
+ *
20
+ * Falls back to the minimal handler when the type failed to parse or nothing
21
+ * resolved (no point paying for highlighting then).
22
+ *
23
+ * @param {import('mdast-util-to-hast').State} state
24
+ * @param {import('mdast').Node} node
25
+ * @returns {import('hast').Element}
26
+ */
27
+ export const typeAnnotationToHighlightedHast = (state, node) => {
28
+ const links = node.data?.links ?? [];
29
+
30
+ if (node.data?.parseError || links.length === 0) {
31
+ return typeAnnotationToHast(state, node);
32
+ }
33
+
34
+ const root = highlighter.shiki.codeToHast(node.value, {
35
+ lang: node.data?.typescript ? 'typescript' : 'text',
36
+ themes: { light: lightTheme, dark: darkTheme },
37
+ decorations: links.map(({ start, end, href }) => ({
38
+ start,
39
+ end,
40
+ tagName: 'a',
41
+ properties: { href, class: 'type-link' },
42
+ alwaysWrap: true,
43
+ })),
44
+ });
45
+
46
+ // codeToHast wraps the highlighted line in <pre><code>; re-shape that into
47
+ // a single inline <code> element ("only the outermost type opens/closes
48
+ // the code fragment")
49
+ const [preElement] = root.children;
50
+ const [codeElement] = preElement.children;
51
+
52
+ const result = {
53
+ type: 'element',
54
+ tagName: 'code',
55
+ properties: {
56
+ class: `${preElement.properties.class} type`,
57
+ },
58
+ children: codeElement.children,
59
+ };
60
+
61
+ state.patch(node, result);
62
+
63
+ return state.applyData(node, result);
64
+ };