@doc-kit/core 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/package.json +4 -4
- package/src/generators/index.mjs +1 -0
- package/src/generators/loader.mjs +8 -3
- package/src/generators/metadata/types.d.ts +19 -0
- package/src/generators/pipeline.mjs +169 -0
- package/src/generators/types.d.ts +20 -0
- package/src/generators.mjs +19 -12
- package/src/parsers/markdown.mjs +5 -6
- package/src/utils/configuration/constants.mjs +13 -0
- package/src/utils/configuration/index.mjs +7 -2
- package/src/utils/remark-shiki.mjs +29 -0
- package/src/utils/remark.mjs +26 -48
- package/src/utils/type-annotations/hast.mjs +0 -58
- package/src/utils/type-annotations/highlighted.mjs +64 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# @doc-kit/core
|
|
2
2
|
|
|
3
|
+
## 1.1.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- [#1090](https://github.com/nodejs/doc-kit/pull/1090) [`0a9dc37`](https://github.com/nodejs/doc-kit/commit/0a9dc37dc96f8044c76c3ca9b35e97895a87d1fd) Thanks [@avivkeller](https://github.com/avivkeller)! - perf: build the site one page at a time, and `all.html` from the module pages
|
|
8
|
+
|
|
9
|
+
- [#1057](https://github.com/nodejs/doc-kit/pull/1057) [`da5d8e6`](https://github.com/nodejs/doc-kit/commit/da5d8e63d0d33fe7a95c46afb8d5140d576827b4) Thanks [@avivkeller](https://github.com/avivkeller)! - feat: `dependent` generators and the `section-pages` generator
|
|
10
|
+
|
|
3
11
|
## 1.0.1
|
|
4
12
|
|
|
5
13
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@doc-kit/core",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.0
|
|
4
|
+
"version": "1.1.0",
|
|
5
5
|
"description": "The doc-kit engine: the pipeline and shared APIs that power the doc-kit generators and CLI",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
],
|
|
45
45
|
"dependencies": {
|
|
46
46
|
"@actions/core": "^3.0.0",
|
|
47
|
-
"@node-core/rehype-shiki": "^1.
|
|
47
|
+
"@node-core/rehype-shiki": "^1.5.0",
|
|
48
48
|
"@swc/html-wasm": "^1.15.46",
|
|
49
49
|
"@swc/wasm": "^1.15.46",
|
|
50
50
|
"acorn": "^8.17.0",
|
|
@@ -53,7 +53,7 @@
|
|
|
53
53
|
"github-slugger": "^2.0.0",
|
|
54
54
|
"glob-parent": "^6.0.2",
|
|
55
55
|
"hastscript": "^9.0.1",
|
|
56
|
-
"piscina": "^5.3.
|
|
56
|
+
"piscina": "^5.3.1",
|
|
57
57
|
"rehype-stringify": "^10.0.1",
|
|
58
58
|
"remark-gfm": "^4.0.1",
|
|
59
59
|
"remark-mdx": "^3.1.1",
|
|
@@ -61,7 +61,7 @@
|
|
|
61
61
|
"remark-rehype": "^11.1.2",
|
|
62
62
|
"remark-stringify": "^11.0.0",
|
|
63
63
|
"semver": "^7.8.5",
|
|
64
|
-
"shiki": "^4.3
|
|
64
|
+
"shiki": "^4.4.3",
|
|
65
65
|
"tinyglobby": "^0.2.17",
|
|
66
66
|
"unified": "^11.0.5",
|
|
67
67
|
"unist-builder": "^4.0.0",
|
package/src/generators/index.mjs
CHANGED
|
@@ -22,6 +22,7 @@ export const publicGenerators = {
|
|
|
22
22
|
'llms-txt': '@doc-kit/generator-react/llms-txt',
|
|
23
23
|
sitemap: '@doc-kit/generator-react/sitemap',
|
|
24
24
|
html: '@doc-kit/generator-react/html',
|
|
25
|
+
'section-pages': '@doc-kit/generator-react/section-pages',
|
|
25
26
|
};
|
|
26
27
|
|
|
27
28
|
// These ones are special since they don't produce standard output,
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
import { isAbsolute } from 'node:path';
|
|
4
4
|
import { pathToFileURL } from 'node:url';
|
|
5
5
|
|
|
6
|
+
import { enforceArray } from '#utils/array.mjs';
|
|
7
|
+
|
|
6
8
|
import { allGenerators } from './index.mjs';
|
|
7
9
|
|
|
8
10
|
/**
|
|
@@ -67,7 +69,7 @@ export const loadGenerator = async specifier => {
|
|
|
67
69
|
|
|
68
70
|
/**
|
|
69
71
|
* Loads the given generators plus the transitive closure of their
|
|
70
|
-
* dependencies (via `dependsOn`).
|
|
72
|
+
* dependencies (via `dependsOn`) and dependents (via `dependent`).
|
|
71
73
|
*
|
|
72
74
|
* @param {string[]} targets - Shorthand names, paths, or import specifiers
|
|
73
75
|
* @returns {Promise<Map<string, GeneratorMetadata>>} Loaded generators keyed
|
|
@@ -87,8 +89,11 @@ export const loadGenerators = async targets => {
|
|
|
87
89
|
const generator = await loadGenerator(specifier);
|
|
88
90
|
generators.set(specifier, generator);
|
|
89
91
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
+
for (const related of [
|
|
93
|
+
...enforceArray(generator.dependsOn ?? []),
|
|
94
|
+
...enforceArray(generator.dependent ?? []),
|
|
95
|
+
]) {
|
|
96
|
+
queue.push(resolveGeneratorSpecifier(related));
|
|
92
97
|
}
|
|
93
98
|
}
|
|
94
99
|
|
|
@@ -120,6 +120,23 @@ export interface StabilityData extends Data {
|
|
|
120
120
|
*/
|
|
121
121
|
export type StabilityNode = NodeWithData<Blockquote, StabilityData>;
|
|
122
122
|
|
|
123
|
+
/**
|
|
124
|
+
* Describes a chunk page: a section of a module rendered as a page of its own
|
|
125
|
+
* (see the `section-pages` generator). Present on every entry of a chunk page.
|
|
126
|
+
*/
|
|
127
|
+
export interface ChunkInfo {
|
|
128
|
+
/** The `api` of the module the chunk was split from */
|
|
129
|
+
api: string;
|
|
130
|
+
/** The `path` of the module the chunk was split from */
|
|
131
|
+
path: string;
|
|
132
|
+
/** The section's anchor on the full module page */
|
|
133
|
+
slug: string;
|
|
134
|
+
/** The chunk's position within the module, in document order */
|
|
135
|
+
index: number;
|
|
136
|
+
/** The original depth of the chunk's heading */
|
|
137
|
+
depth: number;
|
|
138
|
+
}
|
|
139
|
+
|
|
123
140
|
/**
|
|
124
141
|
* Complete metadata entry for a single API documentation element.
|
|
125
142
|
*
|
|
@@ -136,6 +153,8 @@ export interface MetadataEntry extends YAMLProperties {
|
|
|
136
153
|
synthetic?: boolean;
|
|
137
154
|
/** Whether this page was authored as MDX (JSX-in-Markdown). */
|
|
138
155
|
mdx?: boolean;
|
|
156
|
+
/** Set when this entry belongs to a chunk page rather than a full page. */
|
|
157
|
+
chunk?: ChunkInfo;
|
|
139
158
|
/** Processed heading with metadata */
|
|
140
159
|
heading: HeadingNode;
|
|
141
160
|
/** Stability classification information */
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
import { enforceArray } from '#utils/array.mjs';
|
|
4
|
+
|
|
5
|
+
import { resolveGeneratorSpecifier } from './loader.mjs';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The resolved specifiers of the generators a generator is delivered through.
|
|
9
|
+
*
|
|
10
|
+
* @param {GeneratorMetadata | undefined} generator
|
|
11
|
+
* @returns {string[]}
|
|
12
|
+
*/
|
|
13
|
+
const dependentsOf = generator =>
|
|
14
|
+
enforceArray(generator?.dependent ?? []).map(resolveGeneratorSpecifier);
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Collects the generators reached from the given ones through `dependsOn`.
|
|
18
|
+
*
|
|
19
|
+
* @param {string[]} specifiers - Resolved generator specifiers
|
|
20
|
+
* @param {Map<string, GeneratorMetadata>} generators - Loaded generators
|
|
21
|
+
* @returns {Set<string>}
|
|
22
|
+
*/
|
|
23
|
+
const dependencyClosure = (specifiers, generators) => {
|
|
24
|
+
const closure = new Set();
|
|
25
|
+
const stack = [...specifiers];
|
|
26
|
+
|
|
27
|
+
while (stack.length > 0) {
|
|
28
|
+
const specifier = stack.pop();
|
|
29
|
+
|
|
30
|
+
if (closure.has(specifier)) {
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
closure.add(specifier);
|
|
35
|
+
|
|
36
|
+
const { dependsOn } = generators.get(specifier) ?? {};
|
|
37
|
+
|
|
38
|
+
if (dependsOn) {
|
|
39
|
+
stack.push(resolveGeneratorSpecifier(dependsOn));
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return closure;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Splices a generator into a dependent's pipeline: walking from the dependent
|
|
48
|
+
* up its dependency chain, the first generator that consumes the splicing
|
|
49
|
+
* generator's own input is rewired to consume the splicing generator instead.
|
|
50
|
+
*
|
|
51
|
+
* @param {string} specifier - The generator to splice in
|
|
52
|
+
* @param {string} dependent - The pipeline to splice it into
|
|
53
|
+
* @param {Map<string, string | undefined>} inputOf - Effective inputs, mutated
|
|
54
|
+
* @param {string} name - The splicing generator's name, for errors
|
|
55
|
+
*/
|
|
56
|
+
const splice = (specifier, dependent, inputOf, name) => {
|
|
57
|
+
const upstream = inputOf.get(specifier);
|
|
58
|
+
const visited = new Set();
|
|
59
|
+
|
|
60
|
+
let current = dependent;
|
|
61
|
+
|
|
62
|
+
while (current !== undefined && current !== specifier) {
|
|
63
|
+
if (visited.has(current)) {
|
|
64
|
+
break;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
visited.add(current);
|
|
68
|
+
|
|
69
|
+
if (inputOf.get(current) === upstream) {
|
|
70
|
+
inputOf.set(current, specifier);
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
current = inputOf.get(current);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
throw new Error(
|
|
78
|
+
`Generator "${name}" declares "${dependent}" as its dependent, but ` +
|
|
79
|
+
`nothing in that pipeline consumes "${upstream ?? 'no input'}", so ` +
|
|
80
|
+
'there is nowhere to splice it in.'
|
|
81
|
+
);
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Follows a generator's `dependent` links to the generators that finally
|
|
86
|
+
* deliver its output. When some of its dependents are already part of the
|
|
87
|
+
* run, only those deliver it; otherwise all of them do.
|
|
88
|
+
*
|
|
89
|
+
* @param {string} specifier - Resolved generator specifier
|
|
90
|
+
* @param {Map<string, GeneratorMetadata>} generators - Loaded generators
|
|
91
|
+
* @param {Set<string>} requested - Generators already part of the run
|
|
92
|
+
* @param {Set<string>} [seen] - Guards against dependent cycles
|
|
93
|
+
* @returns {string[]} Resolved specifiers of the terminal generators
|
|
94
|
+
*/
|
|
95
|
+
const resolveTerminals = (
|
|
96
|
+
specifier,
|
|
97
|
+
generators,
|
|
98
|
+
requested,
|
|
99
|
+
seen = new Set()
|
|
100
|
+
) => {
|
|
101
|
+
if (seen.has(specifier)) {
|
|
102
|
+
return [];
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
seen.add(specifier);
|
|
106
|
+
|
|
107
|
+
const dependents = dependentsOf(generators.get(specifier));
|
|
108
|
+
|
|
109
|
+
if (dependents.length === 0) {
|
|
110
|
+
return [specifier];
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const active = dependents.filter(dependent => requested.has(dependent));
|
|
114
|
+
|
|
115
|
+
return (active.length > 0 ? active : dependents).flatMap(dependent =>
|
|
116
|
+
resolveTerminals(dependent, generators, requested, seen)
|
|
117
|
+
);
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Resolves the effective pipeline for a run: which generator each generator
|
|
122
|
+
* reads its input from, and which generators the run finally collects.
|
|
123
|
+
*
|
|
124
|
+
* By default a generator reads from its `dependsOn`. A generator that declares
|
|
125
|
+
* one or more `dependent`s is spliced into each dependent's pipeline (see
|
|
126
|
+
* {@link splice}). Several generators may splice at the same point; each later
|
|
127
|
+
* one is inserted upstream of the earlier ones, forming a chain.
|
|
128
|
+
*
|
|
129
|
+
* A requested generator that declares dependents is delivered through them,
|
|
130
|
+
* so the run collects their output rather than the generator's own: through
|
|
131
|
+
* the dependents that are already part of the run when there are any, and
|
|
132
|
+
* through all of them otherwise.
|
|
133
|
+
*
|
|
134
|
+
* @param {string[]} targets - Requested targets, as resolved specifiers
|
|
135
|
+
* @param {Map<string, GeneratorMetadata>} generators - Loaded generators,
|
|
136
|
+
* including every dependency and dependent of the targets
|
|
137
|
+
* @returns {{ targets: string[], inputOf: Map<string, string | undefined> }}
|
|
138
|
+
* The generators to collect, and each generator's effective input generator
|
|
139
|
+
*/
|
|
140
|
+
export const resolvePipeline = (targets, generators) => {
|
|
141
|
+
/** @type {Map<string, string | undefined>} */
|
|
142
|
+
const inputOf = new Map();
|
|
143
|
+
|
|
144
|
+
for (const [specifier, { dependsOn }] of generators) {
|
|
145
|
+
inputOf.set(
|
|
146
|
+
specifier,
|
|
147
|
+
dependsOn ? resolveGeneratorSpecifier(dependsOn) : undefined
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
for (const [specifier, generator] of generators) {
|
|
152
|
+
for (const dependent of dependentsOf(generator)) {
|
|
153
|
+
splice(specifier, dependent, inputOf, generator.name);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const requested = dependencyClosure(targets, generators);
|
|
158
|
+
const collected = [];
|
|
159
|
+
|
|
160
|
+
for (const target of targets) {
|
|
161
|
+
for (const terminal of resolveTerminals(target, generators, requested)) {
|
|
162
|
+
if (!collected.includes(terminal)) {
|
|
163
|
+
collected.push(terminal);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
return { targets: collected, inputOf };
|
|
169
|
+
};
|
|
@@ -97,6 +97,26 @@ declare global {
|
|
|
97
97
|
*/
|
|
98
98
|
dependsOn: string | undefined;
|
|
99
99
|
|
|
100
|
+
/**
|
|
101
|
+
* The generator(s) this generator's output is delivered through, declared
|
|
102
|
+
* as import specifiers. This is the inverse of `dependsOn`: rather than
|
|
103
|
+
* pulling a dependency in, it pushes this generator into another
|
|
104
|
+
* generator's pipeline.
|
|
105
|
+
*
|
|
106
|
+
* The orchestrator splices this generator in front of the first generator
|
|
107
|
+
* in each dependent's dependency chain that consumes the same `dependsOn`
|
|
108
|
+
* as this one. For example, a generator with
|
|
109
|
+
* `dependsOn: '@doc-kit/core/metadata'` and
|
|
110
|
+
* `dependent: '@doc-kit/generator-react/html'` is inserted between
|
|
111
|
+
* `metadata` and `jsx-ast` (the consumer of `metadata` on the way to
|
|
112
|
+
* `html`), so its output is what `jsx-ast` — and therefore `html` — sees.
|
|
113
|
+
*
|
|
114
|
+
* Requesting a generator that declares dependents runs them as well, and
|
|
115
|
+
* the run's result is their output: when some of the dependents are
|
|
116
|
+
* already part of the run, only those; otherwise all of them.
|
|
117
|
+
*/
|
|
118
|
+
dependent?: string | string[];
|
|
119
|
+
|
|
100
120
|
/**
|
|
101
121
|
* Generators are abstract and the different generators have different sort of inputs and outputs.
|
|
102
122
|
* For example, a MDX generator would take the raw AST and output MDX with React Components;
|
package/src/generators.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
loadGenerators,
|
|
6
6
|
resolveGeneratorSpecifier,
|
|
7
7
|
} from './generators/loader.mjs';
|
|
8
|
+
import { resolvePipeline } from './generators/pipeline.mjs';
|
|
8
9
|
import logger from './logger/index.mjs';
|
|
9
10
|
import createWorkerPool from './threading/index.mjs';
|
|
10
11
|
import createParallelWorker from './threading/parallel.mjs';
|
|
@@ -44,9 +45,12 @@ const createGenerator = () => {
|
|
|
44
45
|
*
|
|
45
46
|
* @param {string} specifier - Resolved generator specifier to schedule
|
|
46
47
|
* @param {Map<string, GeneratorMetadata>} generators - Loaded generators
|
|
48
|
+
* @param {Map<string, string | undefined>} inputOf - Each generator's
|
|
49
|
+
* effective input generator (its dependency, unless another generator was
|
|
50
|
+
* spliced in front of it via `dependent`)
|
|
47
51
|
* @param {import('./utils/configuration/types').Configuration} configuration - Runtime options
|
|
48
52
|
*/
|
|
49
|
-
const scheduleGenerator = (specifier, generators, configuration) => {
|
|
53
|
+
const scheduleGenerator = (specifier, generators, inputOf, configuration) => {
|
|
50
54
|
if (cache.has(specifier)) {
|
|
51
55
|
return;
|
|
52
56
|
}
|
|
@@ -54,12 +58,11 @@ const createGenerator = () => {
|
|
|
54
58
|
const generator = generators.get(specifier);
|
|
55
59
|
const { name, generate, hasParallelProcessor } = generator;
|
|
56
60
|
|
|
57
|
-
const dependsOn =
|
|
58
|
-
generator.dependsOn && resolveGeneratorSpecifier(generator.dependsOn);
|
|
61
|
+
const dependsOn = inputOf.get(specifier);
|
|
59
62
|
|
|
60
63
|
// Schedule dependency first
|
|
61
64
|
if (dependsOn && !cache.has(dependsOn)) {
|
|
62
|
-
scheduleGenerator(dependsOn, generators, configuration);
|
|
65
|
+
scheduleGenerator(dependsOn, generators, inputOf, configuration);
|
|
63
66
|
}
|
|
64
67
|
|
|
65
68
|
generatorsLogger.debug(`Scheduling "${name}"`, {
|
|
@@ -104,8 +107,16 @@ const createGenerator = () => {
|
|
|
104
107
|
|
|
105
108
|
// Resolve shorthand names and load the full dependency closure up front,
|
|
106
109
|
// so scheduling below is fully synchronous.
|
|
107
|
-
const
|
|
108
|
-
|
|
110
|
+
const generators = await loadGenerators(
|
|
111
|
+
target.map(resolveGeneratorSpecifier)
|
|
112
|
+
);
|
|
113
|
+
|
|
114
|
+
// Work out who reads from whom once generators declaring a `dependent`
|
|
115
|
+
// have been spliced in, and which generators the run finally collects.
|
|
116
|
+
const { targets, inputOf } = resolvePipeline(
|
|
117
|
+
target.map(resolveGeneratorSpecifier),
|
|
118
|
+
generators
|
|
119
|
+
);
|
|
109
120
|
|
|
110
121
|
generatorsLogger.debug(`Starting pipeline`, {
|
|
111
122
|
generators: targets.join(', '),
|
|
@@ -114,18 +125,14 @@ const createGenerator = () => {
|
|
|
114
125
|
|
|
115
126
|
// Compute consumer counts up front so dependencies can be evicted as soon
|
|
116
127
|
// as their last consumer runs (must be ready before any generator starts).
|
|
117
|
-
cache.populateConsumerCounts(targets, specifier =>
|
|
118
|
-
const { dependsOn } = generators.get(specifier);
|
|
119
|
-
|
|
120
|
-
return dependsOn && resolveGeneratorSpecifier(dependsOn);
|
|
121
|
-
});
|
|
128
|
+
cache.populateConsumerCounts(targets, specifier => inputOf.get(specifier));
|
|
122
129
|
|
|
123
130
|
// Create worker pool
|
|
124
131
|
pool = createWorkerPool(threads);
|
|
125
132
|
|
|
126
133
|
// Schedule all generators
|
|
127
134
|
for (const specifier of targets) {
|
|
128
|
-
scheduleGenerator(specifier, generators, configuration);
|
|
135
|
+
scheduleGenerator(specifier, generators, inputOf, configuration);
|
|
129
136
|
}
|
|
130
137
|
|
|
131
138
|
// Start all collections in parallel (don't await sequentially). Consuming
|
package/src/parsers/markdown.mjs
CHANGED
|
@@ -22,9 +22,7 @@ const NODE_LTS_VERSION_REGEX = /Long Term Support/i;
|
|
|
22
22
|
export const parseChangelog = async path => {
|
|
23
23
|
const changelog = await loadFromURL(path);
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
return nodeMajors.map(match => ({
|
|
25
|
+
return Array.from(changelog.matchAll(NODE_VERSIONS_REGEX), match => ({
|
|
28
26
|
version: coerce(match[1]),
|
|
29
27
|
isLts: NODE_LTS_VERSION_REGEX.test(match[2]),
|
|
30
28
|
}));
|
|
@@ -43,7 +41,8 @@ export const parseIndex = async path => {
|
|
|
43
41
|
|
|
44
42
|
const index = await loadFromURL(path);
|
|
45
43
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
44
|
+
return Array.from(index.matchAll(LIST_ITEM_REGEX), ([, section, api]) => ({
|
|
45
|
+
section,
|
|
46
|
+
api,
|
|
47
|
+
}));
|
|
49
48
|
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The default `threads` ceiling. Each worker that highlights code holds Shiki's
|
|
5
|
+
* grammars and regex engine (~300MB) on top of the pages it is building, so
|
|
6
|
+
* past a few threads memory, not CPU, is what runs out. `--threads` raises it.
|
|
7
|
+
*/
|
|
8
|
+
export const DEFAULT_MAX_THREADS = 4;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* The default number of items each worker task processes.
|
|
12
|
+
*/
|
|
13
|
+
export const DEFAULT_CHUNK_SIZE = 10;
|
|
@@ -18,6 +18,8 @@ import { enforceArray } from '#utils/array.mjs';
|
|
|
18
18
|
import { leftHandAssign } from '#utils/generators.mjs';
|
|
19
19
|
import { deepMerge } from '#utils/misc.mjs';
|
|
20
20
|
|
|
21
|
+
import { DEFAULT_CHUNK_SIZE, DEFAULT_MAX_THREADS } from './constants.mjs';
|
|
22
|
+
|
|
21
23
|
const configExplorer = cosmiconfig('doc-kit');
|
|
22
24
|
|
|
23
25
|
/**
|
|
@@ -68,8 +70,11 @@ export const getDefaultConfig = (generators, config) =>
|
|
|
68
70
|
// riscv64 with sv39. Running multiple generators that use wasm in
|
|
69
71
|
// parallel could cause failures to allocate new wasm instance.
|
|
70
72
|
// See also https://github.com/nodejs/node/pull/60591
|
|
71
|
-
threads:
|
|
72
|
-
|
|
73
|
+
threads:
|
|
74
|
+
process.arch === 'riscv64'
|
|
75
|
+
? 1
|
|
76
|
+
: Math.min(cpus().length, DEFAULT_MAX_THREADS),
|
|
77
|
+
chunkSize: DEFAULT_CHUNK_SIZE,
|
|
73
78
|
})
|
|
74
79
|
);
|
|
75
80
|
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
import rehypeStringify from 'rehype-stringify';
|
|
4
|
+
import remarkParse from 'remark-parse';
|
|
5
|
+
import remarkRehype from 'remark-rehype';
|
|
6
|
+
import { unified } from 'unified';
|
|
7
|
+
|
|
8
|
+
import syntaxHighlighter from './highlighter.mjs';
|
|
9
|
+
import { lazy } from './misc.mjs';
|
|
10
|
+
import { rehypeOptions } from './remark.mjs';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Retrieves an instance of Remark configured to output stringified HTML code
|
|
14
|
+
* including parsing Code Boxes with syntax highlighting.
|
|
15
|
+
*
|
|
16
|
+
* This lives apart from `./remark.mjs` because importing it loads Shiki (every
|
|
17
|
+
* grammar plus the regex engine, ~250MB per process). Only the generators that
|
|
18
|
+
* highlight code — `legacy-html` here — should pay for that.
|
|
19
|
+
*/
|
|
20
|
+
export const getRemarkRehypeWithShiki = lazy(() =>
|
|
21
|
+
unified()
|
|
22
|
+
.use(remarkParse)
|
|
23
|
+
// legacy-html gets the minimal (unhighlighted) type rendering
|
|
24
|
+
.use(remarkRehype, rehypeOptions)
|
|
25
|
+
// This is a custom ad-hoc within the Shiki Rehype plugin, used to highlight code
|
|
26
|
+
// and transform them into HAST nodes
|
|
27
|
+
.use(syntaxHighlighter)
|
|
28
|
+
.use(rehypeStringify, { allowDangerousHtml: true })
|
|
29
|
+
);
|
package/src/utils/remark.mjs
CHANGED
|
@@ -8,13 +8,13 @@ import remarkRehype from 'remark-rehype';
|
|
|
8
8
|
import remarkStringify from 'remark-stringify';
|
|
9
9
|
import { unified } from 'unified';
|
|
10
10
|
|
|
11
|
-
import syntaxHighlighter from './highlighter.mjs';
|
|
12
11
|
import { lazy } from './misc.mjs';
|
|
13
12
|
import { typeAnnotationToHast } from './type-annotations/hast.mjs';
|
|
14
13
|
import remarkTypeAnnotations from './type-annotations/remark.mjs';
|
|
15
14
|
|
|
16
|
-
//
|
|
17
|
-
|
|
15
|
+
// Nothing in this module loads Shiki: the `ast` and `metadata` stages (and
|
|
16
|
+
// every worker that runs them) import it, and none of them highlight code.
|
|
17
|
+
// The highlighting pipeline lives in `./remark-shiki.mjs`.
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
20
|
* Renders an MDX JSX element as just its children, so the surrounding prose
|
|
@@ -30,15 +30,28 @@ const mdxElementToChildren = (state, node) => state.all(node);
|
|
|
30
30
|
*/
|
|
31
31
|
const dropNode = () => undefined;
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
33
|
+
/**
|
|
34
|
+
* The `remark-rehype` options shared by the HTML-string pipelines.
|
|
35
|
+
*
|
|
36
|
+
* Existing HTML nodes pass through untouched (they were created during the
|
|
37
|
+
* rehype process), and dangerous HTML is allowed since the Markdown sources
|
|
38
|
+
* are trusted. The MDX node types cannot be rendered to an HTML string (that
|
|
39
|
+
* is the React generators' job): JSX elements degrade to their children so the
|
|
40
|
+
* surrounding prose still renders, and expressions/ESM are dropped.
|
|
41
|
+
*
|
|
42
|
+
* @type {import('remark-rehype').Options}
|
|
43
|
+
*/
|
|
44
|
+
export const rehypeOptions = {
|
|
45
|
+
allowDangerousHtml: true,
|
|
46
|
+
passThrough: ['element'],
|
|
47
|
+
handlers: {
|
|
48
|
+
typeAnnotation: typeAnnotationToHast,
|
|
49
|
+
mdxJsxTextElement: mdxElementToChildren,
|
|
50
|
+
mdxJsxFlowElement: mdxElementToChildren,
|
|
51
|
+
mdxFlowExpression: dropNode,
|
|
52
|
+
mdxTextExpression: dropNode,
|
|
53
|
+
mdxjsEsm: dropNode,
|
|
54
|
+
},
|
|
42
55
|
};
|
|
43
56
|
|
|
44
57
|
/**
|
|
@@ -72,41 +85,6 @@ export const getRemarkMdx = lazy(() =>
|
|
|
72
85
|
export const getRemarkRehype = lazy(() =>
|
|
73
86
|
unified()
|
|
74
87
|
.use(remarkParse)
|
|
75
|
-
|
|
76
|
-
// as these are nodes we manually created during the rehype process
|
|
77
|
-
// We also allow dangerous HTML to be passed through, since we have HTML within our Markdown
|
|
78
|
-
// and we trust the sources of the Markdown files
|
|
79
|
-
.use(remarkRehype, {
|
|
80
|
-
allowDangerousHtml: true,
|
|
81
|
-
passThrough,
|
|
82
|
-
handlers: { typeAnnotation: typeAnnotationToHast, ...mdxToHastHandlers },
|
|
83
|
-
})
|
|
84
|
-
// We allow dangerous HTML to be passed through, since we have HTML within our Markdown
|
|
85
|
-
// and we trust the sources of the Markdown files
|
|
86
|
-
.use(rehypeStringify, { allowDangerousHtml: true })
|
|
87
|
-
);
|
|
88
|
-
|
|
89
|
-
/**
|
|
90
|
-
* Retrieves an instance of Remark configured to output stringified HTML code
|
|
91
|
-
* including parsing Code Boxes with syntax highlighting
|
|
92
|
-
*/
|
|
93
|
-
export const getRemarkRehypeWithShiki = lazy(() =>
|
|
94
|
-
unified()
|
|
95
|
-
.use(remarkParse)
|
|
96
|
-
// We make Rehype ignore existing HTML nodes (just the node itself, not its children)
|
|
97
|
-
// as these are nodes we manually created during the rehype process
|
|
98
|
-
// We also allow dangerous HTML to be passed through, since we have HTML within our Markdown
|
|
99
|
-
// and we trust the sources of the Markdown files
|
|
100
|
-
.use(remarkRehype, {
|
|
101
|
-
allowDangerousHtml: true,
|
|
102
|
-
passThrough,
|
|
103
|
-
// legacy-html gets the minimal (unhighlighted) type rendering
|
|
104
|
-
handlers: { typeAnnotation: typeAnnotationToHast, ...mdxToHastHandlers },
|
|
105
|
-
})
|
|
106
|
-
// This is a custom ad-hoc within the Shiki Rehype plugin, used to highlight code
|
|
107
|
-
// and transform them into HAST nodes
|
|
108
|
-
.use(syntaxHighlighter)
|
|
109
|
-
// We allow dangerous HTML to be passed through, since we have HTML within our Markdown
|
|
110
|
-
// and we trust the sources of the Markdown files
|
|
88
|
+
.use(remarkRehype, rehypeOptions)
|
|
111
89
|
.use(rehypeStringify, { allowDangerousHtml: true })
|
|
112
90
|
);
|
|
@@ -1,9 +1,5 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
import { highlighter } from '#utils/highlighter.mjs';
|
|
4
|
-
|
|
5
|
-
const [lightTheme, darkTheme] = highlighter.shiki.getLoadedThemes();
|
|
6
|
-
|
|
7
3
|
/**
|
|
8
4
|
* Slices a type's text by its resolved link ranges into hast children —
|
|
9
5
|
* plain text segments interleaved with `<a class="type-link">` anchors.
|
|
@@ -59,57 +55,3 @@ export const typeAnnotationToHast = (state, node) => {
|
|
|
59
55
|
|
|
60
56
|
return state.applyData(node, result);
|
|
61
57
|
};
|
|
62
|
-
|
|
63
|
-
/**
|
|
64
|
-
* Syntax-highlighted mdast→hast handler for `typeAnnotation` nodes, used by
|
|
65
|
-
* the web (JSX) pipeline. The whole type is highlighted as one inline
|
|
66
|
-
* fragment, and each resolved identifier's exact character range is wrapped
|
|
67
|
-
* in an `<a>` via Shiki decorations. Values that are not TypeScript (display
|
|
68
|
-
* names such as `HTTP/2 Headers Object`) are highlighted as plain text, so
|
|
69
|
-
* their prose is not coloured as operators and numeric literals.
|
|
70
|
-
*
|
|
71
|
-
* Falls back to the minimal handler when the type failed to parse or nothing
|
|
72
|
-
* resolved (no point paying for highlighting then).
|
|
73
|
-
*
|
|
74
|
-
* @param {import('mdast-util-to-hast').State} state
|
|
75
|
-
* @param {import('mdast').Node} node
|
|
76
|
-
* @returns {import('hast').Element}
|
|
77
|
-
*/
|
|
78
|
-
export const typeAnnotationToHighlightedHast = (state, node) => {
|
|
79
|
-
const links = node.data?.links ?? [];
|
|
80
|
-
|
|
81
|
-
if (node.data?.parseError || links.length === 0) {
|
|
82
|
-
return typeAnnotationToHast(state, node);
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
const root = highlighter.shiki.codeToHast(node.value, {
|
|
86
|
-
lang: node.data?.typescript ? 'typescript' : 'text',
|
|
87
|
-
themes: { light: lightTheme, dark: darkTheme },
|
|
88
|
-
decorations: links.map(({ start, end, href }) => ({
|
|
89
|
-
start,
|
|
90
|
-
end,
|
|
91
|
-
tagName: 'a',
|
|
92
|
-
properties: { href, class: 'type-link' },
|
|
93
|
-
alwaysWrap: true,
|
|
94
|
-
})),
|
|
95
|
-
});
|
|
96
|
-
|
|
97
|
-
// codeToHast wraps the highlighted line in <pre><code>; re-shape that into
|
|
98
|
-
// a single inline <code> element ("only the outermost type opens/closes
|
|
99
|
-
// the code fragment")
|
|
100
|
-
const [preElement] = root.children;
|
|
101
|
-
const [codeElement] = preElement.children;
|
|
102
|
-
|
|
103
|
-
const result = {
|
|
104
|
-
type: 'element',
|
|
105
|
-
tagName: 'code',
|
|
106
|
-
properties: {
|
|
107
|
-
class: `${preElement.properties.class} type`,
|
|
108
|
-
},
|
|
109
|
-
children: codeElement.children,
|
|
110
|
-
};
|
|
111
|
-
|
|
112
|
-
state.patch(node, result);
|
|
113
|
-
|
|
114
|
-
return state.applyData(node, result);
|
|
115
|
-
};
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
import { highlighter } from '#utils/highlighter.mjs';
|
|
4
|
+
|
|
5
|
+
import { typeAnnotationToHast } from './hast.mjs';
|
|
6
|
+
|
|
7
|
+
// Kept apart from `./hast.mjs` on purpose: importing this module loads Shiki
|
|
8
|
+
// (every grammar plus the regex engine), which only the pipelines that
|
|
9
|
+
// highlight should pay for. The `ast` and `metadata` stages never do.
|
|
10
|
+
const [lightTheme, darkTheme] = highlighter.shiki.getLoadedThemes();
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Syntax-highlighted mdast→hast handler for `typeAnnotation` nodes, used by
|
|
14
|
+
* the web (JSX) pipeline. The whole type is highlighted as one inline
|
|
15
|
+
* fragment, and each resolved identifier's exact character range is wrapped
|
|
16
|
+
* in an `<a>` via Shiki decorations. Values that are not TypeScript (display
|
|
17
|
+
* names such as `HTTP/2 Headers Object`) are highlighted as plain text, so
|
|
18
|
+
* their prose is not coloured as operators and numeric literals.
|
|
19
|
+
*
|
|
20
|
+
* Falls back to the minimal handler when the type failed to parse or nothing
|
|
21
|
+
* resolved (no point paying for highlighting then).
|
|
22
|
+
*
|
|
23
|
+
* @param {import('mdast-util-to-hast').State} state
|
|
24
|
+
* @param {import('mdast').Node} node
|
|
25
|
+
* @returns {import('hast').Element}
|
|
26
|
+
*/
|
|
27
|
+
export const typeAnnotationToHighlightedHast = (state, node) => {
|
|
28
|
+
const links = node.data?.links ?? [];
|
|
29
|
+
|
|
30
|
+
if (node.data?.parseError || links.length === 0) {
|
|
31
|
+
return typeAnnotationToHast(state, node);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const root = highlighter.shiki.codeToHast(node.value, {
|
|
35
|
+
lang: node.data?.typescript ? 'typescript' : 'text',
|
|
36
|
+
themes: { light: lightTheme, dark: darkTheme },
|
|
37
|
+
decorations: links.map(({ start, end, href }) => ({
|
|
38
|
+
start,
|
|
39
|
+
end,
|
|
40
|
+
tagName: 'a',
|
|
41
|
+
properties: { href, class: 'type-link' },
|
|
42
|
+
alwaysWrap: true,
|
|
43
|
+
})),
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
// codeToHast wraps the highlighted line in <pre><code>; re-shape that into
|
|
47
|
+
// a single inline <code> element ("only the outermost type opens/closes
|
|
48
|
+
// the code fragment")
|
|
49
|
+
const [preElement] = root.children;
|
|
50
|
+
const [codeElement] = preElement.children;
|
|
51
|
+
|
|
52
|
+
const result = {
|
|
53
|
+
type: 'element',
|
|
54
|
+
tagName: 'code',
|
|
55
|
+
properties: {
|
|
56
|
+
class: `${preElement.properties.class} type`,
|
|
57
|
+
},
|
|
58
|
+
children: codeElement.children,
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
state.patch(node, result);
|
|
62
|
+
|
|
63
|
+
return state.applyData(node, result);
|
|
64
|
+
};
|