@terrazzo/parser 2.0.0-alpha.6 → 2.0.0-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +9 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +205 -86
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
- package/src/parse/load.ts +9 -8
- package/src/parse/process.ts +144 -16
- package/src/parse/token.ts +24 -10
- package/src/resolver/load.ts +0 -1
- package/src/resolver/normalize.ts +106 -79
- package/src/types.ts +8 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@terrazzo/parser",
|
|
3
|
-
"version": "2.0.0-alpha.
|
|
3
|
+
"version": "2.0.0-alpha.7",
|
|
4
4
|
"description": "Parser/validator for the Design Tokens Community Group (DTCG) standard.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -43,8 +43,8 @@
|
|
|
43
43
|
"picocolors": "^1.1.1",
|
|
44
44
|
"scule": "^1.3.0",
|
|
45
45
|
"wildcard-match": "^5.1.4",
|
|
46
|
-
"@terrazzo/json-schema-tools": "^0.
|
|
47
|
-
"@terrazzo/token-tools": "^2.0.0-alpha.
|
|
46
|
+
"@terrazzo/json-schema-tools": "^0.2.0",
|
|
47
|
+
"@terrazzo/token-tools": "^2.0.0-alpha.7"
|
|
48
48
|
},
|
|
49
49
|
"peerDependencies": {
|
|
50
50
|
"yaml-to-momoa": "0.0.8"
|
package/src/parse/load.ts
CHANGED
|
@@ -2,10 +2,10 @@ import * as momoa from '@humanwhocodes/momoa';
|
|
|
2
2
|
import {
|
|
3
3
|
type BundleOptions,
|
|
4
4
|
bundle,
|
|
5
|
+
encodeFragment,
|
|
5
6
|
getObjMember,
|
|
6
7
|
type InputSource,
|
|
7
8
|
type InputSourceWithDocument,
|
|
8
|
-
type RefMap,
|
|
9
9
|
replaceNode,
|
|
10
10
|
traverse,
|
|
11
11
|
} from '@terrazzo/json-schema-tools';
|
|
@@ -75,9 +75,6 @@ export async function loadSources(
|
|
|
75
75
|
}));
|
|
76
76
|
/** The sources array, indexed by filename */
|
|
77
77
|
let sourceByFilename: Record<string, InputSourceWithDocument> = {};
|
|
78
|
-
/** Mapping of all final $ref resolutions. This will be used to generate the graph later. */
|
|
79
|
-
let refMap: RefMap = {};
|
|
80
|
-
|
|
81
78
|
try {
|
|
82
79
|
const result = await bundle(sources, {
|
|
83
80
|
req,
|
|
@@ -86,7 +83,6 @@ export async function loadSources(
|
|
|
86
83
|
});
|
|
87
84
|
document = result.document;
|
|
88
85
|
sourceByFilename = result.sources;
|
|
89
|
-
refMap = result.refMap;
|
|
90
86
|
for (const [filename, source] of Object.entries(result.sources)) {
|
|
91
87
|
const i = sources.findIndex((s) => s.filename.href === filename);
|
|
92
88
|
if (i === -1) {
|
|
@@ -111,9 +107,14 @@ export async function loadSources(
|
|
|
111
107
|
}
|
|
112
108
|
logger.debug({ ...entry, message: `JSON loaded`, timing: performance.now() - firstLoad });
|
|
113
109
|
|
|
114
|
-
const rootSource = {
|
|
110
|
+
const rootSource = {
|
|
111
|
+
filename: sources[0]!.filename!,
|
|
112
|
+
document,
|
|
113
|
+
src: momoa.print(document, { indent: 2 }).replace(/\\\//g, '/'),
|
|
114
|
+
};
|
|
115
|
+
|
|
115
116
|
return {
|
|
116
|
-
tokens: processTokens(rootSource, { config, logger,
|
|
117
|
+
tokens: processTokens(rootSource, { config, logger, sources, sourceByFilename }),
|
|
117
118
|
sources,
|
|
118
119
|
};
|
|
119
120
|
}
|
|
@@ -141,7 +142,7 @@ function transformer(transform: TransformVisitors): BundleOptions['parse'] {
|
|
|
141
142
|
const ctx = { filename, parent, path };
|
|
142
143
|
const next$type = getObjMember(node, '$type');
|
|
143
144
|
if (next$type?.type === 'String') {
|
|
144
|
-
const jsonPath =
|
|
145
|
+
const jsonPath = encodeFragment(path);
|
|
145
146
|
if (jsonPath.startsWith(lastPath)) {
|
|
146
147
|
last$type = next$type.value;
|
|
147
148
|
}
|
package/src/parse/process.ts
CHANGED
|
@@ -1,11 +1,22 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
1
|
+
import * as momoa from '@humanwhocodes/momoa';
|
|
2
|
+
import {
|
|
3
|
+
encodeFragment,
|
|
4
|
+
findNode,
|
|
5
|
+
getObjMember,
|
|
6
|
+
type InputSourceWithDocument,
|
|
7
|
+
mergeObjects,
|
|
8
|
+
parseRef,
|
|
9
|
+
replaceNode,
|
|
10
|
+
traverse,
|
|
11
|
+
} from '@terrazzo/json-schema-tools';
|
|
12
|
+
import { type GroupNormalized, isAlias, type TokenNormalizedSet } from '@terrazzo/token-tools';
|
|
3
13
|
import { filterResolverPaths } from '../lib/resolver-utils.js';
|
|
4
14
|
import type Logger from '../logger.js';
|
|
5
15
|
import { isLikelyResolver } from '../resolver/validate.js';
|
|
6
|
-
import type { ConfigInit } from '../types.js';
|
|
16
|
+
import type { ConfigInit, RefMap } from '../types.js';
|
|
7
17
|
import { normalize } from './normalize.js';
|
|
8
18
|
import {
|
|
19
|
+
aliasToGroupRef,
|
|
9
20
|
graphAliases,
|
|
10
21
|
groupFromNode,
|
|
11
22
|
refToTokenID,
|
|
@@ -18,17 +29,136 @@ export interface ProcessTokensOptions {
|
|
|
18
29
|
config: ConfigInit;
|
|
19
30
|
logger: Logger;
|
|
20
31
|
sourceByFilename: Record<string, InputSourceWithDocument>;
|
|
21
|
-
refMap: RefMap;
|
|
22
32
|
sources: InputSourceWithDocument[];
|
|
23
33
|
}
|
|
24
34
|
|
|
25
35
|
export function processTokens(
|
|
26
36
|
rootSource: InputSourceWithDocument,
|
|
27
|
-
{ config, logger, sourceByFilename
|
|
37
|
+
{ config, logger, sourceByFilename }: ProcessTokensOptions,
|
|
28
38
|
): TokenNormalizedSet {
|
|
29
39
|
const entry = { group: 'parser' as const, label: 'init' };
|
|
30
40
|
|
|
31
|
-
//
|
|
41
|
+
// 1. Inline $refs to discover any additional tokens
|
|
42
|
+
const refMap: RefMap = {};
|
|
43
|
+
function resolveRef(node: momoa.StringNode, chain: string[]): momoa.AnyNode {
|
|
44
|
+
const { subpath } = parseRef(node.value);
|
|
45
|
+
if (!subpath) {
|
|
46
|
+
logger.error({ ...entry, message: 'Can’t resolve $ref', node, src: rootSource.src });
|
|
47
|
+
// exit
|
|
48
|
+
}
|
|
49
|
+
const next = findNode(rootSource.document, subpath);
|
|
50
|
+
if (next?.type === 'Object') {
|
|
51
|
+
const next$ref = getObjMember(next, '$ref');
|
|
52
|
+
if (next$ref && next$ref.type === 'String') {
|
|
53
|
+
if (chain.includes(next$ref.value)) {
|
|
54
|
+
logger.error({
|
|
55
|
+
...entry,
|
|
56
|
+
message: `Circular $ref detected: ${JSON.stringify(next$ref.value)}`,
|
|
57
|
+
node: next$ref,
|
|
58
|
+
src: rootSource.src,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
chain.push(next$ref.value);
|
|
62
|
+
return resolveRef(next$ref, chain);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return next;
|
|
66
|
+
}
|
|
67
|
+
const inlineStart = performance.now();
|
|
68
|
+
traverse(rootSource.document, {
|
|
69
|
+
enter(node, _parent, rawPath) {
|
|
70
|
+
if (rawPath.includes('$extensions') || node.type !== 'Object') {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
const $ref = node.type === 'Object' ? (getObjMember(node, '$ref') as momoa.StringNode) : undefined;
|
|
74
|
+
if (!$ref) {
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
if ($ref.type !== 'String') {
|
|
78
|
+
logger.error({ ...entry, message: 'Invalid $ref. Expected string.', node: $ref, src: rootSource.src });
|
|
79
|
+
}
|
|
80
|
+
const jsonID = encodeFragment(rawPath);
|
|
81
|
+
refMap[jsonID] = { filename: rootSource.filename.href, refChain: [$ref.value] };
|
|
82
|
+
const resolved = resolveRef($ref, refMap[jsonID]!.refChain);
|
|
83
|
+
if (resolved.type === 'Object') {
|
|
84
|
+
node.members.splice(
|
|
85
|
+
node.members.findIndex((m) => m.name.type === 'String' && m.name.value === '$ref'),
|
|
86
|
+
1,
|
|
87
|
+
);
|
|
88
|
+
replaceNode(node, mergeObjects(resolved, node));
|
|
89
|
+
} else {
|
|
90
|
+
replaceNode(node, resolved);
|
|
91
|
+
}
|
|
92
|
+
},
|
|
93
|
+
});
|
|
94
|
+
logger.debug({ ...entry, message: 'Inline aliases', timing: performance.now() - inlineStart });
|
|
95
|
+
|
|
96
|
+
// 2. Resolve $extends to discover any more additional tokens
|
|
97
|
+
function flatten$extends(node: momoa.ObjectNode, chain: string[]) {
|
|
98
|
+
const memberKeys = node.members.map((m) => m.name.type === 'String' && m.name.value).filter(Boolean) as string[];
|
|
99
|
+
|
|
100
|
+
let extended: momoa.ObjectNode | undefined;
|
|
101
|
+
|
|
102
|
+
if (memberKeys.includes('$extends')) {
|
|
103
|
+
const $extends = getObjMember(node, '$extends') as momoa.StringNode;
|
|
104
|
+
if ($extends.type !== 'String') {
|
|
105
|
+
logger.error({ ...entry, message: '$extends must be a string', node: $extends, src: rootSource.src });
|
|
106
|
+
}
|
|
107
|
+
if (memberKeys.includes('$value')) {
|
|
108
|
+
logger.error({ ...entry, message: '$extends can’t exist within a token', node: $extends, src: rootSource.src });
|
|
109
|
+
}
|
|
110
|
+
const next = isAlias($extends.value) ? aliasToGroupRef($extends.value) : undefined;
|
|
111
|
+
if (!next) {
|
|
112
|
+
logger.error({ ...entry, message: '$extends must be a valid alias', node: $extends, src: rootSource.src });
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
if (
|
|
116
|
+
chain.includes(next!.$ref) ||
|
|
117
|
+
// Check that $extends is not importing from higher up (could go in either direction, which is why we check both ways)
|
|
118
|
+
chain.some((value) => value.startsWith(next!.$ref) || next!.$ref.startsWith(value))
|
|
119
|
+
) {
|
|
120
|
+
logger.error({ ...entry, message: 'Circular $extends detected', node: $extends, src: rootSource.src });
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
chain.push(next!.$ref);
|
|
124
|
+
extended = findNode(rootSource.document, parseRef(next!.$ref).subpath ?? []);
|
|
125
|
+
if (!extended) {
|
|
126
|
+
logger.error({ ...entry, message: 'Could not resolve $extends', node: $extends, src: rootSource.src });
|
|
127
|
+
}
|
|
128
|
+
if (extended!.type !== 'Object') {
|
|
129
|
+
logger.error({ ...entry, message: '$extends must resolve to a group of tokens', node });
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// To ensure this is resolvable, try and flatten this node first (will catch circular refs)
|
|
133
|
+
flatten$extends(extended!, chain);
|
|
134
|
+
|
|
135
|
+
replaceNode(node, mergeObjects(extended!, node));
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// Deeply-traverse for any interior $extends (even if it wasn’t at the top level)
|
|
139
|
+
for (const member of node.members) {
|
|
140
|
+
if (
|
|
141
|
+
member.value.type === 'Object' &&
|
|
142
|
+
member.name.type === 'String' &&
|
|
143
|
+
!['$value', '$extensions'].includes(member.name.value)
|
|
144
|
+
) {
|
|
145
|
+
traverse(member.value, {
|
|
146
|
+
enter(subnode, _parent) {
|
|
147
|
+
if (subnode.type === 'Object') {
|
|
148
|
+
flatten$extends(subnode, chain);
|
|
149
|
+
}
|
|
150
|
+
},
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const extendsStart = performance.now();
|
|
157
|
+
const extendsChain: string[] = [];
|
|
158
|
+
flatten$extends(rootSource.document.body as momoa.ObjectNode, extendsChain);
|
|
159
|
+
logger.debug({ ...entry, message: 'Resolving $extends', timing: performance.now() - extendsStart });
|
|
160
|
+
|
|
161
|
+
// 3. Parse discovered tokens
|
|
32
162
|
const firstPass = performance.now();
|
|
33
163
|
const tokens: TokenNormalizedSet = {};
|
|
34
164
|
// micro-optimization: while we’re iterating over tokens, keeping a “hot”
|
|
@@ -37,7 +167,7 @@ export function processTokens(
|
|
|
37
167
|
const tokenIDs: string[] = [];
|
|
38
168
|
const groups: Record<string, GroupNormalized> = {};
|
|
39
169
|
|
|
40
|
-
//
|
|
170
|
+
// 3a. Token & group population
|
|
41
171
|
const isResolver = isLikelyResolver(rootSource.document);
|
|
42
172
|
traverse(rootSource.document, {
|
|
43
173
|
enter(node, _parent, rawPath) {
|
|
@@ -61,7 +191,7 @@ export function processTokens(
|
|
|
61
191
|
logger.debug({ ...entry, message: 'Parsing: 1st pass', timing: performance.now() - firstPass });
|
|
62
192
|
const secondPass = performance.now();
|
|
63
193
|
|
|
64
|
-
//
|
|
194
|
+
// 3b. Resolve originalValue and original sources
|
|
65
195
|
for (const source of Object.values(sourceByFilename)) {
|
|
66
196
|
traverse(source.document, {
|
|
67
197
|
enter(node, _parent, path) {
|
|
@@ -82,18 +212,18 @@ export function processTokens(
|
|
|
82
212
|
});
|
|
83
213
|
}
|
|
84
214
|
|
|
85
|
-
//
|
|
215
|
+
// 3c. DTCG alias resolution
|
|
86
216
|
// Unlike $refs which can be resolved as we go, these can’t happen until the final, flattened set
|
|
87
217
|
resolveAliases(tokens, { logger, sources: sourceByFilename, refMap });
|
|
88
218
|
logger.debug({ ...entry, message: 'Parsing: 2nd pass', timing: performance.now() - secondPass });
|
|
89
219
|
|
|
90
|
-
//
|
|
220
|
+
// 4. Alias graph
|
|
91
221
|
// We’ve resolved aliases, but we need this pass for reverse linking i.e. “aliasedBy”
|
|
92
222
|
const aliasStart = performance.now();
|
|
93
223
|
graphAliases(refMap, { tokens, logger, sources: sourceByFilename });
|
|
94
224
|
logger.debug({ ...entry, message: 'Alias graph built', timing: performance.now() - aliasStart });
|
|
95
225
|
|
|
96
|
-
//
|
|
226
|
+
// 5. normalize
|
|
97
227
|
// Allow for some minor variance in inputs, and be nice to folks.
|
|
98
228
|
const normalizeStart = performance.now();
|
|
99
229
|
for (const id of tokenIDs) {
|
|
@@ -102,15 +232,12 @@ export function processTokens(
|
|
|
102
232
|
}
|
|
103
233
|
logger.debug({ ...entry, message: 'Normalized values', timing: performance.now() - normalizeStart });
|
|
104
234
|
|
|
105
|
-
//
|
|
235
|
+
// 6. alphabetize & filter
|
|
106
236
|
// This can’t happen until the last step, where we’re 100% sure we’ve resolved everything.
|
|
237
|
+
const sortStart = performance.now();
|
|
107
238
|
const tokensSorted: TokenNormalizedSet = {};
|
|
108
239
|
tokenIDs.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
|
|
109
240
|
for (const path of tokenIDs) {
|
|
110
|
-
// Filter out any tokens in $defs (we needed to reference them earlier, but shouldn’t include them in the final assortment)
|
|
111
|
-
if (path.includes('/$defs/')) {
|
|
112
|
-
continue;
|
|
113
|
-
}
|
|
114
241
|
const id = refToTokenID(path)!;
|
|
115
242
|
tokensSorted[id] = tokens[path]!;
|
|
116
243
|
}
|
|
@@ -118,6 +245,7 @@ export function processTokens(
|
|
|
118
245
|
for (const group of Object.values(groups)) {
|
|
119
246
|
group.tokens.sort((a, b) => a.localeCompare(b, 'en-us', { numeric: true }));
|
|
120
247
|
}
|
|
248
|
+
logger.debug({ ...entry, message: 'Sorted tokens', timing: performance.now() - sortStart });
|
|
121
249
|
|
|
122
250
|
return tokensSorted;
|
|
123
251
|
}
|
package/src/parse/token.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as momoa from '@humanwhocodes/momoa';
|
|
2
|
-
import { getObjMember, type InputSourceWithDocument, parseRef
|
|
2
|
+
import { encodeFragment, getObjMember, type InputSourceWithDocument, parseRef } from '@terrazzo/json-schema-tools';
|
|
3
3
|
import {
|
|
4
4
|
type GroupNormalized,
|
|
5
5
|
isAlias,
|
|
@@ -9,10 +9,20 @@ import {
|
|
|
9
9
|
} from '@terrazzo/token-tools';
|
|
10
10
|
import wcmatch from 'wildcard-match';
|
|
11
11
|
import type { default as Logger } from '../logger.js';
|
|
12
|
-
import type { Config, ReferenceObject } from '../types.js';
|
|
12
|
+
import type { Config, ReferenceObject, RefMap } from '../types.js';
|
|
13
13
|
|
|
14
14
|
/** Convert valid DTCG alias to $ref */
|
|
15
|
-
export function
|
|
15
|
+
export function aliasToGroupRef(alias: string): ReferenceObject | undefined {
|
|
16
|
+
const id = parseAlias(alias);
|
|
17
|
+
// if this is invalid, stop
|
|
18
|
+
if (id === alias) {
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
return { $ref: `#/${id.replace(/~/g, '~0').replace(/\//g, '~1').replace(/\./g, '/')}` };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Convert valid DTCG alias to $ref */
|
|
25
|
+
export function aliasToTokenRef(alias: string, mode?: string): ReferenceObject | undefined {
|
|
16
26
|
const id = parseAlias(alias);
|
|
17
27
|
// if this is invalid, stop
|
|
18
28
|
if (id === alias) {
|
|
@@ -40,12 +50,12 @@ export function tokenFromNode(
|
|
|
40
50
|
return undefined;
|
|
41
51
|
}
|
|
42
52
|
|
|
43
|
-
const jsonID =
|
|
53
|
+
const jsonID = encodeFragment(path);
|
|
44
54
|
const id = path.join('.');
|
|
45
55
|
|
|
46
56
|
const originalToken = momoa.evaluate(node) as any;
|
|
47
57
|
|
|
48
|
-
const groupID =
|
|
58
|
+
const groupID = encodeFragment(path.slice(0, -1));
|
|
49
59
|
const group = groups[groupID]!;
|
|
50
60
|
if (group?.tokens && !group.tokens.includes(id)) {
|
|
51
61
|
group.tokens.push(id);
|
|
@@ -132,7 +142,7 @@ export function tokenRawValuesFromNode(
|
|
|
132
142
|
return undefined;
|
|
133
143
|
}
|
|
134
144
|
|
|
135
|
-
const jsonID =
|
|
145
|
+
const jsonID = encodeFragment(path);
|
|
136
146
|
const rawValues: TokenRawValues = {
|
|
137
147
|
jsonID,
|
|
138
148
|
originalValue: momoa.evaluate(node),
|
|
@@ -173,7 +183,7 @@ export function groupFromNode(
|
|
|
173
183
|
{ path, groups }: { path: string[]; groups: Record<string, GroupNormalized> },
|
|
174
184
|
): GroupNormalized {
|
|
175
185
|
const id = path.join('.');
|
|
176
|
-
const jsonID =
|
|
186
|
+
const jsonID = encodeFragment(path);
|
|
177
187
|
|
|
178
188
|
// group
|
|
179
189
|
if (!groups[jsonID]) {
|
|
@@ -354,7 +364,7 @@ export function aliasToMomoa(
|
|
|
354
364
|
end: { line: -1, column: -1, offset: 0 },
|
|
355
365
|
},
|
|
356
366
|
): momoa.ObjectNode | undefined {
|
|
357
|
-
const $ref =
|
|
367
|
+
const $ref = aliasToTokenRef(alias);
|
|
358
368
|
if (!$ref) {
|
|
359
369
|
return;
|
|
360
370
|
}
|
|
@@ -383,6 +393,10 @@ export function refToTokenID($ref: ReferenceObject | string): string | undefined
|
|
|
383
393
|
return;
|
|
384
394
|
}
|
|
385
395
|
const { subpath } = parseRef(path);
|
|
396
|
+
// if this ID comes from #/$defs/…, strip the first 2 segments to get the global ID
|
|
397
|
+
if (subpath?.[0] === '$defs') {
|
|
398
|
+
subpath.splice(0, 2);
|
|
399
|
+
}
|
|
386
400
|
return (subpath?.length && subpath.join('.').replace(/\.(\$value|\$extensions).*$/, '')) || undefined;
|
|
387
401
|
}
|
|
388
402
|
|
|
@@ -422,7 +436,7 @@ const EXPECTED_NESTED_ALIAS: Record<string, Record<string, string[]>> = {
|
|
|
422
436
|
};
|
|
423
437
|
|
|
424
438
|
/**
|
|
425
|
-
* Resolve DTCG aliases
|
|
439
|
+
* Resolve DTCG aliases, $extends, and $ref
|
|
426
440
|
*/
|
|
427
441
|
export function resolveAliases(
|
|
428
442
|
tokens: TokenNormalizedSet,
|
|
@@ -438,7 +452,7 @@ export function resolveAliases(
|
|
|
438
452
|
|
|
439
453
|
for (const mode of Object.keys(token.mode)) {
|
|
440
454
|
function resolveInner(alias: string, refChain: string[]): string {
|
|
441
|
-
const nextRef =
|
|
455
|
+
const nextRef = aliasToTokenRef(alias, mode)?.$ref;
|
|
442
456
|
if (!nextRef) {
|
|
443
457
|
logger.error({ ...aliasEntry, message: `Internal error resolving ${JSON.stringify(refChain)}` });
|
|
444
458
|
throw new Error('Internal error');
|
package/src/resolver/load.ts
CHANGED
|
@@ -1,15 +1,8 @@
|
|
|
1
1
|
import * as momoa from '@humanwhocodes/momoa';
|
|
2
|
-
import { bundle,
|
|
2
|
+
import { bundle, encodeFragment, parseRef, replaceNode } from '@terrazzo/json-schema-tools';
|
|
3
3
|
import type yamlToMomoa from 'yaml-to-momoa';
|
|
4
4
|
import type Logger from '../logger.js';
|
|
5
|
-
import type {
|
|
6
|
-
ResolverModifierInline,
|
|
7
|
-
ResolverModifierNormalized,
|
|
8
|
-
ResolverSetInline,
|
|
9
|
-
ResolverSetNormalized,
|
|
10
|
-
ResolverSourceNormalized,
|
|
11
|
-
} from '../types.js';
|
|
12
|
-
import { validateModifier, validateSet } from './validate.js';
|
|
5
|
+
import type { Group, ReferenceObject, ResolverSourceNormalized } from '../types.js';
|
|
13
6
|
|
|
14
7
|
export interface NormalizeResolverOptions {
|
|
15
8
|
logger: Logger;
|
|
@@ -21,11 +14,46 @@ export interface NormalizeResolverOptions {
|
|
|
21
14
|
|
|
22
15
|
/** Normalize resolver (assuming it’s been validated) */
|
|
23
16
|
export async function normalizeResolver(
|
|
24
|
-
|
|
25
|
-
{ filename, req, src,
|
|
17
|
+
document: momoa.DocumentNode,
|
|
18
|
+
{ logger, filename, req, src, yamlToMomoa }: NormalizeResolverOptions,
|
|
26
19
|
): Promise<ResolverSourceNormalized> {
|
|
27
|
-
|
|
28
|
-
|
|
20
|
+
// Important note: think about sets, modifiers, and resolutionOrder all
|
|
21
|
+
// containing their own partial tokens documents. Now think about JSON $refs
|
|
22
|
+
// inside those. Because we want to treat them all as one _eventual_ document,
|
|
23
|
+
// we defer resolving $refs until the very last step. In most setups, this has
|
|
24
|
+
// no effect on the final result, however, in the scenario where remote
|
|
25
|
+
// documents are loaded and they conflict in unexpected ways, resolving too
|
|
26
|
+
// early will produce incorrect results.
|
|
27
|
+
//
|
|
28
|
+
// To prevent this, we bundle ONCE at the very top level, with the `$defs` at
|
|
29
|
+
// the top level now containing all partial documents (as opposed to bundling
|
|
30
|
+
// every sub document individually). So all that said, we are deciding to
|
|
31
|
+
// choose the “all-in-one“ method for closer support with DTCG aliases, but at
|
|
32
|
+
// the expense of some edge cases of $refs behaving unexpectedly.
|
|
33
|
+
const resolverBundle = await bundle([{ filename, src }], { req, yamlToMomoa });
|
|
34
|
+
const resolverSource = momoa.evaluate(resolverBundle.document) as unknown as ResolverSourceNormalized;
|
|
35
|
+
|
|
36
|
+
// Resolve $refs, but in a very different way than everywhere else These are
|
|
37
|
+
// all _evaluated_, meaning initialized in JS memory. Unlike in the AST, when
|
|
38
|
+
// we resolve these they’ll share memory points (which isn’t possible in the
|
|
39
|
+
// AST—values must be duplicated). This code is unique because it’s the only
|
|
40
|
+
// place where we’re dealing with shared, initialized JS memory.
|
|
41
|
+
replaceNode(document, resolverBundle.document); // inject $defs into the root document
|
|
42
|
+
for (const set of Object.values(resolverSource.sets ?? {})) {
|
|
43
|
+
for (const source of set.sources) {
|
|
44
|
+
resolvePartials(source, { resolver: resolverSource, logger });
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
for (const modifier of Object.values(resolverSource.modifiers ?? {})) {
|
|
48
|
+
for (const context of Object.values(modifier.contexts)) {
|
|
49
|
+
for (const source of context) {
|
|
50
|
+
resolvePartials(source, { resolver: resolverSource, logger });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
for (const item of resolverSource.resolutionOrder ?? []) {
|
|
55
|
+
resolvePartials(item, { resolver: resolverSource, logger });
|
|
56
|
+
}
|
|
29
57
|
|
|
30
58
|
return {
|
|
31
59
|
name: resolverSource.name,
|
|
@@ -33,74 +61,73 @@ export async function normalizeResolver(
|
|
|
33
61
|
description: resolverSource.description,
|
|
34
62
|
sets: resolverSource.sets,
|
|
35
63
|
modifiers: resolverSource.modifiers,
|
|
36
|
-
resolutionOrder:
|
|
37
|
-
resolutionOrder.elements.map(async (element, i) => {
|
|
38
|
-
const layer = element.value as momoa.ObjectNode;
|
|
39
|
-
const members = getObjMembers(layer);
|
|
40
|
-
|
|
41
|
-
// If this is an inline set or modifier it’s already been validated; we only need
|
|
42
|
-
// to resolve & validate $refs here which haven’t yet been parsed
|
|
43
|
-
let item = layer as unknown as ResolverSetInline | ResolverModifierInline;
|
|
44
|
-
|
|
45
|
-
// 1. $ref
|
|
46
|
-
if (members.$ref) {
|
|
47
|
-
const entry = { group: 'parser', label: 'init', node: members.$ref, src } as const;
|
|
48
|
-
const { url, subpath } = parseRef((members.$ref as unknown as momoa.StringNode).value);
|
|
49
|
-
if (url === '.') {
|
|
50
|
-
// 1a. local $ref: pull from local document
|
|
51
|
-
if (!subpath?.[0]) {
|
|
52
|
-
logger.error({ ...entry, message: '$ref can’t refer to the root document.' });
|
|
53
|
-
} else if (subpath[0] !== 'sets' && subpath[0] !== 'modifiers') {
|
|
54
|
-
// Note: technically we could allow $defs, but that’s just unnecessary shenanigans here.
|
|
55
|
-
logger.error({
|
|
56
|
-
...entry,
|
|
57
|
-
message: 'Local $ref in resolutionOrder must point to either #/sets/[set] or #/modifiers/[modifiers].',
|
|
58
|
-
});
|
|
59
|
-
} else {
|
|
60
|
-
const resolvedItem = resolverSource[subpath[0] as 'sets' | 'modifiers']?.[subpath[1]!];
|
|
61
|
-
if (!resolvedItem) {
|
|
62
|
-
logger.error({ ...entry, message: 'Invalid $ref' });
|
|
63
|
-
} else {
|
|
64
|
-
item = {
|
|
65
|
-
type: subpath[0] === 'sets' ? 'set' : 'modifier',
|
|
66
|
-
name: subpath[1],
|
|
67
|
-
...(resolvedItem as any), // Note: as long as this exists, this has already been validated to be correct
|
|
68
|
-
};
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
} else {
|
|
72
|
-
// 1b. remote $ref: load and validate
|
|
73
|
-
const result = await bundle(
|
|
74
|
-
[{ filename: new URL(url, filename), src: resolverSource.resolutionOrder[i]! }],
|
|
75
|
-
{
|
|
76
|
-
req,
|
|
77
|
-
yamlToMomoa,
|
|
78
|
-
},
|
|
79
|
-
);
|
|
80
|
-
if (result.document.body.type === 'Object') {
|
|
81
|
-
const type = getObjMember(result.document.body, 'type');
|
|
82
|
-
if (type?.type === 'String' && type.value === 'set') {
|
|
83
|
-
validateSet(result.document.body as momoa.ObjectNode, true, src);
|
|
84
|
-
item = momoa.evaluate(result.document.body) as unknown as ResolverSetInline;
|
|
85
|
-
} else if (type?.type === 'String' && type.value === 'modifier') {
|
|
86
|
-
validateModifier(result.document.body as momoa.ObjectNode, true, src);
|
|
87
|
-
item = momoa.evaluate(result.document.body) as unknown as ResolverModifierInline;
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
logger.error({ ...entry, message: '$ref did not resolve to a valid Set or Modifier.' });
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
// 2. resolve inline sources & contexts
|
|
95
|
-
const finalResult = await bundle([{ filename, src: item }], { req, yamlToMomoa });
|
|
96
|
-
return momoa.evaluate(finalResult.document.body) as unknown as
|
|
97
|
-
| ResolverSetNormalized
|
|
98
|
-
| ResolverModifierNormalized;
|
|
99
|
-
}),
|
|
100
|
-
),
|
|
64
|
+
resolutionOrder: resolverSource.resolutionOrder,
|
|
101
65
|
_source: {
|
|
102
66
|
filename,
|
|
103
|
-
|
|
67
|
+
document,
|
|
104
68
|
},
|
|
105
69
|
};
|
|
106
70
|
}
|
|
71
|
+
|
|
72
|
+
/** Resolve $refs for already-initialized JS */
|
|
73
|
+
function resolvePartials(
|
|
74
|
+
source: Group | ReferenceObject,
|
|
75
|
+
{
|
|
76
|
+
resolver,
|
|
77
|
+
logger,
|
|
78
|
+
}: {
|
|
79
|
+
resolver: any;
|
|
80
|
+
logger: Logger;
|
|
81
|
+
},
|
|
82
|
+
) {
|
|
83
|
+
if (!source) {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
const entry = { group: 'parser' as const, label: 'resolver' };
|
|
87
|
+
if (Array.isArray(source)) {
|
|
88
|
+
for (const item of source) {
|
|
89
|
+
resolvePartials(item, { resolver, logger });
|
|
90
|
+
}
|
|
91
|
+
} else if (typeof source === 'object') {
|
|
92
|
+
for (const k of Object.keys(source)) {
|
|
93
|
+
if (k === '$ref') {
|
|
94
|
+
const $ref = (source as any)[k] as string;
|
|
95
|
+
const { url, subpath = [] } = parseRef($ref);
|
|
96
|
+
if (url !== '.' || !subpath.length) {
|
|
97
|
+
logger.error({ ...entry, message: `Could not load $ref ${JSON.stringify($ref)}` });
|
|
98
|
+
}
|
|
99
|
+
const found = findObject(resolver, subpath ?? [], logger);
|
|
100
|
+
if (subpath[0] === 'sets' || subpath[0] === 'modifiers') {
|
|
101
|
+
found.type = subpath[0].replace(/s$/, '');
|
|
102
|
+
found.name = subpath[1];
|
|
103
|
+
}
|
|
104
|
+
if (found) {
|
|
105
|
+
for (const k2 of Object.keys(found)) {
|
|
106
|
+
(source as any)[k2] = found[k2];
|
|
107
|
+
}
|
|
108
|
+
delete (source as any).$ref;
|
|
109
|
+
} else {
|
|
110
|
+
logger.error({ ...entry, message: `Could not find ${JSON.stringify($ref)}` });
|
|
111
|
+
}
|
|
112
|
+
} else {
|
|
113
|
+
resolvePartials((source as any)[k], { resolver, logger });
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function findObject(dict: Record<string, any>, path: string[], logger: Logger): any {
|
|
120
|
+
let node = dict;
|
|
121
|
+
for (const idRaw of path) {
|
|
122
|
+
const id = idRaw.replace(/~/g, '~0').replace(/\//g, '~1');
|
|
123
|
+
if (!(id in node)) {
|
|
124
|
+
logger.error({
|
|
125
|
+
group: 'parser',
|
|
126
|
+
label: 'resolver',
|
|
127
|
+
message: `Could not load $ref ${encodeFragment(path)}`,
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
node = node[id];
|
|
131
|
+
}
|
|
132
|
+
return node;
|
|
133
|
+
}
|
package/src/types.ts
CHANGED
|
@@ -377,7 +377,7 @@ export interface ResolverSourceNormalized {
|
|
|
377
377
|
resolutionOrder: (ResolverSetNormalized | ResolverModifierNormalized)[];
|
|
378
378
|
_source: {
|
|
379
379
|
filename?: URL;
|
|
380
|
-
|
|
380
|
+
document: momoa.DocumentNode;
|
|
381
381
|
};
|
|
382
382
|
}
|
|
383
383
|
|
|
@@ -459,3 +459,10 @@ export interface TransformHookOptions {
|
|
|
459
459
|
/** Momoa documents */
|
|
460
460
|
sources: InputSourceWithDocument[];
|
|
461
461
|
}
|
|
462
|
+
|
|
463
|
+
export interface RefMapEntry {
|
|
464
|
+
filename: string;
|
|
465
|
+
refChain: string[];
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
export type RefMap = Record<string, RefMapEntry>;
|