@veda-hq/pariksha 0.5.8 → 0.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +187 -2
- package/dist/cli.js +109 -6
- package/dist/codec-registry.js +360 -0
- package/dist/fault-injection-registry.js +203 -0
- package/dist/git.js +40 -0
- package/dist/module-exports.js +169 -0
- package/dist/registry-ratchet.js +402 -0
- package/package.json +2 -2
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
// codec-registry: every module that hand-parses a format, and every exported
|
|
2
|
+
// writer/reader pair, has a suite that holds it to a reference.
|
|
3
|
+
//
|
|
4
|
+
// A hand parser (a module that reads markdown, HTML, YAML, CSV, shell or
|
|
5
|
+
// TypeScript source by its own regexes) registers with a property suite that
|
|
6
|
+
// compares it with the format's reference parser, over both line endings. A
|
|
7
|
+
// producer/consumer pair (`writeX`/`readX`, `renderX`/`parseX`…) registers with
|
|
8
|
+
// a suite that feeds real producer output through the real consumer; a
|
|
9
|
+
// schema-versioned digest registers with a golden. Discovery reads tracked
|
|
10
|
+
// source as a syntax tree; whatever it finds is registered, exempt with a
|
|
11
|
+
// reason, or in the baseline, and the baseline only shrinks.
|
|
12
|
+
import { readFileSync } from 'node:fs';
|
|
13
|
+
import { join, matchesGlob } from 'node:path';
|
|
14
|
+
import { audit, checkExamples, checkFindingIds, formatOutcome, importSpecifiers, importsModule, importsPackage, isCollected, isFile, list, minNoteLength, namesModule, note, object, onlyKeys, optionalText, parseBaseline as parseLists, parseExamples, parseFindingIds, parseJson, path, paths, readText, reason, records, regexes, required, text, } from "./registry-ratchet.js";
|
|
15
|
+
export const FORMATS = ['markdown', 'html', 'yaml', 'csv', 'shell', 'ts'];
|
|
16
|
+
const PAIR_ID = /^[^#]+#(?:[A-Za-z_$][\w$]*->[A-Za-z_$][\w$]*|digest)$/;
|
|
17
|
+
/**
|
|
18
|
+
* The file an id names. An id is `path#format`, `path#producer->consumer` or
|
|
19
|
+
* `path#digest`, and no suffix holds a `#`, so the path is everything before
|
|
20
|
+
* the last one: a path may itself hold a `#`.
|
|
21
|
+
*/
|
|
22
|
+
export const fileOf = (id) => {
|
|
23
|
+
const hash = id.lastIndexOf('#');
|
|
24
|
+
return hash === -1 ? id : id.slice(0, hash);
|
|
25
|
+
};
|
|
26
|
+
const isFormat = (f) => typeof f === 'string' && FORMATS.includes(f);
|
|
27
|
+
/** The config a JSON text names; throws naming the first bad key. */
|
|
28
|
+
export const parseConfig = (source) => {
|
|
29
|
+
const raw = parseJson(source);
|
|
30
|
+
onlyKeys(raw, ['baseline', 'roots', 'oracles', 'collected', 'surfaces', 'library', 'markers', 'handParsers', 'pairs', 'exempt', 'minNoteLength', 'examples', 'findingIds'], '');
|
|
31
|
+
const skipped = [];
|
|
32
|
+
const min = minNoteLength(raw);
|
|
33
|
+
const oraclesRaw = object(raw, 'oracles', '');
|
|
34
|
+
const oracles = {};
|
|
35
|
+
for (const [format, module] of Object.entries(oraclesRaw)) {
|
|
36
|
+
if (!isFormat(format))
|
|
37
|
+
throw new Error(`oracles: "${format}" is not a format discovery reads (${FORMATS.join(', ')})`);
|
|
38
|
+
if (typeof module !== 'string' || module === '')
|
|
39
|
+
throw new Error(`oracles.${format} must be a module name`);
|
|
40
|
+
oracles[format] = module;
|
|
41
|
+
}
|
|
42
|
+
if (Object.keys(oracles).length === 0)
|
|
43
|
+
throw new Error('"oracles" is empty, so no format would be discovered');
|
|
44
|
+
const surfaces = required(raw, 'surfaces', '', skipped, () => {
|
|
45
|
+
const globs = list(raw, 'surfaces', '');
|
|
46
|
+
if (globs.length === 0)
|
|
47
|
+
throw new Error('"surfaces" is empty; name the globs a property suite must sit on, or null to check no surface');
|
|
48
|
+
return globs.map((g, i) => {
|
|
49
|
+
if (typeof g !== 'string' || g === '')
|
|
50
|
+
throw new Error(`surfaces[${i}] must be a non-empty glob`);
|
|
51
|
+
return g;
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
const markersRaw = object(raw, 'markers', '');
|
|
55
|
+
onlyKeys(markersRaw, ['seed', 'lineEndings', 'golden'], 'markers: ');
|
|
56
|
+
const markers = { seed: text(markersRaw, 'seed', 'markers: '), lineEndings: text(markersRaw, 'lineEndings', 'markers: '), golden: text(markersRaw, 'golden', 'markers: ') };
|
|
57
|
+
const handParsers = records(list(raw, 'handParsers', ''), 'handParsers').map(({ raw: e, where }) => {
|
|
58
|
+
onlyKeys(e, ['source', 'formats', 'suite', 'outOfModel', 'lfOnly', 'motivatedBy'], where);
|
|
59
|
+
const formats = e['formats'];
|
|
60
|
+
if (!Array.isArray(formats) || formats.length === 0 || !formats.every(isFormat))
|
|
61
|
+
throw new Error(`${where}"formats" must be a non-empty list of ${FORMATS.join(', ')}`);
|
|
62
|
+
const unheld = formats.find((f) => oracles[f] === undefined);
|
|
63
|
+
if (unheld !== undefined)
|
|
64
|
+
throw new Error(`${where}"${unheld}" has no entry in "oracles", so its suite has no reference to import`);
|
|
65
|
+
return {
|
|
66
|
+
source: path(e, 'source', where),
|
|
67
|
+
formats,
|
|
68
|
+
suite: path(e, 'suite', where),
|
|
69
|
+
outOfModel: note(e, 'outOfModel', where, min, true) ?? '',
|
|
70
|
+
lfOnly: note(e, 'lfOnly', where, min, false),
|
|
71
|
+
motivatedBy: optionalText(e, 'motivatedBy', where),
|
|
72
|
+
};
|
|
73
|
+
});
|
|
74
|
+
const pairs = records(list(raw, 'pairs', ''), 'pairs').map(({ raw: e, where }) => {
|
|
75
|
+
onlyKeys(e, ['id', 'kind', 'suite', 'motivatedBy'], where);
|
|
76
|
+
const id = text(e, 'id', where);
|
|
77
|
+
if (!PAIR_ID.test(id))
|
|
78
|
+
throw new Error(`${where}"${id}" is not path#producer->consumer or path#digest`);
|
|
79
|
+
const kind = e['kind'];
|
|
80
|
+
if (kind !== 'round-trip' && kind !== 'golden')
|
|
81
|
+
throw new Error(`${where}"kind" must be "round-trip" or "golden"`);
|
|
82
|
+
// The id's shape names the kind, so a pair cannot skip the checks its
|
|
83
|
+
// kind needs: a producer->consumer pair round-trips under the property
|
|
84
|
+
// library and a seed; a digest pins a golden.
|
|
85
|
+
const shaped = id.endsWith('#digest') ? 'golden' : 'round-trip';
|
|
86
|
+
if (kind !== shaped)
|
|
87
|
+
throw new Error(`${where}"kind" is "${kind}", but "${id}" is ${shaped === 'golden' ? 'a digest, which is "golden"' : 'a producer->consumer pair, which is "round-trip"'}`);
|
|
88
|
+
return { id, kind, suite: path(e, 'suite', where), motivatedBy: optionalText(e, 'motivatedBy', where) };
|
|
89
|
+
});
|
|
90
|
+
const exempt = records(list(raw, 'exempt', ''), 'exempt').map(({ raw: e, where }) => {
|
|
91
|
+
onlyKeys(e, ['id', 'reason'], where);
|
|
92
|
+
const id = text(e, 'id', where);
|
|
93
|
+
if (!id.includes('#'))
|
|
94
|
+
throw new Error(`${where}"${id}" is not path#format or a pair id`);
|
|
95
|
+
return { id, reason: reason(e, where, min) };
|
|
96
|
+
});
|
|
97
|
+
const ids = [...handParsers.flatMap((h) => h.formats.map((f) => `${h.source}#${f}`)), ...pairs.map((p) => p.id), ...exempt.map((e) => e.id)];
|
|
98
|
+
const twice = ids.find((id, i) => ids.indexOf(id) !== i);
|
|
99
|
+
if (twice !== undefined)
|
|
100
|
+
throw new Error(`"${twice}" is registered or exempt more than once`);
|
|
101
|
+
const examples = parseExamples(raw, skipped);
|
|
102
|
+
const findingIds = parseFindingIds(raw, skipped);
|
|
103
|
+
checkFindingIds(findingIds.motivatedBy, [
|
|
104
|
+
...handParsers.map((h) => [`handParsers ${h.source}: motivatedBy`, h.motivatedBy]),
|
|
105
|
+
...pairs.map((p) => [`pairs ${p.id}: motivatedBy`, p.motivatedBy]),
|
|
106
|
+
]);
|
|
107
|
+
checkFindingIds(findingIds.example, Object.entries(examples?.fixtures ?? {}).map(([id, f]) => [`examples.fixtures.${id}: example`, f.example]));
|
|
108
|
+
return {
|
|
109
|
+
baseline: path(raw, 'baseline', ''),
|
|
110
|
+
roots: paths(raw, 'roots', ''),
|
|
111
|
+
oracles,
|
|
112
|
+
collected: regexes(raw, 'collected', ''),
|
|
113
|
+
surfaces,
|
|
114
|
+
library: raw['library'] === undefined ? 'fast-check' : text(raw, 'library', ''),
|
|
115
|
+
markers,
|
|
116
|
+
handParsers,
|
|
117
|
+
pairs,
|
|
118
|
+
exempt,
|
|
119
|
+
minNoteLength: min,
|
|
120
|
+
examples,
|
|
121
|
+
findingIds,
|
|
122
|
+
skipped,
|
|
123
|
+
};
|
|
124
|
+
};
|
|
125
|
+
export const LISTS = ['handParsers', 'pairs'];
|
|
126
|
+
/** The baseline a JSON text holds: `handParsers` and `pairs`, lists of ids (at the merge base, a missing list is empty). */
|
|
127
|
+
export const parseBaseline = (source, where, atBase = false) => parseLists(source, LISTS, where, 'ids', atBase);
|
|
128
|
+
// ── discovery ───────────────────────────────────────────────────────────────
|
|
129
|
+
const SOURCE = /\.(?:ts|tsx|mts|cts|js|mjs|cjs)$/;
|
|
130
|
+
const NOT_SOURCE = /\.(?:test|spec)\.|\.d\.ts$|(?:^|\/)(?:__tests__|__mocks__|__fixtures__|fixtures?)\/|\.fixture\./;
|
|
131
|
+
/** Regex-literal bodies that read a format's syntax. */
|
|
132
|
+
const REGEX_SIGNATURES = [
|
|
133
|
+
// ATX headings anchored at line start (CommonMark's 0-3 space indent
|
|
134
|
+
// included), code fences, inline link syntax.
|
|
135
|
+
['markdown', /\^(?:\\s\*|\(\?:\\s\*\)\?| \{0,3\})?#|```|`\{3|~~~|\\\]\\\(/],
|
|
136
|
+
// A frontmatter delimiter.
|
|
137
|
+
['yaml', /\^---|---\\r|---\\n|---\$/],
|
|
138
|
+
// A tag, closing tag, comment or `<script>` block; an opening tag by name
|
|
139
|
+
// (`<div[^>]*>`) too, but not a named group or its backreference
|
|
140
|
+
// (`(?<name>`, `\k<name>`).
|
|
141
|
+
['html', /<script|<\\\/|<\/[a-z]|<\[\^>|<!--|<\[a-z|<\[A-Za-z|(?<!\(\?|\\k)<[a-z]/i],
|
|
142
|
+
// Import/export/require statements in source text.
|
|
143
|
+
['ts', /\bimport\\s|\\bimport\\b|\bexport\\s|\brequire\\\(|from\\s\*\[/],
|
|
144
|
+
];
|
|
145
|
+
/** String literals that delimit a format by themselves. */
|
|
146
|
+
const STRING_SIGNATURES = new Map([
|
|
147
|
+
['---', 'yaml'],
|
|
148
|
+
['```', 'markdown'],
|
|
149
|
+
['~~~', 'markdown'],
|
|
150
|
+
['<script', 'html'],
|
|
151
|
+
]);
|
|
152
|
+
const CSV_TOKEN = /\bcsv\b/i;
|
|
153
|
+
const SHELL_TOKEN = /\b(?:shell|bash|argv|tokeni[sz]e)\b/i;
|
|
154
|
+
/** What one module's source shows: the formats it reads by hand, its exported functions, a versioned hash. */
|
|
155
|
+
export const scanSource = (ts, file, source) => {
|
|
156
|
+
const tree = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true);
|
|
157
|
+
const formats = new Set();
|
|
158
|
+
const exported = [];
|
|
159
|
+
const quoteChars = new Set();
|
|
160
|
+
let createsHash = false;
|
|
161
|
+
let schemaMarked = false;
|
|
162
|
+
/** Functions declared in the module, exported or not, by local name. */
|
|
163
|
+
const functions = new Set();
|
|
164
|
+
/** `export { local as name }` entries, as [local, name]. */
|
|
165
|
+
const listed = [];
|
|
166
|
+
const isExported = (node) => ts.canHaveModifiers(node) && (ts.getModifiers(node) ?? []).some((m) => m.kind === ts.SyntaxKind.ExportKeyword);
|
|
167
|
+
const visit = (node) => {
|
|
168
|
+
if (ts.isRegularExpressionLiteral(node)) {
|
|
169
|
+
const body = node.text.slice(1, node.text.lastIndexOf('/'));
|
|
170
|
+
for (const [format, signature] of REGEX_SIGNATURES)
|
|
171
|
+
if (signature.test(body))
|
|
172
|
+
formats.add(format);
|
|
173
|
+
// A tokenizer written as ONE regex, naming both quotes and the backslash
|
|
174
|
+
// it escapes (`\\` in its source), tracks them as a hand-written one
|
|
175
|
+
// does. Quotes spread across unrelated regexes are no tokenizer.
|
|
176
|
+
const named = new Set(Array.from(body.matchAll(/['"]|\\(?=\\)/g), ([quote]) => quote));
|
|
177
|
+
if (named.size === 3)
|
|
178
|
+
for (const quote of named)
|
|
179
|
+
quoteChars.add(quote);
|
|
180
|
+
}
|
|
181
|
+
else if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) {
|
|
182
|
+
const format = STRING_SIGNATURES.get(node.text);
|
|
183
|
+
if (format !== undefined)
|
|
184
|
+
formats.add(format);
|
|
185
|
+
if (node.text === "'" || node.text === '"' || node.text === '\\')
|
|
186
|
+
quoteChars.add(node.text);
|
|
187
|
+
if (node.text === 'schema' && ts.isCallExpression(node.parent))
|
|
188
|
+
schemaMarked = true;
|
|
189
|
+
}
|
|
190
|
+
else if (ts.isCallExpression(node)) {
|
|
191
|
+
const callee = node.expression;
|
|
192
|
+
const first = node.arguments[0];
|
|
193
|
+
if (ts.isPropertyAccessExpression(callee) && callee.name.text === 'split' && first !== undefined && ts.isStringLiteral(first) && first.text === ',' && CSV_TOKEN.test(source))
|
|
194
|
+
formats.add('csv');
|
|
195
|
+
if (ts.isIdentifier(callee) && callee.text === 'createHash')
|
|
196
|
+
createsHash = true;
|
|
197
|
+
}
|
|
198
|
+
else if (ts.isFunctionDeclaration(node) && node.name) {
|
|
199
|
+
functions.add(node.name.text);
|
|
200
|
+
if (isExported(node))
|
|
201
|
+
exported.push(node.name.text);
|
|
202
|
+
}
|
|
203
|
+
else if (ts.isExportDeclaration(node) && !node.isTypeOnly && node.moduleSpecifier === undefined && node.exportClause !== undefined && ts.isNamedExports(node.exportClause)) {
|
|
204
|
+
// `export { writeEntry, readEntry as read }`: read once every binding is known, below.
|
|
205
|
+
// `export { type writeEntry }` exports no value, so it makes no pair.
|
|
206
|
+
for (const element of node.exportClause.elements)
|
|
207
|
+
if (!element.isTypeOnly)
|
|
208
|
+
listed.push([(element.propertyName ?? element.name).text, element.name.text]);
|
|
209
|
+
}
|
|
210
|
+
else if (ts.isVariableStatement(node)) {
|
|
211
|
+
for (const declaration of node.declarationList.declarations) {
|
|
212
|
+
if (!ts.isIdentifier(declaration.name))
|
|
213
|
+
continue;
|
|
214
|
+
const init = declaration.initializer;
|
|
215
|
+
const isFunction = init !== undefined && (ts.isArrowFunction(init) || ts.isFunctionExpression(init));
|
|
216
|
+
if (isFunction)
|
|
217
|
+
functions.add(declaration.name.text);
|
|
218
|
+
if (!isExported(node))
|
|
219
|
+
continue;
|
|
220
|
+
if (isFunction)
|
|
221
|
+
exported.push(declaration.name.text);
|
|
222
|
+
if (/SCHEMA/.test(declaration.name.text) && init !== undefined && ts.isNumericLiteral(init))
|
|
223
|
+
schemaMarked = true;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
ts.forEachChild(node, visit);
|
|
227
|
+
};
|
|
228
|
+
visit(tree);
|
|
229
|
+
// An export list names functions as exports do, under their exported
|
|
230
|
+
// names; a listed constant or class is not a function.
|
|
231
|
+
for (const [local, name] of listed)
|
|
232
|
+
if (functions.has(local))
|
|
233
|
+
exported.push(name);
|
|
234
|
+
// A tokenizer that tracks both quote kinds and backslashes, in a module
|
|
235
|
+
// about shell commands, is a shell parser.
|
|
236
|
+
if (quoteChars.size === 3 && SHELL_TOKEN.test(source))
|
|
237
|
+
formats.add('shell');
|
|
238
|
+
return { formats, exported, createsHash, schemaMarked };
|
|
239
|
+
};
|
|
240
|
+
const PRODUCER = /^(?:build|write|render|format|serialize|encode|stringify|emit)([A-Z]\w*)$/;
|
|
241
|
+
const CONSUMER = /^(?:parse|read|decode|load)([A-Z]\w*)$/;
|
|
242
|
+
/** Same-module producer/consumer pairs among exported names. */
|
|
243
|
+
export const pairsIn = (file, exported) => {
|
|
244
|
+
const consumers = new Map();
|
|
245
|
+
for (const name of exported) {
|
|
246
|
+
const stem = CONSUMER.exec(name)?.[1];
|
|
247
|
+
if (stem !== undefined)
|
|
248
|
+
consumers.set(stem, [...(consumers.get(stem) ?? []), name]);
|
|
249
|
+
}
|
|
250
|
+
const out = [];
|
|
251
|
+
for (const name of exported) {
|
|
252
|
+
const stem = PRODUCER.exec(name)?.[1];
|
|
253
|
+
if (stem === undefined)
|
|
254
|
+
continue;
|
|
255
|
+
for (const consumer of consumers.get(stem) ?? [])
|
|
256
|
+
out.push(`${file}#${name}->${consumer}`);
|
|
257
|
+
}
|
|
258
|
+
return out;
|
|
259
|
+
};
|
|
260
|
+
/** Hand parsers and producer/consumer pairs among `files` (tracked, root-relative), in the `formats` given. */
|
|
261
|
+
export const discover = (ts, root, files, formats) => {
|
|
262
|
+
const handParsers = [];
|
|
263
|
+
const pairs = [];
|
|
264
|
+
for (const file of files) {
|
|
265
|
+
if (!SOURCE.test(file) || NOT_SOURCE.test(file))
|
|
266
|
+
continue;
|
|
267
|
+
const scan = scanSource(ts, file, readFileSync(join(root, file), 'utf8'));
|
|
268
|
+
for (const format of [...scan.formats].sort())
|
|
269
|
+
if (formats.has(format))
|
|
270
|
+
handParsers.push(`${file}#${format}`);
|
|
271
|
+
pairs.push(...pairsIn(file, scan.exported));
|
|
272
|
+
if (scan.createsHash && scan.schemaMarked)
|
|
273
|
+
pairs.push(`${file}#digest`);
|
|
274
|
+
}
|
|
275
|
+
return { handParsers: handParsers.sort(), pairs: pairs.sort() };
|
|
276
|
+
};
|
|
277
|
+
// ── accounting ──────────────────────────────────────────────────────────────
|
|
278
|
+
/** Discovered ids the registered and exempt entries cover. */
|
|
279
|
+
export const covered = (config) => new Set([...config.handParsers.flatMap((e) => e.formats.map((f) => `${e.source}#${f}`)), ...config.pairs.map((p) => p.id), ...config.exempt.map((e) => e.id)]);
|
|
280
|
+
/** What the baseline must be: every discovered id nothing covers. */
|
|
281
|
+
export const expectedBaseline = (found, cover) => ({
|
|
282
|
+
handParsers: found.handParsers.filter((id) => !cover.has(id)),
|
|
283
|
+
pairs: found.pairs.filter((id) => !cover.has(id)),
|
|
284
|
+
});
|
|
285
|
+
/** Findings about the registered suites: each exists where a runner collects it and holds what its kind needs. */
|
|
286
|
+
export const checkWiring = (ts, root, config) => {
|
|
287
|
+
const out = [];
|
|
288
|
+
const surfaces = config.surfaces ?? [];
|
|
289
|
+
const onSurface = (suite) => config.surfaces === undefined || surfaces.some((g) => matchesGlob(suite, g));
|
|
290
|
+
const common = (what, source, suite) => {
|
|
291
|
+
if (!isFile(root, source))
|
|
292
|
+
out.push(`${what}: ${source} does not exist`);
|
|
293
|
+
if (!isCollected(config.collected, suite))
|
|
294
|
+
out.push(`${what}: no runner collects ${suite}`);
|
|
295
|
+
const body = readText(root, suite);
|
|
296
|
+
if (body === undefined) {
|
|
297
|
+
out.push(`${what}: the suite ${suite} does not exist`);
|
|
298
|
+
return undefined;
|
|
299
|
+
}
|
|
300
|
+
const imports = importSpecifiers(ts, suite, body);
|
|
301
|
+
if (!importsModule(suite, imports, source))
|
|
302
|
+
out.push(`${what}: ${suite} does not import ${source}`);
|
|
303
|
+
return { body, imports };
|
|
304
|
+
};
|
|
305
|
+
for (const entry of config.handParsers) {
|
|
306
|
+
const what = `hand parser ${entry.source}`;
|
|
307
|
+
if (!onSurface(entry.suite))
|
|
308
|
+
out.push(`${what}: ${entry.suite} is not on a property-suite surface (${surfaces.join(', ')})`);
|
|
309
|
+
const suite = common(what, entry.source, entry.suite);
|
|
310
|
+
if (suite === undefined)
|
|
311
|
+
continue;
|
|
312
|
+
const { body, imports } = suite;
|
|
313
|
+
if (!importsPackage(imports, config.library))
|
|
314
|
+
out.push(`${what}: ${entry.suite} does not import ${config.library}`);
|
|
315
|
+
for (const format of entry.formats) {
|
|
316
|
+
const oracle = config.oracles[format] ?? '';
|
|
317
|
+
if (!namesModule(body, oracle))
|
|
318
|
+
out.push(`${what}: ${entry.suite} does not import ${oracle}, the ${format} reference`);
|
|
319
|
+
}
|
|
320
|
+
if (!body.includes(config.markers.seed))
|
|
321
|
+
out.push(`${what}: ${entry.suite} runs no fixed seed (${config.markers.seed})`);
|
|
322
|
+
if (entry.lfOnly === undefined && !body.includes(config.markers.lineEndings))
|
|
323
|
+
out.push(`${what}: ${entry.suite} runs LF only; run CRLF too (${config.markers.lineEndings}) or give lfOnly a reason`);
|
|
324
|
+
}
|
|
325
|
+
for (const pair of config.pairs) {
|
|
326
|
+
const what = `pair ${pair.id}`;
|
|
327
|
+
const source = fileOf(pair.id);
|
|
328
|
+
if (pair.kind === 'round-trip' && !onSurface(pair.suite))
|
|
329
|
+
out.push(`${what}: ${pair.suite} is not on a property-suite surface (${surfaces.join(', ')})`);
|
|
330
|
+
const suite = common(what, source, pair.suite);
|
|
331
|
+
if (suite === undefined)
|
|
332
|
+
continue;
|
|
333
|
+
const { body, imports } = suite;
|
|
334
|
+
if (pair.kind === 'golden') {
|
|
335
|
+
if (!body.includes(config.markers.golden))
|
|
336
|
+
out.push(`${what}: ${pair.suite} pins no golden (${config.markers.golden})`);
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
if (!importsPackage(imports, config.library))
|
|
340
|
+
out.push(`${what}: ${pair.suite} does not import ${config.library}`);
|
|
341
|
+
if (!body.includes(config.markers.seed))
|
|
342
|
+
out.push(`${what}: ${pair.suite} runs no fixed seed (${config.markers.seed})`);
|
|
343
|
+
}
|
|
344
|
+
return out;
|
|
345
|
+
};
|
|
346
|
+
/** Every finding: wiring, examples, accounting against the baseline, and growth when `grown` is given. */
|
|
347
|
+
export const findings = (ts, root, config, found, baseline, grown) => {
|
|
348
|
+
const cover = covered(config);
|
|
349
|
+
const { unaccounted, stale } = audit(expectedBaseline(found, cover), baseline);
|
|
350
|
+
const discovered = new Set([...found.handParsers, ...found.pairs]);
|
|
351
|
+
return [
|
|
352
|
+
...checkWiring(ts, root, config),
|
|
353
|
+
...(config.examples === undefined ? [] : checkExamples(root, config.examples, config.collected)),
|
|
354
|
+
...[...cover].filter((id) => !discovered.has(id)).map((id) => `${id}: registered or exempt, but discovery does not find it; drop the entry, or fix what it names`),
|
|
355
|
+
...unaccounted.map((id) => `${id}: discovered and unaccounted; register a suite for it, or exempt it with a reason`),
|
|
356
|
+
...stale.map((id) => `${id}: in ${config.baseline}, but registered, exempt, gone or in the wrong list; remove it from the baseline`),
|
|
357
|
+
...grown.map((id) => `${id}: added to ${config.baseline} since the merge base; the baseline only shrinks, so register a suite instead`),
|
|
358
|
+
];
|
|
359
|
+
};
|
|
360
|
+
export const formatFindings = (list, found, baseline, notChecked, compared) => formatOutcome('pariksha codec-registry:', list, `${found.handParsers.length} hand parser(s) and ${found.pairs.length} pair(s) discovered, each registered, exempt or among the ${(baseline['handParsers']?.length ?? 0) + (baseline['pairs']?.length ?? 0)} baselined${compared ? '; the baseline does not grow' : ''}`, 'Register a suite for a discovered parser or pair, or exempt it with a reason; the baseline only shrinks.', notChecked);
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
// fault-injection-registry: every unit whose dependencies can fail runs under
|
|
2
|
+
// the repository's fault-injection harness, or says why it need not.
|
|
3
|
+
//
|
|
4
|
+
// A consumer is a unit (a CLI's entry, a service method) whose suite fails its
|
|
5
|
+
// dependencies call by call through the harness and asserts each declared
|
|
6
|
+
// check (an exit contract, containment of a best-effort step, convergence on a
|
|
7
|
+
// rerun) with zero violations. Discovery finds the units that owe one by their
|
|
8
|
+
// text: under each configured group's roots, a tracked source matching the
|
|
9
|
+
// group's marker (a CLI entry point, a best-effort step). Whatever it finds is
|
|
10
|
+
// registered, exempt with a reason, or in the baseline, and the baseline only
|
|
11
|
+
// shrinks.
|
|
12
|
+
import { readFileSync } from 'node:fs';
|
|
13
|
+
import { join } from 'node:path';
|
|
14
|
+
import { audit, checkExamples, checkFindingIds, dict, formatOutcome, importSpecifiers, importsModule, importsPackage, isCollected, isRecord, list, minNoteLength, note, object, onlyKeys, optionalText, parseBaseline as parseLists, parseExamples, parseFindingIds, parseJson, path, paths, readText, reason, records, regex, regexes, text, } from "./registry-ratchet.js";
|
|
15
|
+
import { moduleExports } from "./module-exports.js";
|
|
16
|
+
const IDENTIFIER = /^[A-Za-z_$][\w$]*$/;
|
|
17
|
+
const LIST_NAME = /^[A-Za-z_][\w-]*$/;
|
|
18
|
+
/** The config a JSON text names; throws naming the first bad key. */
|
|
19
|
+
export const parseConfig = (source) => {
|
|
20
|
+
const raw = parseJson(source);
|
|
21
|
+
onlyKeys(raw, ['baseline', 'discover', 'collected', 'harness', 'consumers', 'exempt', 'exemptIn', 'minNoteLength', 'examples', 'findingIds'], '');
|
|
22
|
+
const groupsRaw = object(raw, 'discover', '');
|
|
23
|
+
const discover = dict();
|
|
24
|
+
for (const [name, group] of Object.entries(groupsRaw)) {
|
|
25
|
+
const where = `discover.${name}: `;
|
|
26
|
+
if (!LIST_NAME.test(name))
|
|
27
|
+
throw new Error(`discover: "${name}" is not a baseline list name`);
|
|
28
|
+
if (!isRecord(group))
|
|
29
|
+
throw new Error(`discover.${name} must be an object`);
|
|
30
|
+
onlyKeys(group, ['roots', 'include', 'marker'], where);
|
|
31
|
+
const marker = object(group, 'marker', where);
|
|
32
|
+
onlyKeys(marker, ['pattern', 'flags'], `${where}marker: `);
|
|
33
|
+
const flags = marker['flags'] ?? '';
|
|
34
|
+
if (typeof flags !== 'string' || !/^[imsu]*$/.test(flags))
|
|
35
|
+
throw new Error(`${where}marker.flags must be some of i, m, s, u`);
|
|
36
|
+
discover[name] = { roots: paths(group, 'roots', where), include: regex(group['include'], `${where}"include"`), marker: regex(marker['pattern'], `${where}marker.pattern`, flags) };
|
|
37
|
+
}
|
|
38
|
+
if (Object.keys(discover).length === 0)
|
|
39
|
+
throw new Error('"discover" is empty, so nothing would be discovered');
|
|
40
|
+
const harnessRaw = object(raw, 'harness', '');
|
|
41
|
+
onlyKeys(harnessRaw, ['module', 'imports', 'checks', 'heldBy'], 'harness: ');
|
|
42
|
+
const imports = list(harnessRaw, 'imports', 'harness: ').map((s, i) => {
|
|
43
|
+
if (typeof s !== 'string' || s === '' || s.startsWith('.'))
|
|
44
|
+
throw new Error(`harness.imports[${i}] must be a bare module specifier`);
|
|
45
|
+
return s;
|
|
46
|
+
});
|
|
47
|
+
const checksRaw = object(harnessRaw, 'checks', 'harness: ');
|
|
48
|
+
const checks = dict();
|
|
49
|
+
for (const [id, fn] of Object.entries(checksRaw)) {
|
|
50
|
+
if (typeof fn !== 'string' || !IDENTIFIER.test(fn))
|
|
51
|
+
throw new Error(`harness.checks.${id} must be a function name`);
|
|
52
|
+
checks[id] = fn;
|
|
53
|
+
}
|
|
54
|
+
if (Object.keys(checks).length === 0)
|
|
55
|
+
throw new Error('harness: "checks" is empty, so no suite would assert anything');
|
|
56
|
+
const heldBy = text(harnessRaw, 'heldBy', 'harness: ');
|
|
57
|
+
if (!IDENTIFIER.test(heldBy))
|
|
58
|
+
throw new Error('harness: "heldBy" must be a function name');
|
|
59
|
+
const harness = { module: path(harnessRaw, 'module', 'harness: '), imports, checks, heldBy };
|
|
60
|
+
const skipped = [];
|
|
61
|
+
const min = minNoteLength(raw);
|
|
62
|
+
const consumers = records(list(raw, 'consumers', ''), 'consumers').map(({ raw: e, where }) => {
|
|
63
|
+
onlyKeys(e, ['source', 'unit', 'checks', 'test', 'motivatedBy', 'uncovered'], where);
|
|
64
|
+
const declared = e['checks'];
|
|
65
|
+
if (!Array.isArray(declared) || declared.length === 0)
|
|
66
|
+
throw new Error(`${where}"checks" must be a non-empty list of check ids`);
|
|
67
|
+
const unknown = declared.find((c) => typeof c !== 'string' || checks[c] === undefined);
|
|
68
|
+
if (unknown !== undefined)
|
|
69
|
+
throw new Error(`${where}${JSON.stringify(unknown)} is not a check in harness.checks`);
|
|
70
|
+
return {
|
|
71
|
+
source: path(e, 'source', where),
|
|
72
|
+
unit: text(e, 'unit', where),
|
|
73
|
+
checks: declared,
|
|
74
|
+
test: path(e, 'test', where),
|
|
75
|
+
motivatedBy: optionalText(e, 'motivatedBy', where),
|
|
76
|
+
uncovered: note(e, 'uncovered', where, min, false),
|
|
77
|
+
};
|
|
78
|
+
});
|
|
79
|
+
const exemptInRaw = raw['exemptIn'];
|
|
80
|
+
if (!Array.isArray(exemptInRaw) || exemptInRaw.length === 0)
|
|
81
|
+
throw new Error('"exemptIn" must name the discovery groups an exemption may sit in');
|
|
82
|
+
const exemptIn = exemptInRaw.map((g, i) => {
|
|
83
|
+
if (typeof g !== 'string' || discover[g] === undefined)
|
|
84
|
+
throw new Error(`exemptIn[${i}]: ${JSON.stringify(g)} is not a group in "discover"`);
|
|
85
|
+
return g;
|
|
86
|
+
});
|
|
87
|
+
const exempt = records(list(raw, 'exempt', ''), 'exempt').map(({ raw: e, where }) => {
|
|
88
|
+
onlyKeys(e, ['source', 'reason'], where);
|
|
89
|
+
return { source: path(e, 'source', where), reason: reason(e, where, min) };
|
|
90
|
+
});
|
|
91
|
+
const sources = [...consumers.map((c) => c.source), ...exempt.map((e) => e.source)];
|
|
92
|
+
const twice = sources.find((s, i) => sources.indexOf(s) !== i);
|
|
93
|
+
if (twice !== undefined)
|
|
94
|
+
throw new Error(`"${twice}" is registered or exempt more than once`);
|
|
95
|
+
const examples = parseExamples(raw, skipped);
|
|
96
|
+
const unchecked = Object.keys(examples?.fixtures ?? {}).find((id) => checks[id] === undefined);
|
|
97
|
+
if (unchecked !== undefined)
|
|
98
|
+
throw new Error(`examples.fixtures.${unchecked}: not a check in harness.checks`);
|
|
99
|
+
const findingIds = parseFindingIds(raw, skipped);
|
|
100
|
+
checkFindingIds(findingIds.motivatedBy, consumers.map((c) => [`consumers ${c.source}: motivatedBy`, c.motivatedBy]));
|
|
101
|
+
checkFindingIds(findingIds.example, Object.entries(examples?.fixtures ?? {}).map(([id, f]) => [`examples.fixtures.${id}: example`, f.example]));
|
|
102
|
+
return { baseline: path(raw, 'baseline', ''), discover, collected: regexes(raw, 'collected', ''), harness, consumers, exempt, exemptIn, minNoteLength: min, examples, findingIds, skipped };
|
|
103
|
+
};
|
|
104
|
+
/** The baseline a JSON text holds: one list of paths per discovery group (at the merge base, read against today's groups). */
|
|
105
|
+
export const parseBaseline = (source, where, config, atBase = false) => parseLists(source, Object.keys(config.discover), where, 'paths', atBase);
|
|
106
|
+
// ── discovery ───────────────────────────────────────────────────────────────
|
|
107
|
+
const NOT_SOURCE = /\.test\.|\.spec\.|\.d\.ts$|(^|\/)fixtures?\//;
|
|
108
|
+
/** The sources among `files` (tracked, root-relative, under the group's roots) that owe a suite. */
|
|
109
|
+
export const discoverGroup = (root, files, group) => files
|
|
110
|
+
.filter((file) => group.include.test(file) && !NOT_SOURCE.test(file))
|
|
111
|
+
.filter((file) => group.marker.test(readFileSync(join(root, file)).toString()))
|
|
112
|
+
.sort();
|
|
113
|
+
// ── accounting ──────────────────────────────────────────────────────────────
|
|
114
|
+
/** What the baseline must be now: everything discovered that no consumer or exemption covers. */
|
|
115
|
+
export const expectedBaseline = (found, config) => {
|
|
116
|
+
const covered = new Set([...config.consumers.map((c) => c.source), ...config.exempt.map((e) => e.source)]);
|
|
117
|
+
return Object.fromEntries(Object.entries(found).map(([name, sources]) => [name, sources.filter((s) => !covered.has(s))]));
|
|
118
|
+
};
|
|
119
|
+
/** The identifier a consumer's `unit` opens with, or '' when it opens with none. */
|
|
120
|
+
export const leadName = (unit) => /^[A-Za-z_$][\w$]*/.exec(unit)?.[0] ?? '';
|
|
121
|
+
const escape = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
122
|
+
/**
|
|
123
|
+
* Findings about the harness module: it exists and exports, as a function,
|
|
124
|
+
* every function the config names. Its syntax tree is read, re-exports
|
|
125
|
+
* followed, so a type of that name, a constant holding no function, or a
|
|
126
|
+
* re-export that leads nowhere does not count.
|
|
127
|
+
*/
|
|
128
|
+
export const checkHarness = (ts, root, config) => {
|
|
129
|
+
const { module, checks, heldBy } = config.harness;
|
|
130
|
+
if (readText(root, module) === undefined)
|
|
131
|
+
return [`harness: ${module} does not exist`];
|
|
132
|
+
const { exports, unfollowed } = moduleExports(ts, root, module);
|
|
133
|
+
const why = unfollowed.length === 0 ? '' : ` (not followed: ${unfollowed.join(', ')})`;
|
|
134
|
+
const out = [];
|
|
135
|
+
for (const fn of new Set([...Object.values(checks), heldBy])) {
|
|
136
|
+
const kind = exports.get(fn);
|
|
137
|
+
if (kind === undefined)
|
|
138
|
+
out.push(`harness: ${module} does not export ${fn}${why}`);
|
|
139
|
+
else if (kind === 'unresolved')
|
|
140
|
+
out.push(`harness: ${module} exports ${fn}, but it resolves to no declaration${why}`);
|
|
141
|
+
else if (kind !== 'function')
|
|
142
|
+
out.push(`harness: ${module} exports ${fn} as a ${kind}, not a function`);
|
|
143
|
+
}
|
|
144
|
+
return out;
|
|
145
|
+
};
|
|
146
|
+
/** Findings about the registered consumers: each suite runs every declared check through the harness. */
|
|
147
|
+
export const checkConsumers = (ts, root, config) => {
|
|
148
|
+
const out = [];
|
|
149
|
+
const { harness } = config;
|
|
150
|
+
for (const consumer of config.consumers) {
|
|
151
|
+
const what = `${consumer.source}`;
|
|
152
|
+
const source = readText(root, consumer.source);
|
|
153
|
+
if (source === undefined)
|
|
154
|
+
out.push(`${what}: the registered source does not exist`);
|
|
155
|
+
if (!isCollected(config.collected, consumer.test))
|
|
156
|
+
out.push(`${what}: no runner collects ${consumer.test}`);
|
|
157
|
+
const suite = readText(root, consumer.test);
|
|
158
|
+
if (suite === undefined) {
|
|
159
|
+
out.push(`${what}: the suite ${consumer.test} does not exist`);
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
const imports = importSpecifiers(ts, consumer.test, suite);
|
|
163
|
+
if (!importsModule(consumer.test, imports, harness.module) && !harness.imports.some((s) => importsPackage(imports, s)))
|
|
164
|
+
out.push(`${what}: ${consumer.test} does not import the harness (${[harness.module, ...harness.imports].join(' or ')})`);
|
|
165
|
+
if (!importsModule(consumer.test, imports, consumer.source))
|
|
166
|
+
out.push(`${what}: ${consumer.test} does not import ${consumer.source}`);
|
|
167
|
+
for (const check of consumer.checks) {
|
|
168
|
+
const fn = harness.checks[check] ?? '';
|
|
169
|
+
if (!suite.includes(`${fn}(`))
|
|
170
|
+
out.push(`${what}: ${consumer.test} declares ${check} but never calls ${fn}(`);
|
|
171
|
+
}
|
|
172
|
+
// Registered means held to zero violations, not merely explored.
|
|
173
|
+
if (!suite.includes(`${harness.heldBy}(`))
|
|
174
|
+
out.push(`${what}: ${consumer.test} never calls ${harness.heldBy}(, so no run is held to zero violations`);
|
|
175
|
+
const unit = leadName(consumer.unit);
|
|
176
|
+
if (unit === '')
|
|
177
|
+
out.push(`${what}: unit "${consumer.unit}" opens with no name`);
|
|
178
|
+
else if (source !== undefined && !new RegExp(String.raw `(?<![\w$])${escape(unit)}(?![\w$])`).test(source))
|
|
179
|
+
out.push(`${what}: unit ${unit} is not named in the source`);
|
|
180
|
+
}
|
|
181
|
+
return out;
|
|
182
|
+
};
|
|
183
|
+
/** Every finding: harness, consumers, examples, accounting against the baseline, and growth. */
|
|
184
|
+
export const findings = (ts, root, config, found, baseline, grown) => {
|
|
185
|
+
const { unaccounted, stale } = audit(expectedBaseline(found, config), baseline);
|
|
186
|
+
const exemptable = new Set(config.exemptIn.flatMap((g) => found[g] ?? []));
|
|
187
|
+
const callsCheck = (id, body) => {
|
|
188
|
+
const fn = config.harness.checks[id] ?? '';
|
|
189
|
+
return body.includes(`${fn}(`) ? [] : [`the ${id} example suite never calls ${fn}(`];
|
|
190
|
+
};
|
|
191
|
+
return [
|
|
192
|
+
...checkHarness(ts, root, config),
|
|
193
|
+
...checkConsumers(ts, root, config),
|
|
194
|
+
...(config.examples === undefined ? [] : checkExamples(root, config.examples, config.collected, callsCheck)),
|
|
195
|
+
...config.exempt.filter((e) => !exemptable.has(e.source)).map((e) => `${e.source}: exempt, but discovery does not find it in ${config.exemptIn.join(' or ')}; drop the exemption`),
|
|
196
|
+
...unaccounted.map((s) => `${s}: discovered and unaccounted; register it with a fault-injection suite, or exempt it with a reason`),
|
|
197
|
+
...stale.map((s) => `${s}: in ${config.baseline}, but registered, exempt, renamed, gone or in the wrong list; remove it from the baseline`),
|
|
198
|
+
...grown.map((s) => `${s}: added to ${config.baseline} since the merge base; the baseline only shrinks, so register a suite instead`),
|
|
199
|
+
];
|
|
200
|
+
};
|
|
201
|
+
export const formatFindings = (list, found, baseline, notChecked, compared) => formatOutcome('pariksha fault-injection-registry:', list, `${Object.entries(found)
|
|
202
|
+
.map(([name, s]) => `${s.length} ${name}`)
|
|
203
|
+
.join(' and ')} source(s) discovered, each registered, exempt or among the ${Object.values(baseline).flat().length} baselined${compared ? '; the baseline does not grow' : ''}`, 'Register a fault-injection suite for a discovered source, or exempt it with a reason; the baseline only shrinks.', notChecked);
|
package/dist/git.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// Git, as the checks read it: a function from arguments to stdout.
|
|
2
2
|
import { execFileSync } from 'node:child_process';
|
|
3
|
+
import { existsSync } from 'node:fs';
|
|
4
|
+
import { join } from 'node:path';
|
|
3
5
|
/**
|
|
4
6
|
* Settings that change what git prints, pinned so a user's config cannot:
|
|
5
7
|
* quoted non-ASCII paths would name no file, and a column field would hide a
|
|
@@ -51,6 +53,44 @@ export const must = (git, args) => {
|
|
|
51
53
|
throw new Error(`git ${args.join(' ')} failed`);
|
|
52
54
|
return out;
|
|
53
55
|
};
|
|
56
|
+
/** Tracked paths under `roots` (each a literal path: a folder or one file) that still exist in the working tree. */
|
|
57
|
+
export const trackedUnder = (git, root, roots) => must(git, ['ls-files', '-z', '--', ...roots.map((r) => `:(literal)${r}`)])
|
|
58
|
+
.split('\0')
|
|
59
|
+
.filter((path) => path !== '' && existsSync(join(root, path)));
|
|
60
|
+
/** The merge base of HEAD and `base`; throws when git cannot name one. */
|
|
61
|
+
export const mergeBaseOf = (git, base) => must(git, ['merge-base', 'HEAD', base]).trim();
|
|
62
|
+
/**
|
|
63
|
+
* `path` as committed at `commit`, or null when the commit does not have it.
|
|
64
|
+
* `ls-tree` prints nothing for an absent path and fails on a real read
|
|
65
|
+
* failure, so only a confirmed absence is null.
|
|
66
|
+
*/
|
|
67
|
+
export const fileAt = (git, commit, path) => must(git, ['ls-tree', '--name-only', commit, '--', `:(literal)${path}`]) === '' ? null : must(git, ['show', `${commit}:${path}`]);
|
|
68
|
+
/**
|
|
69
|
+
* The renames in `git diff --name-status -z` output, new path to old path.
|
|
70
|
+
* NUL-separated, so a path arrives as committed. Each entry is a status then
|
|
71
|
+
* its paths: two for a rename (`R`) or copy (`C`), one otherwise. Only a
|
|
72
|
+
* rename counts: a copy leaves its source in place.
|
|
73
|
+
*/
|
|
74
|
+
export const parseRenames = (nameStatus) => {
|
|
75
|
+
const renamed = new Map();
|
|
76
|
+
const fields = nameStatus.split('\0');
|
|
77
|
+
// How many of the fields that follow are the current entry's paths.
|
|
78
|
+
let paths = 0;
|
|
79
|
+
for (const [index, status] of fields.entries()) {
|
|
80
|
+
if (paths > 0) {
|
|
81
|
+
paths -= 1;
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
paths = status.startsWith('R') || status.startsWith('C') ? 2 : 1;
|
|
85
|
+
const from = fields[index + 1];
|
|
86
|
+
const to = fields[index + 2];
|
|
87
|
+
if (status.startsWith('R') && from !== undefined && to !== undefined)
|
|
88
|
+
renamed.set(to, from);
|
|
89
|
+
}
|
|
90
|
+
return renamed;
|
|
91
|
+
};
|
|
92
|
+
/** Files renamed between `commit` and the working tree (staged files count), new path to old path, by git's own rename detection. */
|
|
93
|
+
export const renamesSince = (git, commit) => parseRenames(must(git, ['diff', '--name-status', '-z', '-M', '--no-ext-diff', commit]));
|
|
54
94
|
export const BASE_CANDIDATES = ['origin/main', 'origin/develop', 'origin/master', 'main'];
|
|
55
95
|
/** The first ref that names a commit, or undefined. */
|
|
56
96
|
export const resolveBase = (git, candidates = BASE_CANDIDATES) => candidates.find((ref) => git.run(['rev-parse', '--verify', '--quiet', `${ref}^{commit}`]) !== undefined);
|