@tobi-or-not/quorum 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,463 @@
1
+ /**
2
+ * The `quorum bench` subcommand: it reviews every case of a suite with every configuration, `--trials`
3
+ * times each, records each review in a run directory it can resume, and reports what each configuration
4
+ * caught. It ties the bench modules to the CLI: the flags, the plan, the progress lines and the exit codes.
5
+ */
6
+ import { resolve } from 'node:path';
7
+ import { parseArgs } from 'node:util';
8
+ import packageJson from '../../package.json' with { type: 'json' };
9
+ import { fingerprint, newSalt } from "../bench/fingerprint.js";
10
+ import { MAX_CONFIGURATIONS, reviewKey, RUN_FORMAT } from "../bench/records.js";
11
+ import { buildReport, reportJson, reportMarkdown, reviewScore } from "../bench/report.js";
12
+ import { createRun, inspectRunDir, openRecords, readRecords, REPORT_JSON, REPORT_MARKDOWN, runDifferences, writeAtomic } from "../bench/run-dir.js";
13
+ import { planReviews, runReviews } from "../bench/runner.js";
14
+ import { loadSuite, MAX_TRIALS, SUITE_ID } from "../bench/suite.js";
15
+ import { readEnv } from "../config/env.js";
16
+ import { fromConfig } from "../config/from-config.js";
17
+ import { loadConfig } from "../config/load.js";
18
+ import { ConfigError } from "../core/errors.js";
19
+ import { inertLine } from "../core/sanitize.js";
20
+ import { errorLines, errorMessage } from "./io.js";
21
+ import { profileProviders } from "./render.js";
22
+ export const BENCH_USAGE = [
23
+ 'Usage: quorum bench [options] <suite>',
24
+ '',
25
+ 'Review every case of a suite with every config, --trials times each, and report what each caught.',
26
+ '',
27
+ 'Options:',
28
+ ' -c, --config <path> A config file to compare; repeat it, and the first is the reference',
29
+ ' -p, --profile <id> Run only this profile; repeat it to run several',
30
+ ' --trials <n> Reviews of each case per config: the suite\'s trials, else 1',
31
+ ' --case <id> Run only this case; repeat it to run several',
32
+ ' -j, --jobs <n> The reviews run at once (default 1)',
33
+ ' -o, --out <dir> The run directory (default quorum-bench/<suite name>)',
34
+ ' --retry-incomplete Run again the recorded reviews that came back incomplete',
35
+ ' --dry-run Check the suite and the configs, print the plan, and exit',
36
+ ' -h, --help Print this help and exit',
37
+ '',
38
+ ].join('\n');
39
+ /**
40
+ * The bench's own exit codes (Decision 18), not the review's `EXIT`: `failed` means reviews failed and were
41
+ * not recorded, not that a review left a gap, and `stopped` means the run could not go on.
42
+ */
43
+ const BENCH_EXIT = {
44
+ ok: 0,
45
+ usage: 2,
46
+ stopped: 3,
47
+ failed: 4,
48
+ interrupted: 130,
49
+ };
50
+ const MAX_JOBS = 16;
51
+ const DEFAULT_CONFIGURATION = '(default configuration)';
52
+ const TORN_LINE = 'warning: results.jsonl: its last line was cut short, as a stopped run leaves it: that review runs again';
53
+ const OPTIONS = {
54
+ config: { type: 'string', short: 'c', multiple: true },
55
+ profile: { type: 'string', short: 'p', multiple: true },
56
+ trials: { type: 'string' },
57
+ case: { type: 'string', multiple: true },
58
+ jobs: { type: 'string', short: 'j' },
59
+ out: { type: 'string', short: 'o' },
60
+ 'retry-incomplete': { type: 'boolean' },
61
+ 'dry-run': { type: 'boolean' },
62
+ help: { type: 'boolean', short: 'h' },
63
+ };
64
+ /** The flags of `argv` and its positional arguments; a usage error of `parseArgs` is raised as a `ConfigError`, one issue per line of its message. */
65
+ function parseFlags(argv) {
66
+ try {
67
+ return parseArgs({
68
+ args: [...argv],
69
+ options: OPTIONS,
70
+ strict: true,
71
+ allowPositionals: true,
72
+ });
73
+ }
74
+ catch (error) {
75
+ if (!(error instanceof Error) || !error.code?.startsWith('ERR_PARSE_ARGS_'))
76
+ throw error;
77
+ throw new ConfigError(error.message.split('\n').map(message => ({ path: '', message })));
78
+ }
79
+ }
80
+ function usageIssue(path, message) {
81
+ return new ConfigError([
82
+ { path, message },
83
+ ]);
84
+ }
85
+ /** `text` as a whole number from 1 to `max`; undefined when the flag was not given. */
86
+ function wholeNumber(flag, text, max) {
87
+ if (text === undefined)
88
+ return undefined;
89
+ if (!/^[1-9]\d*$/.test(text) || Number(text) > max)
90
+ throw usageIssue(flag, `must be a whole number from 1 to ${max}`);
91
+ return Number(text);
92
+ }
93
+ /** The options the flags of `argv` give, or the usage error they hold. */
94
+ function checkOptions(values, positionals) {
95
+ const [suite, ...extra] = positionals;
96
+ if (suite === undefined)
97
+ throw new Error('no suite given');
98
+ if (extra.length > 0)
99
+ throw new Error(`unexpected arguments: ${extra.join(' ')}`);
100
+ const trials = wholeNumber('--trials', values.trials, MAX_TRIALS);
101
+ const jobs = wholeNumber('--jobs', values.jobs, MAX_JOBS);
102
+ const configs = values.config ?? [];
103
+ if (configs.includes(''))
104
+ throw usageIssue('--config', 'must name a file');
105
+ if (configs.length > MAX_CONFIGURATIONS)
106
+ throw usageIssue('--config', `names more than ${MAX_CONFIGURATIONS} configurations`);
107
+ if (values.out === '')
108
+ throw usageIssue('--out', 'must name a directory');
109
+ const cases = values.case ?? [];
110
+ if (cases.some(id => !SUITE_ID.test(id)))
111
+ throw usageIssue('--case', `must be a case id matching ${SUITE_ID.source}`);
112
+ if (new Set(cases).size < cases.length)
113
+ throw usageIssue('--case', 'names a case more than once');
114
+ return {
115
+ suite,
116
+ configs,
117
+ profiles: values.profile,
118
+ trials,
119
+ cases,
120
+ jobs: jobs ?? 1,
121
+ out: values.out,
122
+ retryIncomplete: values['retry-incomplete'] === true,
123
+ dryRun: values['dry-run'] === true,
124
+ };
125
+ }
126
+ /** Write `lines` to `stream`, each inert, one write per line. */
127
+ function writeLines(stream, lines) {
128
+ for (const line of lines)
129
+ stream.write(`${inertLine(line)}\n`);
130
+ }
131
+ function nameOf({ path }) {
132
+ return path ?? DEFAULT_CONFIGURATION;
133
+ }
134
+ function plural(count, noun) {
135
+ return `${count} ${noun}${count === 1
136
+ ? ''
137
+ : 's'}`;
138
+ }
139
+ /** The cases of `suite` the flags select, in the suite's order; every `--case` must be one of its cases. */
140
+ function selectCases(suite, ids) {
141
+ const unknown = ids.filter(id => !suite.cases.some(suiteCase => suiteCase.id === id));
142
+ if (unknown.length > 0)
143
+ throw new ConfigError(unknown.map(id => ({ path: '--case', message: `unknown case "${id}"` })));
144
+ return ids.length === 0
145
+ ? suite.cases
146
+ : suite.cases.filter(({ id }) => ids.includes(id));
147
+ }
148
+ /**
149
+ * Each configuration, in order: loaded with the run's `--profile` selection, then built once, as a review
150
+ * builds it, so one that cannot be built is refused before any call, and its warnings are known. With no
151
+ * `--config`, the usual resolution is the one configuration. The first that fails ends the run, after the
152
+ * environment's warnings, since a deprecated or mistyped variable is often what caused the error.
153
+ */
154
+ async function loadConfigurations(io, options) {
155
+ const selection = options.profiles === undefined
156
+ ? undefined
157
+ : { ids: options.profiles, path: '--profile' };
158
+ const paths = options.configs.length === 0
159
+ ? [undefined]
160
+ : options.configs;
161
+ const loaded = [];
162
+ try {
163
+ for (const path of paths) {
164
+ const { config } = await loadConfig({
165
+ cwd: io.cwd,
166
+ env: io.env,
167
+ configPath: path,
168
+ selection,
169
+ });
170
+ const quorum = await fromConfig(config, { env: io.env });
171
+ loaded.push({ path, config, warnings: quorum.warnings });
172
+ }
173
+ }
174
+ catch (error) {
175
+ if (error instanceof ConfigError)
176
+ writeLines(io.stderr, readEnv(io.env).warnings.map(warning => `warning: ${warning}`));
177
+ throw error;
178
+ }
179
+ return loaded;
180
+ }
181
+ /** One issue for each configuration whose fingerprint an earlier one already has. */
182
+ function duplicateIssues(configurations) {
183
+ const earliest = new Map();
184
+ const issues = [];
185
+ for (const configuration of configurations) {
186
+ const earlier = earliest.get(configuration.fingerprint);
187
+ if (earlier === undefined)
188
+ earliest.set(configuration.fingerprint, configuration);
189
+ else
190
+ issues.push({ path: '--config', message: `${nameOf(configuration)} resolves to the same settings as ${nameOf(earlier)}: a QUORUM_* variable overrides every configuration alike` });
191
+ }
192
+ return issues;
193
+ }
194
+ /** What a recorded review says of its case, as its progress line puts it. */
195
+ function verdictOf(suiteCase, score) {
196
+ if (suiteCase.kind === 'control')
197
+ return 'control';
198
+ if (score.keptCatch)
199
+ return 'caught by a kept finding';
200
+ return score.proposedCatch
201
+ ? 'caught by a proposed finding only'
202
+ : 'missed';
203
+ }
204
+ /** The progress line of a review that was recorded, after `[<n>/<total>] trial <t>, case <id>, configuration <i>: `. */
205
+ function recordedText(record, suiteCase, score) {
206
+ const { kept, discarded, unreviewed, meta } = record.result;
207
+ return `${verdictOf(suiteCase, score)}; ${kept.length} kept, ${discarded.length} discarded, ${unreviewed.length} unreviewed, ${meta.incomplete.length} gaps; ${(record.latencyMs / 1000).toFixed(1)} s`;
208
+ }
209
+ /** The plan a run prints before its first call. */
210
+ function planLines(plan) {
211
+ const { suite, selected, loaded, trials, planned, recorded, toRun, again, jobs, directory } = plan;
212
+ const bugs = selected.filter(({ kind }) => kind === 'bug').length;
213
+ return [
214
+ `Suite ${suite.name}: ${selected.length} of ${suite.cases.length} cases, ${plural(bugs, 'bug case')} and ${plural(selected.length - bugs, 'control')}`,
215
+ 'Configurations:',
216
+ ...loaded.flatMap(({ path, config }, index) => [
217
+ ` ${index + 1}. ${path ?? DEFAULT_CONFIGURATION}${loaded.length > 1 && index === 0
218
+ ? ' (reference)'
219
+ : ''}`,
220
+ ...profileProviders(config).map(({ profile, research, panel }) => ` ${profile}: research ${research}; panel ${panel}`),
221
+ ]),
222
+ `Trials: ${trials}`,
223
+ `Reviews: ${planned} planned, ${recorded} recorded, ${toRun} to run${again > 0
224
+ ? ` (${again} of them again)`
225
+ : ''}, at most ${jobs} at a time`,
226
+ `Run directory: ${directory}`,
227
+ ];
228
+ }
229
+ /**
230
+ * Check everything a run depends on before its first call, and write nothing: the suite, the cases, the
231
+ * configurations (loaded and built), the run directory, and that the run it holds, if any, is this one.
232
+ * The fingerprints are keyed with the salt of the run being resumed, else a new one.
233
+ */
234
+ async function check(io, options) {
235
+ const suite = await loadSuite(resolve(io.cwd, options.suite), options.suite);
236
+ const selected = selectCases(suite, options.cases);
237
+ const loaded = await loadConfigurations(io, options);
238
+ const directory = options.out ?? `quorum-bench/${suite.name}`;
239
+ const out = resolve(io.cwd, directory);
240
+ const state = await inspectRunDir(out);
241
+ const salt = state.kind === 'run'
242
+ ? state.run.salt
243
+ : newSalt();
244
+ const configurations = loaded.map(({ path, config }) => ({
245
+ path: path ?? null,
246
+ fingerprint: fingerprint(config, salt),
247
+ }));
248
+ const duplicates = duplicateIssues(configurations);
249
+ if (duplicates.length > 0)
250
+ throw new ConfigError(duplicates);
251
+ const runFile = {
252
+ format: RUN_FORMAT,
253
+ quorum: packageJson.version,
254
+ suite: { name: suite.name, hash: suite.hash },
255
+ salt,
256
+ configurations,
257
+ };
258
+ if (state.kind === 'run') {
259
+ const differences = runDifferences(state.run, runFile);
260
+ if (differences.length > 0)
261
+ throw new ConfigError(differences);
262
+ }
263
+ const byId = new Map(suite.cases.map(suiteCase => [suiteCase.id, suiteCase]));
264
+ return {
265
+ suite,
266
+ selected,
267
+ loaded,
268
+ byId,
269
+ directory,
270
+ out,
271
+ state,
272
+ runFile,
273
+ };
274
+ }
275
+ /** `record` scored against its case and kept in `scores` under its key, replacing an earlier record of the same review. */
276
+ function keepScore(scores, record, suiteCase, window) {
277
+ const scored = reviewScore(record, suiteCase, window);
278
+ scores.set(reviewKey(record.fingerprint, record.case, record.trial), scored);
279
+ return scored;
280
+ }
281
+ /**
282
+ * The score of every review the run being resumed holds, by `reviewKey`, and where its valid lines end
283
+ * (what `openRecords` cuts to). A line that names a configuration the run does not have, or a case the suite
284
+ * does not have, refuses the run directory. A last line a killed run cut short is not scored, and says so.
285
+ * A new run holds nothing.
286
+ */
287
+ async function readScores(io, { suite, byId, out, state, runFile }) {
288
+ const scores = new Map();
289
+ if (state.kind !== 'run')
290
+ return { scores, validBytes: 0 };
291
+ const known = new Set(runFile.configurations.map(({ fingerprint }) => fingerprint));
292
+ const read = await readRecords(out, (record) => {
293
+ const suiteCase = byId.get(record.case);
294
+ if (!known.has(record.fingerprint))
295
+ return 'names a configuration this run does not have';
296
+ if (suiteCase === undefined)
297
+ return 'names a case the suite does not have';
298
+ keepScore(scores, record, suiteCase, suite.window);
299
+ return undefined;
300
+ });
301
+ if (read.torn)
302
+ writeLines(io.stderr, [TORN_LINE]);
303
+ return { scores, validBytes: read.validBytes };
304
+ }
305
+ /**
306
+ * Make the run (when it is new), open `results.jsonl` and run the reviews of `toRun`, each recorded as it
307
+ * resolves and each reported with its progress line. The order matters: `createRun` renames over any
308
+ * run.json and `openRecords` cuts results.jsonl to the bytes it is given, so a run being resumed has been
309
+ * read before either, and only a new run is created. The writer is closed whatever happens; when the run
310
+ * rejects, with the error of a failed write, that error is the one thrown.
311
+ */
312
+ async function execute(io, options, checked, scores, validBytes, toRun) {
313
+ const { suite, loaded, byId, out, state, runFile } = checked;
314
+ if (state.kind !== 'run')
315
+ await createRun(out, runFile);
316
+ const writer = await openRecords(out, validBytes);
317
+ let finished = 0;
318
+ let outcome;
319
+ try {
320
+ outcome = await runReviews({
321
+ suite,
322
+ configurations: loaded.map(({ config }, index) => ({ config, fingerprint: runFile.configurations[index].fingerprint })),
323
+ reviews: toRun,
324
+ jobs: options.jobs,
325
+ env: io.env,
326
+ signal: io.signal,
327
+ append: writer.append,
328
+ events: (event) => {
329
+ finished += 1;
330
+ const { trial, caseId, configuration } = event.review;
331
+ const suiteCase = byId.get(caseId);
332
+ const text = event.type === 'recorded'
333
+ ? recordedText(event.record, suiteCase, keepScore(scores, event.record, suiteCase, suite.window))
334
+ : `failed: ${errorMessage(event.error)}`;
335
+ writeLines(io.stderr, [`[${finished}/${toRun.length}] trial ${trial}, case ${caseId}, configuration ${configuration + 1}: ${text}`]);
336
+ },
337
+ });
338
+ }
339
+ catch (error) {
340
+ await writer.close().catch(() => undefined);
341
+ throw error;
342
+ }
343
+ await writer.close();
344
+ return outcome;
345
+ }
346
+ /** Build the report from the scores, write `report.md` and `report.json` into the run directory, and print the Markdown. */
347
+ async function writeReports(io, { suite, selected, out, runFile }, trials, scores) {
348
+ const report = buildReport({
349
+ suite: { name: suite.name, window: suite.window },
350
+ quorum: packageJson.version,
351
+ cases: selected,
352
+ trials,
353
+ configurations: runFile.configurations,
354
+ scores,
355
+ });
356
+ const markdown = reportMarkdown(report);
357
+ await writeAtomic(out, REPORT_MARKDOWN, markdown);
358
+ await writeAtomic(out, REPORT_JSON, reportJson(report));
359
+ io.stdout.write(markdown);
360
+ }
361
+ /** One whole bench run: the checks, the plan, the reviews and the reports; what ends it early is thrown. */
362
+ async function bench(io, options) {
363
+ const checked = await check(io, options);
364
+ const { suite, selected, loaded, directory, runFile } = checked;
365
+ const { scores, validBytes } = await readScores(io, checked);
366
+ const trials = options.trials ?? suite.trials ?? 1;
367
+ const planned = planReviews({
368
+ trials,
369
+ cases: selected.map(({ id }) => id),
370
+ configurations: runFile.configurations.length,
371
+ });
372
+ const scoreOf = ({ configuration, caseId, trial }) => scores.get(reviewKey(runFile.configurations[configuration].fingerprint, caseId, trial));
373
+ const toRun = planned.filter((review) => {
374
+ const recorded = scoreOf(review);
375
+ return recorded === undefined || (options.retryIncomplete && recorded.incomplete);
376
+ });
377
+ writeLines(io.stderr, loaded.flatMap(({ warnings }, index) => warnings.map(warning => `warning: configuration ${index + 1}: ${warning}`)));
378
+ writeLines(options.dryRun ? io.stdout : io.stderr, planLines({
379
+ suite,
380
+ selected,
381
+ loaded,
382
+ trials,
383
+ planned: planned.length,
384
+ recorded: planned.filter(review => scoreOf(review) !== undefined).length,
385
+ toRun: toRun.length,
386
+ again: toRun.filter(review => scoreOf(review) !== undefined).length,
387
+ jobs: options.jobs,
388
+ directory,
389
+ }));
390
+ if (options.dryRun) {
391
+ writeLines(io.stdout, ['Dry run: no review was run, and nothing was written.']);
392
+ return BENCH_EXIT.ok;
393
+ }
394
+ if (io.signal?.aborted) {
395
+ io.stderr.write('interrupted\n');
396
+ return BENCH_EXIT.interrupted;
397
+ }
398
+ const outcome = await execute(io, options, checked, scores, validBytes, toRun);
399
+ await writeReports(io, checked, trials, scores);
400
+ if (outcome.interrupted) {
401
+ io.stderr.write('interrupted\n');
402
+ return BENCH_EXIT.interrupted;
403
+ }
404
+ return outcome.failed > 0
405
+ ? BENCH_EXIT.failed
406
+ : BENCH_EXIT.ok;
407
+ }
408
+ /** Write a usage error and the usage, and return its exit code. */
409
+ function usageError(io, error) {
410
+ io.stderr.write(`${errorLines(error)}\n${BENCH_USAGE}`);
411
+ return BENCH_EXIT.usage;
412
+ }
413
+ /**
414
+ * Run `quorum bench` with its arguments (those after `bench`) and return the exit code. Never throws.
415
+ *
416
+ * It reviews every case of the suite the argument names with every `--config` (the usual resolution when
417
+ * there is none), `--trials` times each, at most `--jobs` at once, and records each review that resolved
418
+ * in the run directory `--out` names (`quorum-bench/<suite name>` by default), one line of `results.jsonl`
419
+ * at a time. It prints the plan on stderr before the first call (on stdout with `--dry-run`, which then
420
+ * stops and writes nothing), a progress line on stderr for each finished review, and the Markdown report
421
+ * on stdout, which it also writes, with `report.json`, into the run directory.
422
+ *
423
+ * A run directory that holds a run resumes it: the reviews it records are not run again, `--retry-incomplete`
424
+ * runs again those that came back incomplete, and a last line a killed run cut short is dropped, with a
425
+ * warning. A run directory of another suite, another quorum or other configurations, one that holds files
426
+ * of another kind, and a suite or a configuration that does not validate or cannot be built are refused
427
+ * before the first call.
428
+ *
429
+ * Exit codes: 0 when every planned review is recorded, 2 for a usage, suite, configuration or run
430
+ * directory error, 3 when the run stops (a file cannot be written or read), 4 when a review failed (the
431
+ * others are recorded, and the next run runs the failed ones), 130 when interrupted. The reports are
432
+ * written on 0, 4 and 130, so a stopped run shows what it recorded; an interrupt that comes before the
433
+ * first review starts leaves the run directory untouched, even one that does not exist yet.
434
+ *
435
+ * Every line it writes is inert (invariant I3).
436
+ */
437
+ export async function runBench(io, argv) {
438
+ let options;
439
+ try {
440
+ const { values, positionals } = parseFlags(argv);
441
+ if (values.help) {
442
+ io.stdout.write(BENCH_USAGE);
443
+ return BENCH_EXIT.ok;
444
+ }
445
+ options = checkOptions(values, positionals);
446
+ }
447
+ catch (error) {
448
+ return usageError(io, error);
449
+ }
450
+ try {
451
+ return await bench(io, options);
452
+ }
453
+ catch (error) {
454
+ if (io.signal?.aborted) {
455
+ io.stderr.write('interrupted\n');
456
+ return BENCH_EXIT.interrupted;
457
+ }
458
+ io.stderr.write(`${errorLines(error)}\n`);
459
+ return error instanceof ConfigError
460
+ ? BENCH_EXIT.usage
461
+ : BENCH_EXIT.stopped;
462
+ }
463
+ }
@@ -0,0 +1,45 @@
1
+ /**
2
+ * What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, and how a
3
+ * failure is written. `main.ts` and `bench.ts` both import it, so neither imports the other.
4
+ */
5
+ import type { Env } from '../config/env.ts';
6
+ /** Exit codes (ADR-0008, amended by ADR-0012). */
7
+ export declare const EXIT: {
8
+ readonly ok: 0;
9
+ readonly finding: 1;
10
+ readonly usage: 2;
11
+ readonly runtime: 3;
12
+ readonly incomplete: 4;
13
+ readonly interrupted: 130;
14
+ };
15
+ export interface CliIO {
16
+ /** Arguments after the executable and script, e.g. `process.argv.slice(2)`. */
17
+ argv: readonly string[];
18
+ /** The directory a relative diff path and a relative `--config` resolve against, e.g. `process.cwd()`. */
19
+ readonly cwd: string;
20
+ /** Environment variables, e.g. `process.env`. */
21
+ env: Env;
22
+ stdin: AsyncIterable<Uint8Array | string> & {
23
+ isTTY?: boolean;
24
+ };
25
+ stdout: {
26
+ write: (text: string) => unknown;
27
+ };
28
+ stderr: {
29
+ write: (text: string) => unknown;
30
+ };
31
+ /** The review's signal: once it is aborted, a run that ends without a result exits 130. */
32
+ readonly signal?: AbortSignal;
33
+ }
34
+ /**
35
+ * The `Error:` lines of a failure, each inert: a ConfigError's issues one per line, as its message
36
+ * lists them, and any other failure's message on one line, so a line break in a file name or in text
37
+ * a server sent cannot start a line of its own.
38
+ */
39
+ export declare function errorLines(error: unknown): string;
40
+ /**
41
+ * The text of a failure, for any value a promise can reject with: an `Error`'s message, else the
42
+ * value's text form, else a fixed text when even that throws, as it does for an object without a
43
+ * prototype. It never throws, so neither do the lines built from it.
44
+ */
45
+ export declare function errorMessage(error: unknown): string;
package/dist/cli/io.js ADDED
@@ -0,0 +1,45 @@
1
+ /**
2
+ * What every subcommand of the CLI shares: the I/O it runs with, the exit codes of a review, and how a
3
+ * failure is written. `main.ts` and `bench.ts` both import it, so neither imports the other.
4
+ */
5
+ import { ConfigError } from "../core/errors.js";
6
+ import { inertLine } from "../core/sanitize.js";
7
+ /** Exit codes (ADR-0008, amended by ADR-0012). */
8
+ export const EXIT = {
9
+ ok: 0,
10
+ finding: 1,
11
+ usage: 2,
12
+ runtime: 3,
13
+ incomplete: 4,
14
+ interrupted: 130,
15
+ };
16
+ /**
17
+ * The `Error:` lines of a failure, each inert: a ConfigError's issues one per line, as its message
18
+ * lists them, and any other failure's message on one line, so a line break in a file name or in text
19
+ * a server sent cannot start a line of its own.
20
+ */
21
+ export function errorLines(error) {
22
+ const lines = error instanceof ConfigError
23
+ ? error.message.split('\n')
24
+ : [
25
+ errorMessage(error),
26
+ ];
27
+ return lines.map((line, index) => inertLine(index === 0
28
+ ? `Error: ${line}`
29
+ : line)).join('\n');
30
+ }
31
+ /**
32
+ * The text of a failure, for any value a promise can reject with: an `Error`'s message, else the
33
+ * value's text form, else a fixed text when even that throws, as it does for an object without a
34
+ * prototype. It never throws, so neither do the lines built from it.
35
+ */
36
+ export function errorMessage(error) {
37
+ if (error instanceof Error)
38
+ return error.message;
39
+ try {
40
+ return String(error);
41
+ }
42
+ catch {
43
+ return 'an error with no text form';
44
+ }
45
+ }
@@ -1,35 +1,8 @@
1
- import type { Env } from '../config/env.ts';
2
1
  import type { QuorumResult } from '../core/types.ts';
3
- /** Exit codes (ADR-0008, amended by ADR-0012). */
4
- export declare const EXIT: {
5
- readonly ok: 0;
6
- readonly finding: 1;
7
- readonly usage: 2;
8
- readonly runtime: 3;
9
- readonly incomplete: 4;
10
- readonly interrupted: 130;
11
- };
12
- /** The most diff the CLI reads, from stdin or a file, in bytes: a larger one is refused and reading stops there. */
13
- export declare const MAX_DIFF_BYTES: number;
14
- export interface CliIO {
15
- /** Arguments after the executable and script, e.g. `process.argv.slice(2)`. */
16
- argv: readonly string[];
17
- /** The directory a relative diff path and a relative `--config` resolve against, e.g. `process.cwd()`. */
18
- readonly cwd: string;
19
- /** Environment variables, e.g. `process.env`. */
20
- env: Env;
21
- stdin: AsyncIterable<Uint8Array | string> & {
22
- isTTY?: boolean;
23
- };
24
- stdout: {
25
- write: (text: string) => unknown;
26
- };
27
- stderr: {
28
- write: (text: string) => unknown;
29
- };
30
- /** The review's signal: once it is aborted, a run that ends without a result exits 130. */
31
- readonly signal?: AbortSignal;
32
- }
2
+ import { errorMessage, EXIT } from './io.ts';
3
+ import type { CliIO } from './io.ts';
4
+ export { errorMessage, EXIT };
5
+ export type { CliIO };
33
6
  /**
34
7
  * Run one review and return the process exit code. Never throws: usage and configuration problems
35
8
  * exit 2, runtime failures exit 3, an interrupt exits 130, and an incomplete review (a finding its
@@ -54,6 +27,9 @@ export interface CliIO {
54
27
  * The run is interrupted when `io.signal` aborts: it then writes `interrupted` and returns 130, whatever
55
28
  * error ended the review, and writes no result. A review that has already resolved is complete.
56
29
  *
30
+ * The one subcommand is `bench`: when the first argument is `bench`, the rest goes to `runBench` and its exit
31
+ * code is returned, before any review flag is read. A file named `bench` is reviewed as `quorum ./bench`.
32
+ *
57
33
  * Everything the CLI writes is inert (invariant I3): each stderr line passes through `inertLine`, and
58
34
  * the JSON result through `inertJson`.
59
35
  */
@@ -83,9 +59,3 @@ export declare function interruptSignal(source: SignalSource, exit: (code: numbe
83
59
  export declare function exitCodeFor(result: QuorumResult, options?: {
84
60
  readonly allowIncomplete?: boolean;
85
61
  }): number;
86
- /**
87
- * The text of a failure, for any value a promise can reject with: an `Error`'s message, else the
88
- * value's text form, else a fixed text when even that throws, as it does for an object without a
89
- * prototype. It never throws, so neither do the lines built from it.
90
- */
91
- export declare function errorMessage(error: unknown): string;