@orkestrel/scaffold 0.0.122 → 0.0.123
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/skills/orkestrel-harden/scripts/interleave.js +539 -0
- package/dist/agents/skills/orkestrel-parallel/scripts/lane.js +1163 -0
- package/dist/bin/main.js +8 -6
- package/dist/bin/main.js.map +1 -1
- package/dist/host/agents/orchestration.md +13 -7
- package/dist/host/agents/skills/orkestrel-dispatch/SKILL.md +4 -3
- package/dist/host/agents/skills/orkestrel-dispatch/references/launch.md +1 -1
- package/dist/host/agents/skills/orkestrel-falsify/SKILL.md +4 -2
- package/dist/host/agents/skills/orkestrel-falsify/references/brief.md +2 -2
- package/dist/host/agents/skills/orkestrel-harden/SKILL.md +16 -3
- package/dist/host/agents/skills/orkestrel-harden/references/bindings.md +18 -12
- package/dist/host/agents/skills/orkestrel-harden/references/document.md +1 -0
- package/dist/host/agents/skills/orkestrel-harden/references/equivalence.md +2 -1
- package/dist/host/agents/skills/orkestrel-harden/references/hardening.md +1 -1
- package/dist/host/agents/skills/orkestrel-harden/references/methods.md +642 -0
- package/dist/host/agents/skills/orkestrel-harden/scripts/interleave.ts +665 -0
- package/dist/host/agents/skills/orkestrel-parallel/SKILL.md +14 -13
- package/dist/host/agents/skills/orkestrel-parallel/references/claude-code.md +13 -13
- package/dist/host/agents/skills/orkestrel-parallel/references/t3.md +1 -0
- package/dist/host/agents/skills/orkestrel-parallel/scripts/lane.ts +1433 -0
- package/dist/host/agents/skills/orkestrel-publish/SKILL.md +2 -1
- package/dist/host/agents/skills/orkestrel-publish/references/wave.md +18 -1
- package/dist/host/claude/AGENTS.md +1 -1
- package/dist/host/claude/agents/analyst.md +1 -1
- package/dist/host/claude/agents/builder.md +1 -1
- package/dist/host/claude/agents/distiller.md +2 -1
- package/dist/host/claude/agents/grok.md +2 -2
- package/dist/host/claude/agents/opus.md +2 -2
- package/dist/host/claude/agents/orkestrel.md +7 -7
- package/dist/host/claude/agents/reviewer.md +2 -2
- package/dist/host/claude/agents/scout.md +2 -1
- package/dist/host/claude/agents/verifier.md +1 -1
- package/dist/host/claude/rules/campaign.md +12 -11
- package/dist/host/claude/rules/documentation.md +1 -0
- package/dist/host/claude/rules/patterns.md +14 -0
- package/dist/host/claude/rules/quality.md +4 -1
- package/dist/host/claude/rules/tests.md +4 -0
- package/dist/host/claude/rules/writing.md +1 -1
- package/dist/host/claude/skills/orkestrel-parallel/SKILL.md +1 -1
- package/dist/host/codex/agents/analyst.toml +4 -1
- package/dist/host/codex/agents/astra.toml +1 -1
- package/dist/host/codex/agents/builder.toml +1 -1
- package/dist/host/codex/agents/distiller.toml +4 -2
- package/dist/host/codex/agents/grok.toml +5 -2
- package/dist/host/codex/agents/opus.toml +3 -3
- package/dist/host/codex/agents/reviewer.toml +6 -2
- package/dist/host/codex/agents/scout.toml +5 -1
- package/dist/host/codex/agents/verifier.toml +6 -1
- package/dist/host/guides/agent.md +1119 -457
- package/dist/host/guides/ndjson.md +81 -21
- package/dist/host/guides/ollama.md +22 -18
- package/dist/host/guides/probe.md +29 -27
- package/dist/host/guides/scaffold.md +36 -14
- package/dist/host/guides/study.md +538 -450
- package/dist/host/manifest.json +66 -48
- package/dist/src/core/index.cjs +8 -7
- package/dist/src/core/index.cjs.map +1 -1
- package/dist/src/core/index.d.cts +1 -4
- package/dist/src/core/index.d.ts +1 -4
- package/dist/src/core/index.js +8 -7
- package/dist/src/core/index.js.map +1 -1
- package/dist/src/server/index.cjs +87 -8
- package/dist/src/server/index.cjs.map +1 -1
- package/dist/src/server/index.d.cts +50 -6
- package/dist/src/server/index.d.ts +50 -6
- package/dist/src/server/index.js +87 -10
- package/dist/src/server/index.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,539 @@
|
|
|
1
|
+
// Compare two commands sample by sample under one host load. Run from the checkout root:
|
|
2
|
+
// node .agents/skills/orkestrel-harden/scripts/interleave.ts --plan FILE.json --rounds N [--json] [--out FILE.json|FILE.txt]
|
|
3
|
+
// The plan is a JSON object `{ "baseline": SIDE, "candidate": SIDE }`, each side
|
|
4
|
+
// `{ "command": "...", "args": ["..."] }` with an optional `"cwd"` (a directory, relative to the
|
|
5
|
+
// working directory) and `"env"` (a record of strings laid over this process's environment by
|
|
6
|
+
// case-folded name). Identical sides are a supported A/A comparison, which calibrates the run.
|
|
7
|
+
// `N` is an even whole number of at least 2. `baseline` runs first in each odd round and `candidate`
|
|
8
|
+
// in each even round; an odd count is refused because the side with the extra lead puts its leading
|
|
9
|
+
// readings at the median, and that order cost reads as a difference between the sides.
|
|
10
|
+
// Each sample is a fresh process with no shell and closed stdin. The reading is the number the
|
|
11
|
+
// child prints as its last non-empty standard output line, a finite non-negative decimal with an
|
|
12
|
+
// optional fraction and exponent. The script reads no clock, so the child's marked region alone
|
|
13
|
+
// decides what a reading covers, and process start stays out of it unless that region includes it.
|
|
14
|
+
// A sample that cannot start, ends by a signal, exits nonzero, or prints no reading fails the run:
|
|
15
|
+
// the script names the side and the round on stderr, runs no further sample, and prints no report,
|
|
16
|
+
// so a statistic never omits a failed sample.
|
|
17
|
+
// The report carries each side's samples in round order, its median, its deviation (the median
|
|
18
|
+
// absolute deviation, which stays on the median's scale and moves little under one load spike, where
|
|
19
|
+
// a standard deviation or a range follows the spike), its minimum and maximum, then `difference`
|
|
20
|
+
// (candidate median minus baseline median) and `relative` (`difference` over the baseline median,
|
|
21
|
+
// absent when that median is 0). It rules nothing: it carries no bar and no verdict.
|
|
22
|
+
// Print text by default or one JSON line with `--json`; `--out` writes `.json` or `.txt`
|
|
23
|
+
// independently. Launch the script through `launch.ts` from the dispatch skill for a cap. The script
|
|
24
|
+
// refuses a flag it does not read, the `--flag=value` form included, and every positional argument.
|
|
25
|
+
// Exit 0 when every sample read, 2 when a sample failed or `--out` could not be written, 64 on usage.
|
|
26
|
+
import { spawnSync } from 'node:child_process';
|
|
27
|
+
import { mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs';
|
|
28
|
+
import { dirname, extname, resolve } from 'node:path';
|
|
29
|
+
/** Lists the flags that take a value. */
|
|
30
|
+
export const VALUE_FLAGS = ['--plan', '--rounds', '--out'];
|
|
31
|
+
/** Lists every flag the script reads, value flags and switches alike. */
|
|
32
|
+
export const FLAGS = [...VALUE_FLAGS, '--json'];
|
|
33
|
+
/** Holds the usage line printed when a call names no plan or no round count. */
|
|
34
|
+
export const USAGE = 'usage: interleave.ts --plan FILE.json --rounds N [--json] [--out FILE.json|FILE.txt]';
|
|
35
|
+
/** Caps the characters of an untrusted text a diagnostic echoes. */
|
|
36
|
+
export const QUOTE_LIMIT = 200;
|
|
37
|
+
/** Holds the significant digits a text report gives a computed statistic. */
|
|
38
|
+
export const REPORT_PRECISION = 6;
|
|
39
|
+
/** Matches a reading: a non-negative decimal with an optional fraction and exponent. */
|
|
40
|
+
export const READING = /^\d+(?:\.\d+)?(?:[eE][-+]?\d+)?$/u;
|
|
41
|
+
/** Matches a whole number with no sign, exponent, or leading zero. */
|
|
42
|
+
export const WHOLE = /^[1-9]\d*$/u;
|
|
43
|
+
/**
|
|
44
|
+
* Reads the value that follows a flag.
|
|
45
|
+
*
|
|
46
|
+
* @remarks
|
|
47
|
+
* The first occurrence of a flag wins, a following value that opens with `--` is a missing value,
|
|
48
|
+
* and the `--flag=value` form is not read.
|
|
49
|
+
*
|
|
50
|
+
* @param argv - The arguments after the script path.
|
|
51
|
+
* @param name - The flag, such as `--plan`.
|
|
52
|
+
* @returns The value after the flag, or undefined when the flag is absent, last, or followed by
|
|
53
|
+
* another flag.
|
|
54
|
+
*/
|
|
55
|
+
export function readOption(argv, name) {
|
|
56
|
+
const index = argv.indexOf(name);
|
|
57
|
+
const value = index === -1 ? undefined : argv[index + 1];
|
|
58
|
+
return value === undefined || value.startsWith('--') ? undefined : value;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Names the value flags a caller gave with no value.
|
|
62
|
+
*
|
|
63
|
+
* @param argv - The arguments after the script path.
|
|
64
|
+
* @param names - The flags that take a value.
|
|
65
|
+
* @returns Each named flag that appears last or is followed by another flag, in argument order.
|
|
66
|
+
*/
|
|
67
|
+
export function readMissingFlags(argv, names) {
|
|
68
|
+
return names.filter((name) => argv.includes(name) && readOption(argv, name) === undefined);
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Names the flags a caller gave that the script does not read.
|
|
72
|
+
*
|
|
73
|
+
* @remarks
|
|
74
|
+
* A `--flag=value` argument is unknown, because the script reads no such form. A bare `--` ends the
|
|
75
|
+
* scan.
|
|
76
|
+
*
|
|
77
|
+
* @param argv - The arguments after the script path.
|
|
78
|
+
* @param flags - Every flag the script reads, value flags and switches alike.
|
|
79
|
+
* @returns Each argument that opens with `--` and is not a read flag, in argument order.
|
|
80
|
+
*/
|
|
81
|
+
export function readUnknownFlags(argv, flags) {
|
|
82
|
+
const end = argv.indexOf('--');
|
|
83
|
+
return (end === -1 ? argv : argv.slice(0, end)).filter((argument) => argument.startsWith('--') && !flags.includes(argument));
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Reads the arguments the script has no place for: a positional, or a bare `--`, which would open a
|
|
87
|
+
* command this script does not pass on.
|
|
88
|
+
*
|
|
89
|
+
* @param argv - The arguments after the script path.
|
|
90
|
+
* @param valueFlags - The flags that take a value.
|
|
91
|
+
* @returns Each such argument in order.
|
|
92
|
+
*/
|
|
93
|
+
export function readStrayArguments(argv, valueFlags) {
|
|
94
|
+
return argv.filter((argument, index) => (argument === '--' || !argument.startsWith('--')) &&
|
|
95
|
+
!valueFlags.includes(argv[index - 1] ?? ''));
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Checks whether a value is a non-null, non-array object.
|
|
99
|
+
*
|
|
100
|
+
* @param value - The value to inspect.
|
|
101
|
+
* @returns True if the value is an object other than null or an array; false otherwise.
|
|
102
|
+
*/
|
|
103
|
+
export function isRecord(value) {
|
|
104
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Checks whether a value is an array of strings.
|
|
108
|
+
*
|
|
109
|
+
* @param value - The value to inspect.
|
|
110
|
+
* @returns True if the value is an array whose every member is a string; false otherwise.
|
|
111
|
+
*/
|
|
112
|
+
export function isStringArray(value) {
|
|
113
|
+
return Array.isArray(value) && value.every((member) => typeof member === 'string');
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Checks whether a value is a record whose every member is a string.
|
|
117
|
+
*
|
|
118
|
+
* @param value - The value to inspect.
|
|
119
|
+
* @returns True if the value is a record with only string members; false otherwise.
|
|
120
|
+
*/
|
|
121
|
+
export function isStringRecord(value) {
|
|
122
|
+
return isRecord(value) && Object.values(value).every((member) => typeof member === 'string');
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Quotes an untrusted text for a diagnostic, JSON-encoded and cut to a bounded length.
|
|
126
|
+
*
|
|
127
|
+
* @param text - The text to quote.
|
|
128
|
+
* @returns The JSON string literal of the text, ending in an ellipsis inside the quotes when cut.
|
|
129
|
+
*/
|
|
130
|
+
export function quote(text) {
|
|
131
|
+
return JSON.stringify(text.length > QUOTE_LIMIT ? `${text.slice(0, QUOTE_LIMIT)}…` : text);
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Parses a JSON text without throwing.
|
|
135
|
+
*
|
|
136
|
+
* @param text - The text to parse.
|
|
137
|
+
* @returns The parsed value, or undefined when the text is not JSON.
|
|
138
|
+
*/
|
|
139
|
+
export function parseJson(text) {
|
|
140
|
+
try {
|
|
141
|
+
return JSON.parse(text);
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
return undefined;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Lists what is wrong with one side of a plan.
|
|
149
|
+
*
|
|
150
|
+
* @param entry - The parsed JSON value under the side's key.
|
|
151
|
+
* @param name - The side's name, named in each problem.
|
|
152
|
+
* @returns Each problem in the entry, empty when the entry is a side.
|
|
153
|
+
*/
|
|
154
|
+
export function readSideProblems(entry, name) {
|
|
155
|
+
if (entry === undefined)
|
|
156
|
+
return [`the plan holds no "${name}" side`];
|
|
157
|
+
if (!isRecord(entry))
|
|
158
|
+
return [`${name} is not an object`];
|
|
159
|
+
const known = ['command', 'args', 'cwd', 'env'];
|
|
160
|
+
const unexpected = Object.keys(entry).filter((key) => !known.includes(key));
|
|
161
|
+
return [
|
|
162
|
+
...(typeof entry.command === 'string' && entry.command !== ''
|
|
163
|
+
? []
|
|
164
|
+
: [`${name} "command" must be a non-empty string`]),
|
|
165
|
+
...(isStringArray(entry.args) ? [] : [`${name} "args" must be an array of strings`]),
|
|
166
|
+
...(entry.cwd === undefined || (typeof entry.cwd === 'string' && entry.cwd !== '')
|
|
167
|
+
? []
|
|
168
|
+
: [`${name} "cwd" must be a non-empty string when present`]),
|
|
169
|
+
...(entry.env === undefined || isStringRecord(entry.env)
|
|
170
|
+
? []
|
|
171
|
+
: [`${name} "env" must be a record of strings when present`]),
|
|
172
|
+
...unexpected.map((key) => `${name} carries the unexpected key ${quote(key)}`),
|
|
173
|
+
];
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Reads a validated plan entry as a side.
|
|
177
|
+
*
|
|
178
|
+
* @param entry - A plan entry that `readSideProblems` found no problem in.
|
|
179
|
+
* @returns The side, or undefined when the entry is not a valid side.
|
|
180
|
+
*/
|
|
181
|
+
export function readSide(entry) {
|
|
182
|
+
if (!isRecord(entry))
|
|
183
|
+
return undefined;
|
|
184
|
+
const { command, args, cwd, env } = entry;
|
|
185
|
+
if (typeof command !== 'string' || !isStringArray(args))
|
|
186
|
+
return undefined;
|
|
187
|
+
return {
|
|
188
|
+
command,
|
|
189
|
+
args,
|
|
190
|
+
cwd: typeof cwd === 'string' ? cwd : undefined,
|
|
191
|
+
env: isStringRecord(env) ? env : {},
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Reads a plan from its JSON text.
|
|
196
|
+
*
|
|
197
|
+
* @param text - The plan file's text.
|
|
198
|
+
* @returns The sides when the plan is valid, and every problem otherwise: not a JSON object, an
|
|
199
|
+
* unexpected key, a missing side, or a malformed side.
|
|
200
|
+
*/
|
|
201
|
+
export function parsePlan(text) {
|
|
202
|
+
const parsed = parseJson(text);
|
|
203
|
+
if (!isRecord(parsed))
|
|
204
|
+
return {
|
|
205
|
+
sides: undefined,
|
|
206
|
+
problems: ['the plan is not a JSON object with "baseline" and "candidate" sides'],
|
|
207
|
+
};
|
|
208
|
+
const problems = [
|
|
209
|
+
...Object.keys(parsed)
|
|
210
|
+
.filter((key) => key !== 'baseline' && key !== 'candidate')
|
|
211
|
+
.map((key) => `the plan carries the unexpected key ${quote(key)}`),
|
|
212
|
+
...readSideProblems(parsed.baseline, 'baseline'),
|
|
213
|
+
...readSideProblems(parsed.candidate, 'candidate'),
|
|
214
|
+
];
|
|
215
|
+
const baseline = readSide(parsed.baseline);
|
|
216
|
+
const candidate = readSide(parsed.candidate);
|
|
217
|
+
return problems.length > 0 || baseline === undefined || candidate === undefined
|
|
218
|
+
? { sides: undefined, problems }
|
|
219
|
+
: { sides: { baseline, candidate }, problems: [] };
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Reads a plan file's text without throwing.
|
|
223
|
+
*
|
|
224
|
+
* @param path - The plan file.
|
|
225
|
+
* @returns The file's text, or undefined when it cannot be read.
|
|
226
|
+
*/
|
|
227
|
+
export function readPlanText(path) {
|
|
228
|
+
try {
|
|
229
|
+
return readFileSync(path, 'utf8');
|
|
230
|
+
}
|
|
231
|
+
catch {
|
|
232
|
+
return undefined;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Parses a round count.
|
|
237
|
+
*
|
|
238
|
+
* @param text - The value of `--rounds`.
|
|
239
|
+
* @returns The count when the text is an even whole number of at least 2, and a safe integer;
|
|
240
|
+
* undefined otherwise.
|
|
241
|
+
*/
|
|
242
|
+
export function parseRounds(text) {
|
|
243
|
+
const rounds = WHOLE.test(text) ? Number(text) : undefined;
|
|
244
|
+
return rounds !== undefined && Number.isSafeInteger(rounds) && rounds % 2 === 0
|
|
245
|
+
? rounds
|
|
246
|
+
: undefined;
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Reads the last non-empty line of a child's standard output.
|
|
250
|
+
*
|
|
251
|
+
* @param output - The child's standard output.
|
|
252
|
+
* @returns The line with its surrounding whitespace trimmed, or an empty string when no line holds
|
|
253
|
+
* text.
|
|
254
|
+
*/
|
|
255
|
+
export function readLastLine(output) {
|
|
256
|
+
return (output
|
|
257
|
+
.split(/\r\n|\n/u)
|
|
258
|
+
.map((line) => line.trim())
|
|
259
|
+
.findLast((line) => line !== '') ?? '');
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Parses one line as a reading.
|
|
263
|
+
*
|
|
264
|
+
* @param line - The line the child printed.
|
|
265
|
+
* @returns The finite non-negative number the whole line spells, or undefined otherwise.
|
|
266
|
+
*/
|
|
267
|
+
export function parseReading(line) {
|
|
268
|
+
const reading = READING.test(line) ? Number(line) : undefined;
|
|
269
|
+
return reading !== undefined && Number.isFinite(reading) ? reading : undefined;
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Lays added variables over an inherited environment by case-folded name.
|
|
273
|
+
*
|
|
274
|
+
* @remarks
|
|
275
|
+
* A Windows environment block folds names by case, so an inherited `NPM_CONFIG_CACHE` would
|
|
276
|
+
* outrank an added `npm_config_cache`; every inherited variable whose lower-cased name an added
|
|
277
|
+
* variable claims is dropped first.
|
|
278
|
+
*
|
|
279
|
+
* @param inherited - The environment the child would inherit.
|
|
280
|
+
* @param added - The variables the side sets.
|
|
281
|
+
* @returns The inherited variables no added name claims, followed by the added variables.
|
|
282
|
+
*/
|
|
283
|
+
export function mergeEnv(inherited, added) {
|
|
284
|
+
const claimed = new Set(Object.keys(added).map((name) => name.toLowerCase()));
|
|
285
|
+
return {
|
|
286
|
+
...Object.fromEntries(Object.entries(inherited).filter(([name]) => !claimed.has(name.toLowerCase()))),
|
|
287
|
+
...added,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Computes the order a run takes its samples in.
|
|
292
|
+
*
|
|
293
|
+
* @param rounds - The round count.
|
|
294
|
+
* @returns One slot per side per round, numbered from 1, with `baseline` first in each odd round and
|
|
295
|
+
* `candidate` first in each even round.
|
|
296
|
+
*/
|
|
297
|
+
export function computeSchedule(rounds) {
|
|
298
|
+
return Array.from({ length: rounds }, (_, index) => index + 1).flatMap((round) => {
|
|
299
|
+
const order = round % 2 === 1 ? ['baseline', 'candidate'] : ['candidate', 'baseline'];
|
|
300
|
+
return order.map((side) => ({ round, side }));
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* Runs one sample in a fresh process with no shell and closed stdin.
|
|
305
|
+
*
|
|
306
|
+
* @param side - The side's command.
|
|
307
|
+
* @returns The reading, or the reason the sample has none.
|
|
308
|
+
*/
|
|
309
|
+
export function runSample(side) {
|
|
310
|
+
const result = spawnSync(side.command, side.args, {
|
|
311
|
+
cwd: side.cwd === undefined ? undefined : resolve(side.cwd),
|
|
312
|
+
env: mergeEnv(process.env, side.env),
|
|
313
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
314
|
+
encoding: 'utf8',
|
|
315
|
+
windowsHide: true,
|
|
316
|
+
shell: false,
|
|
317
|
+
});
|
|
318
|
+
if (result.error !== undefined)
|
|
319
|
+
return { success: false, error: `did not finish: ${quote(result.error.message)}` };
|
|
320
|
+
if (result.signal !== null)
|
|
321
|
+
return { success: false, error: `ended by ${result.signal}` };
|
|
322
|
+
if (result.status !== 0)
|
|
323
|
+
return {
|
|
324
|
+
success: false,
|
|
325
|
+
error: `exited ${String(result.status)}: ${quote(result.stderr.trim())}`,
|
|
326
|
+
};
|
|
327
|
+
const line = readLastLine(result.stdout);
|
|
328
|
+
const reading = parseReading(line);
|
|
329
|
+
return reading === undefined
|
|
330
|
+
? { success: false, error: `printed no reading, its last line is ${quote(line)}` }
|
|
331
|
+
: { success: true, value: reading };
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* Runs a schedule in order and stops at the first sample that fails.
|
|
335
|
+
*
|
|
336
|
+
* @param sides - The commands.
|
|
337
|
+
* @param schedule - The samples to take.
|
|
338
|
+
* @returns The readings in the order they ran, and the failure when a sample failed.
|
|
339
|
+
*/
|
|
340
|
+
export function runSchedule(sides, schedule) {
|
|
341
|
+
const readings = [];
|
|
342
|
+
for (const slot of schedule) {
|
|
343
|
+
const sample = runSample(sides[slot.side]);
|
|
344
|
+
if (!sample.success)
|
|
345
|
+
return { readings, failure: { ...slot, problem: sample.error } };
|
|
346
|
+
readings.push({ ...slot, reading: sample.value });
|
|
347
|
+
}
|
|
348
|
+
return { readings, failure: undefined };
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* Computes the median of a list of numbers.
|
|
352
|
+
*
|
|
353
|
+
* @param values - The numbers, in any order.
|
|
354
|
+
* @returns The middle value, or the mean of the two middle values for an even count.
|
|
355
|
+
* @throws Thrown when the list is empty.
|
|
356
|
+
*/
|
|
357
|
+
export function computeMedian(values) {
|
|
358
|
+
if (values.length === 0)
|
|
359
|
+
throw new Error('interleave: a median needs at least one number');
|
|
360
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
361
|
+
const middle = sorted.slice(Math.floor((sorted.length - 1) / 2), Math.floor(sorted.length / 2) + 1);
|
|
362
|
+
return middle.reduce((sum, value) => sum + value, 0) / middle.length;
|
|
363
|
+
}
|
|
364
|
+
/**
|
|
365
|
+
* Computes the median absolute deviation of a list of numbers.
|
|
366
|
+
*
|
|
367
|
+
* @param values - The numbers, in any order.
|
|
368
|
+
* @returns The median distance of each number from the list's median.
|
|
369
|
+
* @throws Thrown when the list is empty.
|
|
370
|
+
*/
|
|
371
|
+
export function computeDeviation(values) {
|
|
372
|
+
const median = computeMedian(values);
|
|
373
|
+
return computeMedian(values.map((value) => Math.abs(value - median)));
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* Computes one side's statistics.
|
|
377
|
+
*
|
|
378
|
+
* @param samples - The side's samples in round order.
|
|
379
|
+
* @returns The samples with their median, deviation, minimum, and maximum.
|
|
380
|
+
* @throws Thrown when the list is empty.
|
|
381
|
+
*/
|
|
382
|
+
export function computeSummary(samples) {
|
|
383
|
+
return {
|
|
384
|
+
samples,
|
|
385
|
+
median: computeMedian(samples),
|
|
386
|
+
deviation: computeDeviation(samples),
|
|
387
|
+
minimum: Math.min(...samples),
|
|
388
|
+
maximum: Math.max(...samples),
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
/**
|
|
392
|
+
* Computes the report of a finished run.
|
|
393
|
+
*
|
|
394
|
+
* @param readings - Every reading of a run that no sample failed.
|
|
395
|
+
* @returns Each side's summary, the candidate median minus the baseline median, and that difference
|
|
396
|
+
* over the baseline median, undefined when that median is 0.
|
|
397
|
+
* @throws Thrown when a side has no reading.
|
|
398
|
+
*/
|
|
399
|
+
export function computeReport(readings) {
|
|
400
|
+
const baseline = computeSummary(readings.filter((entry) => entry.side === 'baseline').map((entry) => entry.reading));
|
|
401
|
+
const candidate = computeSummary(readings.filter((entry) => entry.side === 'candidate').map((entry) => entry.reading));
|
|
402
|
+
const difference = candidate.median - baseline.median;
|
|
403
|
+
return {
|
|
404
|
+
rounds: baseline.samples.length,
|
|
405
|
+
baseline,
|
|
406
|
+
candidate,
|
|
407
|
+
difference,
|
|
408
|
+
relative: baseline.median === 0 ? undefined : difference / baseline.median,
|
|
409
|
+
};
|
|
410
|
+
}
|
|
411
|
+
/**
|
|
412
|
+
* Renders a computed statistic to the report's precision.
|
|
413
|
+
*
|
|
414
|
+
* @param value - The statistic.
|
|
415
|
+
* @returns The number at `REPORT_PRECISION` significant digits, with no trailing zero.
|
|
416
|
+
*/
|
|
417
|
+
export function renderNumber(value) {
|
|
418
|
+
return String(Number(value.toPrecision(REPORT_PRECISION)));
|
|
419
|
+
}
|
|
420
|
+
/**
|
|
421
|
+
* Renders one side's line of the text report.
|
|
422
|
+
*
|
|
423
|
+
* @param name - The side's name.
|
|
424
|
+
* @param summary - The side's statistics.
|
|
425
|
+
* @returns The line, with the samples as they were read.
|
|
426
|
+
*/
|
|
427
|
+
export function renderSummary(name, summary) {
|
|
428
|
+
return [
|
|
429
|
+
name,
|
|
430
|
+
`median=${renderNumber(summary.median)}`,
|
|
431
|
+
`deviation=${renderNumber(summary.deviation)}`,
|
|
432
|
+
`minimum=${renderNumber(summary.minimum)}`,
|
|
433
|
+
`maximum=${renderNumber(summary.maximum)}`,
|
|
434
|
+
`samples=${summary.samples.map(String).join(',')}`,
|
|
435
|
+
].join(' ');
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* Renders the text report.
|
|
439
|
+
*
|
|
440
|
+
* @param report - The finished comparison.
|
|
441
|
+
* @returns A rounds line, one line per side, and a line with the difference and the relative change
|
|
442
|
+
* in percent, `none` when the baseline median is 0.
|
|
443
|
+
*/
|
|
444
|
+
export function renderReport(report) {
|
|
445
|
+
const relative = report.relative === undefined ? 'none' : `${renderNumber(report.relative * 100)}%`;
|
|
446
|
+
return [
|
|
447
|
+
`rounds=${String(report.rounds)}`,
|
|
448
|
+
renderSummary('baseline', report.baseline),
|
|
449
|
+
renderSummary('candidate', report.candidate),
|
|
450
|
+
`difference=${renderNumber(report.difference)} relative=${relative}`,
|
|
451
|
+
].join('\n');
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* Writes a report form to a file, creating its parent directories, without throwing.
|
|
455
|
+
*
|
|
456
|
+
* @param path - The output file.
|
|
457
|
+
* @param text - The report form.
|
|
458
|
+
* @returns The reason the file could not be written, or undefined when it was.
|
|
459
|
+
*/
|
|
460
|
+
export function writeOutput(path, text) {
|
|
461
|
+
try {
|
|
462
|
+
const target = resolve(path);
|
|
463
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
464
|
+
writeFileSync(target, `${text}\n`);
|
|
465
|
+
return undefined;
|
|
466
|
+
}
|
|
467
|
+
catch (error) {
|
|
468
|
+
return error instanceof Error ? error.message : String(error);
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
/**
|
|
472
|
+
* Runs the script on its arguments.
|
|
473
|
+
*
|
|
474
|
+
* @param argv - The arguments after the script path.
|
|
475
|
+
* @returns The exit code: 0 when every sample read, 2 when a sample failed or `--out` could not be
|
|
476
|
+
* written, 64 on usage.
|
|
477
|
+
*/
|
|
478
|
+
export function main(argv) {
|
|
479
|
+
const unknown = readUnknownFlags(argv, FLAGS);
|
|
480
|
+
if (unknown.length > 0) {
|
|
481
|
+
console.error(`interleave: ${unknown.join(', ')} is not a flag this script reads; ${USAGE}`);
|
|
482
|
+
return 64;
|
|
483
|
+
}
|
|
484
|
+
const missing = readMissingFlags(argv, VALUE_FLAGS);
|
|
485
|
+
if (missing.length > 0) {
|
|
486
|
+
console.error(`interleave: ${missing.join(', ')} given with no value`);
|
|
487
|
+
return 64;
|
|
488
|
+
}
|
|
489
|
+
const stray = readStrayArguments(argv, VALUE_FLAGS);
|
|
490
|
+
if (stray.length > 0) {
|
|
491
|
+
console.error(`interleave: ${stray.map(quote).join(', ')} is not an argument; ${USAGE}`);
|
|
492
|
+
return 64;
|
|
493
|
+
}
|
|
494
|
+
const path = readOption(argv, '--plan');
|
|
495
|
+
const count = readOption(argv, '--rounds');
|
|
496
|
+
const out = readOption(argv, '--out');
|
|
497
|
+
if (path === undefined || count === undefined) {
|
|
498
|
+
console.error(USAGE);
|
|
499
|
+
return 64;
|
|
500
|
+
}
|
|
501
|
+
if (out !== undefined && !['.json', '.txt'].includes(extname(out))) {
|
|
502
|
+
console.error(`interleave: --out takes a .json or .txt file, got ${quote(out)}`);
|
|
503
|
+
return 64;
|
|
504
|
+
}
|
|
505
|
+
const rounds = parseRounds(count);
|
|
506
|
+
if (rounds === undefined) {
|
|
507
|
+
console.error(`interleave: --rounds takes an even whole number of at least 2, got ${quote(count)}`);
|
|
508
|
+
return 64;
|
|
509
|
+
}
|
|
510
|
+
const text = readPlanText(path);
|
|
511
|
+
if (text === undefined) {
|
|
512
|
+
console.error(`interleave: the plan ${quote(path)} cannot be read`);
|
|
513
|
+
return 64;
|
|
514
|
+
}
|
|
515
|
+
const plan = parsePlan(text);
|
|
516
|
+
if (plan.sides === undefined) {
|
|
517
|
+
console.error(`interleave: ${quote(path)} is not a valid plan: ${plan.problems.join('; ')}`);
|
|
518
|
+
return 64;
|
|
519
|
+
}
|
|
520
|
+
const run = runSchedule(plan.sides, computeSchedule(rounds));
|
|
521
|
+
if (run.failure !== undefined) {
|
|
522
|
+
const { side, round, problem } = run.failure;
|
|
523
|
+
console.error(`interleave: ${side} round ${String(round)}: ${problem}`);
|
|
524
|
+
return 2;
|
|
525
|
+
}
|
|
526
|
+
const report = computeReport(run.readings);
|
|
527
|
+
const json = JSON.stringify(report);
|
|
528
|
+
const rendered = renderReport(report);
|
|
529
|
+
console.log(argv.includes('--json') ? json : rendered);
|
|
530
|
+
const refusal = out === undefined ? undefined : writeOutput(out, extname(out) === '.json' ? json : rendered);
|
|
531
|
+
if (refusal !== undefined) {
|
|
532
|
+
console.error(`interleave: --out could not be written: ${quote(refusal)}`);
|
|
533
|
+
return 2;
|
|
534
|
+
}
|
|
535
|
+
return 0;
|
|
536
|
+
}
|
|
537
|
+
if (process.argv[1] !== undefined && realpathSync(process.argv[1]) === import.meta.filename) {
|
|
538
|
+
process.exitCode = main(process.argv.slice(2));
|
|
539
|
+
}
|