bitfab 0.52.2 → 0.52.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-XL5M3ETX.js → chunk-34B7PVAM.js} +79 -28
- package/dist/chunk-34B7PVAM.js.map +1 -0
- package/dist/{chunk-RU7KJE2D.js → chunk-AUROMI67.js} +61 -11
- package/dist/chunk-AUROMI67.js.map +1 -0
- package/dist/chunk-HKTODDFP.js +747 -0
- package/dist/chunk-HKTODDFP.js.map +1 -0
- package/dist/{chunk-ESIEHZHQ.js → chunk-IG3CPSRQ.js} +48 -21
- package/dist/chunk-IG3CPSRQ.js.map +1 -0
- package/dist/{chunk-SIMPLMRG.js → chunk-U2D7ACC4.js} +3 -2
- package/dist/chunk-U2D7ACC4.js.map +1 -0
- package/dist/{http-2NFFBFYV.js → http-F3AXLOJT.js} +2 -2
- package/dist/{http-RULNEWBV.js → http-RFNQ7LSU.js} +2 -2
- package/dist/index.cjs +134 -31
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +36 -2
- package/dist/index.d.ts +36 -2
- package/dist/index.js +5 -3
- package/dist/node.cjs +134 -31
- package/dist/node.cjs.map +1 -1
- package/dist/node.d.cts +1 -1
- package/dist/node.d.ts +1 -1
- package/dist/node.js +5 -3
- package/dist/node.js.map +1 -1
- package/dist/{replay-6SQSBWLX.js → replay-ZOTYMBMD.js} +5 -3
- package/dist/replayCli.js +18 -687
- package/dist/replayCli.js.map +1 -1
- package/dist/seedCli.js +30 -0
- package/dist/seedCli.js.map +1 -0
- package/package.json +3 -2
- package/dist/chunk-ESIEHZHQ.js.map +0 -1
- package/dist/chunk-RU7KJE2D.js.map +0 -1
- package/dist/chunk-SIMPLMRG.js.map +0 -1
- package/dist/chunk-XL5M3ETX.js.map +0 -1
- /package/dist/{http-2NFFBFYV.js.map → http-F3AXLOJT.js.map} +0 -0
- /package/dist/{http-RULNEWBV.js.map → http-RFNQ7LSU.js.map} +0 -0
- /package/dist/{replay-6SQSBWLX.js.map → replay-ZOTYMBMD.js.map} +0 -0
package/dist/replayCli.js
CHANGED
|
@@ -1,701 +1,32 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
// src/serialize.ts
|
|
13
|
-
import superjson from "superjson";
|
|
14
|
-
|
|
15
|
-
// src/replay.ts
|
|
16
|
-
var BITFAB_PROGRESS_PREFIX = "@@bitfab:progress ";
|
|
17
|
-
function reportReplayProgress(progress) {
|
|
18
|
-
const stderr = typeof process !== "undefined" ? process.stderr : void 0;
|
|
19
|
-
if (!stderr) {
|
|
20
|
-
return;
|
|
21
|
-
}
|
|
22
|
-
try {
|
|
23
|
-
stderr.write(
|
|
24
|
-
`${BITFAB_PROGRESS_PREFIX}${JSON.stringify(progress, replayJsonReplacer)}
|
|
25
|
-
`
|
|
26
|
-
);
|
|
27
|
-
} catch {
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
var ReplayError = class extends BitfabError {
|
|
31
|
-
constructor(message, items, testRunId, testRunUrl, cause) {
|
|
32
|
-
super(message, testRunUrl);
|
|
33
|
-
this.items = items;
|
|
34
|
-
this.testRunId = testRunId;
|
|
35
|
-
this.testRunUrl = testRunUrl;
|
|
36
|
-
this.cause = cause;
|
|
37
|
-
this.name = "ReplayError";
|
|
38
|
-
}
|
|
39
|
-
};
|
|
40
|
-
var DbBranchReplayError = class extends BitfabError {
|
|
41
|
-
constructor(code, message, originalTraceId, cause) {
|
|
42
|
-
super(message);
|
|
43
|
-
this.code = code;
|
|
44
|
-
this.originalTraceId = originalTraceId;
|
|
45
|
-
this.cause = cause;
|
|
46
|
-
this.name = "DbBranchReplayError";
|
|
47
|
-
}
|
|
48
|
-
};
|
|
49
|
-
function replayJsonReplacer(_key, value) {
|
|
50
|
-
if (value instanceof Error) {
|
|
51
|
-
const serialized = {
|
|
52
|
-
name: value.name,
|
|
53
|
-
message: value.message,
|
|
54
|
-
stack: value.stack
|
|
55
|
-
};
|
|
56
|
-
if (value instanceof DbBranchReplayError) {
|
|
57
|
-
serialized.code = value.code;
|
|
58
|
-
serialized.originalTraceId = value.originalTraceId;
|
|
59
|
-
if (value.cause !== void 0) {
|
|
60
|
-
serialized.cause = value.cause;
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
return serialized;
|
|
64
|
-
}
|
|
65
|
-
return value;
|
|
66
|
-
}
|
|
67
|
-
function serializeReplayResult(result) {
|
|
68
|
-
return JSON.stringify(result, replayJsonReplacer, 2);
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
// src/replayRegistry.ts
|
|
72
|
-
var VALUE_FLAGS = /* @__PURE__ */ new Set([
|
|
73
|
-
"--limit",
|
|
74
|
-
"--attempts",
|
|
75
|
-
"--trace-ids",
|
|
76
|
-
"--name",
|
|
77
|
-
"--concurrency",
|
|
78
|
-
"--max-concurrency",
|
|
79
|
-
"--code-change",
|
|
80
|
-
"--experiment-group-id",
|
|
81
|
-
"--dataset-id",
|
|
82
|
-
"--dataset-ids",
|
|
83
|
-
"--grader-ids",
|
|
84
|
-
"--mock",
|
|
85
|
-
"--param",
|
|
86
|
-
"--params"
|
|
87
|
-
]);
|
|
88
|
-
var BOOLEAN_FLAGS = /* @__PURE__ */ new Set([
|
|
89
|
-
"--only-with-assertions",
|
|
90
|
-
"--db-branch",
|
|
91
|
-
"--no-db-branch",
|
|
92
|
-
"--no-code-change",
|
|
93
|
-
"--dry-run"
|
|
94
|
-
]);
|
|
95
|
-
var HELP_FLAGS = /* @__PURE__ */ new Set(["--help", "-h"]);
|
|
96
|
-
var MAX_ATTEMPTS = 100;
|
|
97
|
-
var MAX_PINNED_TRACE_IDS = 100;
|
|
98
|
-
async function pinnedDatasetMembers(client, datasetIds, limit) {
|
|
99
|
-
const members = /* @__PURE__ */ new Set();
|
|
100
|
-
for (const datasetId of datasetIds) {
|
|
101
|
-
const { traceIds } = await client.datasets.listTraces(datasetId);
|
|
102
|
-
for (const traceId of traceIds) {
|
|
103
|
-
members.add(traceId);
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
const ordered = [...members].sort();
|
|
107
|
-
if (limit >= ordered.length) {
|
|
108
|
-
return void 0;
|
|
109
|
-
}
|
|
110
|
-
if (limit > MAX_PINNED_TRACE_IDS) {
|
|
111
|
-
throw new BitfabError(
|
|
112
|
-
`Bounding a dataset replay to ${limit} names one trace ID per replayed item, and a replay accepts at most ${MAX_PINNED_TRACE_IDS}. Drop the limit to replay every selected dataset in full.`
|
|
113
|
-
);
|
|
114
|
-
}
|
|
115
|
-
return ordered.slice(0, limit);
|
|
116
|
-
}
|
|
117
|
-
function usage(registry) {
|
|
118
|
-
return `Usage: bitfab-replay --registry <path> <${Object.keys(registry).join("|")}> [options]
|
|
119
|
-
|
|
120
|
-
Options:
|
|
121
|
-
--limit N (bounds a trace ID or dataset selection instead of replacing it)
|
|
122
|
-
--attempts N (replay each trace N times in this run, max ${MAX_ATTEMPTS})
|
|
123
|
-
--trace-ids id1,id2
|
|
124
|
-
--name NAME
|
|
125
|
-
--concurrency N, --max-concurrency N
|
|
126
|
-
--code-change PATH, --no-code-change
|
|
127
|
-
--experiment-group-id UUID
|
|
128
|
-
--dataset-ids uuid1,uuid2 (--dataset-id is the same flag)
|
|
129
|
-
--grader-ids id1,id2
|
|
130
|
-
--only-with-assertions (replay only traces carrying an assertion)
|
|
131
|
-
--mock none|all|marked
|
|
132
|
-
--db-branch, --no-db-branch
|
|
133
|
-
--dry-run (resolve inputs, run nothing)
|
|
134
|
-
--params PATH
|
|
135
|
-
--param name=value (repeatable)
|
|
136
|
-
-h, --help`;
|
|
137
|
-
}
|
|
138
|
-
var ReplayCliHelp = class extends Error {
|
|
139
|
-
};
|
|
140
|
-
function parseSeedCases(raw, path, run) {
|
|
141
|
-
const trimmed = raw.trim();
|
|
142
|
-
if (trimmed.length === 0) {
|
|
143
|
-
throw new BitfabError(`Seed file '${path}' is empty.`);
|
|
144
|
-
}
|
|
145
|
-
const values = trimmed.startsWith("[") ? JSON.parse(trimmed) : trimmed.split("\n").map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
|
|
146
|
-
return values.map((value, index) => {
|
|
147
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
148
|
-
throw new BitfabError(
|
|
149
|
-
`Seed case ${index} in '${path}' must be an object with an "input" array.`
|
|
150
|
-
);
|
|
151
|
-
}
|
|
152
|
-
const { input } = value;
|
|
153
|
-
if (!Array.isArray(input)) {
|
|
154
|
-
throw new BitfabError(
|
|
155
|
-
`Seed case ${index} in '${path}' is missing an "input" array. Wrap a single argument as [arg].`
|
|
156
|
-
);
|
|
157
|
-
}
|
|
158
|
-
if (run && "expected" in value) {
|
|
159
|
-
throw new BitfabError(
|
|
160
|
-
`Seed case ${index} in '${path}' carries "expected", which --run does not record: the case is run once and its output is what the run produced. Remove the field, or drop --run to record the case without running it.`
|
|
161
|
-
);
|
|
162
|
-
}
|
|
163
|
-
return value;
|
|
164
|
-
});
|
|
165
|
-
}
|
|
166
|
-
async function seedFromRegistry(registry, pipeline, cases, options = {}) {
|
|
167
|
-
const registration = registry[pipeline];
|
|
168
|
-
if (registration === void 0) {
|
|
169
|
-
throw new BitfabError(
|
|
170
|
-
`Unknown pipeline '${pipeline}'. Registered: ${Object.keys(registry).join(", ")}`
|
|
171
|
-
);
|
|
172
|
-
}
|
|
173
|
-
const traceFunctionKey = resolveTraceFunctionKey(registration);
|
|
174
|
-
if (options.run === true) {
|
|
175
|
-
const traceIds2 = [];
|
|
176
|
-
for (const seedCase of cases) {
|
|
177
|
-
traceIds2.push(
|
|
178
|
-
await registration.client.seedTrace(traceFunctionKey, registration.fn, {
|
|
179
|
-
args: seedCase.input,
|
|
180
|
-
metadata: seedCase.metadata,
|
|
181
|
-
sessionId: seedCase.sessionId
|
|
182
|
-
})
|
|
183
|
-
);
|
|
184
|
-
}
|
|
185
|
-
return { pipeline, traceFunctionKey, traceIds: traceIds2 };
|
|
186
|
-
}
|
|
187
|
-
const traceIds = cases.map(
|
|
188
|
-
(seedCase) => registration.client.seedTrace(traceFunctionKey, {
|
|
189
|
-
input: seedCase.input,
|
|
190
|
-
expected: seedCase.expected,
|
|
191
|
-
fn: registration.fn,
|
|
192
|
-
metadata: seedCase.metadata,
|
|
193
|
-
sessionId: seedCase.sessionId
|
|
194
|
-
})
|
|
195
|
-
);
|
|
196
|
-
const { flushTraces: flushTraces2 } = await import("./http-2NFFBFYV.js");
|
|
197
|
-
await flushTraces2(3e4);
|
|
198
|
-
return { pipeline, traceFunctionKey, traceIds };
|
|
199
|
-
}
|
|
200
|
-
var SEED_USAGE = 'Usage: bitfab-replay --registry <path> <pipeline> --seed <cases.jsonl> [--run]\n\nEach case is a JSON object with an "input" array, plus optional "expected", "metadata", and "sessionId". A JSON array of those objects works too.\n\nWithout --run, each case is written as a trace directly: "input" becomes the root span input and "expected" its output, and nothing executes.\n\nWith --run, each case is run once through the registered function with capture off and the execution is recorded, so the output is what the run produced. Cases carrying "expected" are rejected.';
|
|
201
|
-
async function runSeedCli(registry, argv, io = {}) {
|
|
202
|
-
const stdout = io.stdout ?? console.log;
|
|
203
|
-
const stderr = io.stderr ?? console.error;
|
|
204
|
-
if (argv.some((value) => HELP_FLAGS.has(value))) {
|
|
205
|
-
throw new ReplayCliHelp(SEED_USAGE);
|
|
206
|
-
}
|
|
207
|
-
const pipeline = argv[0];
|
|
208
|
-
if (pipeline === void 0 || registry[pipeline] === void 0) {
|
|
209
|
-
throw new BitfabError(SEED_USAGE);
|
|
210
|
-
}
|
|
211
|
-
const seedIndex = argv.indexOf("--seed");
|
|
212
|
-
const casesPath = argv[seedIndex + 1];
|
|
213
|
-
if (casesPath === void 0 || casesPath.startsWith("--")) {
|
|
214
|
-
throw new BitfabError("--seed requires a path to a cases file.");
|
|
215
|
-
}
|
|
216
|
-
const run = argv.includes("--run");
|
|
217
|
-
const readFile = io.readFile ?? defaultReadFile;
|
|
218
|
-
const cases = parseSeedCases(await readFile(casesPath), casesPath, run);
|
|
219
|
-
stderr(
|
|
220
|
-
run ? `[seed] Running ${cases.length} case(s) through "${pipeline}"...` : `[seed] Seeding ${cases.length} case(s) into "${pipeline}"...`
|
|
221
|
-
);
|
|
222
|
-
const result = await seedFromRegistry(registry, pipeline, cases, { run });
|
|
223
|
-
stderr(
|
|
224
|
-
`[seed] ${run ? "Recorded" : "Wrote"} ${result.traceIds.length} trace(s) for "${result.traceFunctionKey}". Replay them with --trace-ids ${result.traceIds.slice(0, 3).join(",")}${result.traceIds.length > 3 ? ",..." : ""}`
|
|
225
|
-
);
|
|
226
|
-
stdout(JSON.stringify(result, null, 2));
|
|
227
|
-
return result;
|
|
228
|
-
}
|
|
229
|
-
function requirePositiveInteger(flag, raw, max) {
|
|
230
|
-
const value = Number(raw);
|
|
231
|
-
if (!Number.isInteger(value) || value < 1) {
|
|
232
|
-
throw new BitfabError(
|
|
233
|
-
`${flag} must be a positive integer (received '${raw}').`
|
|
234
|
-
);
|
|
235
|
-
}
|
|
236
|
-
if (max !== void 0 && value > max) {
|
|
237
|
-
throw new BitfabError(`${flag} must be at most ${max} (received '${raw}').`);
|
|
238
|
-
}
|
|
239
|
-
return value;
|
|
240
|
-
}
|
|
241
|
-
function commaSeparated(flag, raw) {
|
|
242
|
-
const values = raw.split(",").map((value) => value.trim()).filter((value) => value.length > 0);
|
|
243
|
-
if (values.length === 0) {
|
|
244
|
-
throw new BitfabError(`${flag} must contain at least one value.`);
|
|
245
|
-
}
|
|
246
|
-
return values;
|
|
247
|
-
}
|
|
248
|
-
function parseReplayCliArgs(registry, argv) {
|
|
249
|
-
if (argv.some((value) => HELP_FLAGS.has(value))) {
|
|
250
|
-
throw new ReplayCliHelp(usage(registry));
|
|
251
|
-
}
|
|
252
|
-
const pipeline = argv[0];
|
|
253
|
-
if (pipeline === void 0 || registry[pipeline] === void 0) {
|
|
254
|
-
throw new BitfabError(usage(registry));
|
|
255
|
-
}
|
|
256
|
-
const parsed = { pipeline, params: [] };
|
|
257
|
-
for (let index = 1; index < argv.length; index += 1) {
|
|
258
|
-
const flag = argv[index];
|
|
259
|
-
if (BOOLEAN_FLAGS.has(flag)) {
|
|
260
|
-
switch (flag) {
|
|
261
|
-
case "--db-branch":
|
|
262
|
-
if (parsed.dbBranch === false) {
|
|
263
|
-
throw new BitfabError(
|
|
264
|
-
"--db-branch and --no-db-branch cannot be used together."
|
|
265
|
-
);
|
|
266
|
-
}
|
|
267
|
-
parsed.dbBranch = true;
|
|
268
|
-
break;
|
|
269
|
-
case "--no-db-branch":
|
|
270
|
-
if (parsed.dbBranch === true) {
|
|
271
|
-
throw new BitfabError(
|
|
272
|
-
"--db-branch and --no-db-branch cannot be used together."
|
|
273
|
-
);
|
|
274
|
-
}
|
|
275
|
-
parsed.dbBranch = false;
|
|
276
|
-
break;
|
|
277
|
-
case "--only-with-assertions":
|
|
278
|
-
parsed.onlyWithAssertions = true;
|
|
279
|
-
break;
|
|
280
|
-
case "--no-code-change":
|
|
281
|
-
parsed.noCodeChange = true;
|
|
282
|
-
break;
|
|
283
|
-
case "--dry-run":
|
|
284
|
-
parsed.dryRun = true;
|
|
285
|
-
break;
|
|
286
|
-
}
|
|
287
|
-
continue;
|
|
288
|
-
}
|
|
289
|
-
if (!VALUE_FLAGS.has(flag)) {
|
|
290
|
-
throw new BitfabError(
|
|
291
|
-
`Unknown replay option '${flag}'.
|
|
292
|
-
${usage(registry)}`
|
|
293
|
-
);
|
|
294
|
-
}
|
|
295
|
-
const raw = argv[index + 1];
|
|
296
|
-
if (raw === void 0 || raw.startsWith("--")) {
|
|
297
|
-
throw new BitfabError(`${flag} requires a value.`);
|
|
298
|
-
}
|
|
299
|
-
index += 1;
|
|
300
|
-
switch (flag) {
|
|
301
|
-
case "--limit":
|
|
302
|
-
parsed.limit = requirePositiveInteger(flag, raw);
|
|
303
|
-
break;
|
|
304
|
-
case "--attempts":
|
|
305
|
-
parsed.attempts = requirePositiveInteger(flag, raw, MAX_ATTEMPTS);
|
|
306
|
-
break;
|
|
307
|
-
case "--trace-ids":
|
|
308
|
-
parsed.traceIds = commaSeparated(flag, raw);
|
|
309
|
-
break;
|
|
310
|
-
case "--name":
|
|
311
|
-
parsed.name = raw;
|
|
312
|
-
break;
|
|
313
|
-
case "--concurrency":
|
|
314
|
-
case "--max-concurrency":
|
|
315
|
-
parsed.maxConcurrency = requirePositiveInteger(flag, raw);
|
|
316
|
-
break;
|
|
317
|
-
case "--code-change":
|
|
318
|
-
parsed.codeChangePath = raw;
|
|
319
|
-
break;
|
|
320
|
-
case "--experiment-group-id":
|
|
321
|
-
parsed.experimentGroupId = raw;
|
|
322
|
-
break;
|
|
323
|
-
case "--dataset-id":
|
|
324
|
-
case "--dataset-ids":
|
|
325
|
-
parsed.datasetIds = commaSeparated(flag, raw);
|
|
326
|
-
break;
|
|
327
|
-
case "--grader-ids":
|
|
328
|
-
parsed.graderIds = commaSeparated(flag, raw);
|
|
329
|
-
break;
|
|
330
|
-
case "--mock":
|
|
331
|
-
if (raw !== "none" && raw !== "all" && raw !== "marked") {
|
|
332
|
-
throw new BitfabError(
|
|
333
|
-
`--mock must be one of: none, all, marked (received '${raw}').`
|
|
334
|
-
);
|
|
335
|
-
}
|
|
336
|
-
parsed.mock = raw;
|
|
337
|
-
break;
|
|
338
|
-
case "--params":
|
|
339
|
-
parsed.paramsPath = raw;
|
|
340
|
-
break;
|
|
341
|
-
case "--param":
|
|
342
|
-
parsed.params.push(raw);
|
|
343
|
-
break;
|
|
344
|
-
}
|
|
345
|
-
}
|
|
346
|
-
if (parsed.traceIds !== void 0 && parsed.datasetIds !== void 0) {
|
|
347
|
-
throw new BitfabError(
|
|
348
|
-
"--trace-ids and --dataset-ids select different replay sources and cannot be used together."
|
|
349
|
-
);
|
|
350
|
-
}
|
|
351
|
-
if (parsed.codeChangePath !== void 0 && parsed.noCodeChange === true) {
|
|
352
|
-
throw new BitfabError(
|
|
353
|
-
"--code-change and --no-code-change cannot be used together."
|
|
354
|
-
);
|
|
355
|
-
}
|
|
356
|
-
return parsed;
|
|
357
|
-
}
|
|
358
|
-
function resolveRegistrationDatasetIds(options) {
|
|
359
|
-
if (options.datasetId !== void 0 && options.datasetIds !== void 0) {
|
|
360
|
-
throw new BitfabError(
|
|
361
|
-
"Replay registry options datasetId and datasetIds are the same selector: set one of them."
|
|
362
|
-
);
|
|
363
|
-
}
|
|
364
|
-
if (options.datasetId !== void 0) {
|
|
365
|
-
return [options.datasetId];
|
|
366
|
-
}
|
|
367
|
-
return options.datasetIds;
|
|
368
|
-
}
|
|
369
|
-
function describeSelection(options) {
|
|
370
|
-
if (options.traceIds !== void 0) {
|
|
371
|
-
return `${options.traceIds.length} traces`;
|
|
372
|
-
}
|
|
373
|
-
if (options.limit !== void 0) {
|
|
374
|
-
return `${options.limit} traces`;
|
|
375
|
-
}
|
|
376
|
-
const datasetCount = options.datasetIds?.length ?? 1;
|
|
377
|
-
return datasetCount > 1 ? `${datasetCount} datasets' traces` : "dataset traces";
|
|
378
|
-
}
|
|
379
|
-
function resolveTraceFunctionKey(registration) {
|
|
380
|
-
const wrappedKey = registration.fn._bitfabTraceFunctionKey;
|
|
381
|
-
const key = registration.traceFunctionKey ?? wrappedKey;
|
|
382
|
-
if (key === void 0) {
|
|
383
|
-
throw new BitfabError(
|
|
384
|
-
"Replay registry entry uses a plain function. Set traceFunctionKey to the key its production handler records."
|
|
385
|
-
);
|
|
386
|
-
}
|
|
387
|
-
return key;
|
|
388
|
-
}
|
|
389
|
-
async function defaultReadFile(path) {
|
|
390
|
-
const fs = await import("fs/promises").catch(() => null);
|
|
391
|
-
if (fs === null) {
|
|
392
|
-
throw new BitfabError(
|
|
393
|
-
"--code-change requires a runtime that can read local files."
|
|
394
|
-
);
|
|
395
|
-
}
|
|
396
|
-
return fs.readFile(path, "utf8");
|
|
397
|
-
}
|
|
398
|
-
async function loadCodeChange(path, readFile) {
|
|
399
|
-
if (path === void 0) {
|
|
400
|
-
return void 0;
|
|
401
|
-
}
|
|
402
|
-
const value = JSON.parse(await readFile(path));
|
|
403
|
-
if (typeof value.description !== "string" || !Array.isArray(value.files)) {
|
|
404
|
-
throw new BitfabError(
|
|
405
|
-
`Invalid --code-change file '${path}': expected { description, files }.`
|
|
406
|
-
);
|
|
407
|
-
}
|
|
408
|
-
return { description: value.description, files: value.files };
|
|
409
|
-
}
|
|
410
|
-
function parseParameter(raw) {
|
|
411
|
-
const separator = raw.indexOf("=");
|
|
412
|
-
const key = raw.slice(0, separator).trim();
|
|
413
|
-
if (separator < 1 || key.length === 0) {
|
|
414
|
-
throw new BitfabError(
|
|
415
|
-
`Invalid --param '${raw}': expected a non-empty name=value pair.`
|
|
416
|
-
);
|
|
417
|
-
}
|
|
418
|
-
const value = raw.slice(separator + 1);
|
|
419
|
-
try {
|
|
420
|
-
return [key, JSON.parse(value)];
|
|
421
|
-
} catch {
|
|
422
|
-
return [key, value];
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
async function loadParameters(path, rawParameters, readFile) {
|
|
426
|
-
let params = {};
|
|
427
|
-
if (path !== void 0) {
|
|
428
|
-
const value = JSON.parse(await readFile(path));
|
|
429
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
430
|
-
throw new BitfabError(
|
|
431
|
-
`Invalid --params file '${path}': expected a JSON object.`
|
|
432
|
-
);
|
|
433
|
-
}
|
|
434
|
-
params = { ...value };
|
|
435
|
-
}
|
|
436
|
-
for (const raw of rawParameters) {
|
|
437
|
-
const [key, value] = parseParameter(raw);
|
|
438
|
-
params[key] = value;
|
|
439
|
-
}
|
|
440
|
-
return params;
|
|
441
|
-
}
|
|
442
|
-
function valuesEqual(left, right) {
|
|
443
|
-
try {
|
|
444
|
-
return JSON.stringify(left) === JSON.stringify(right);
|
|
445
|
-
} catch {
|
|
446
|
-
return Object.is(left, right);
|
|
447
|
-
}
|
|
448
|
-
}
|
|
449
|
-
function renderSummary(pipeline, result, stderr) {
|
|
450
|
-
let same = 0;
|
|
451
|
-
let changed = 0;
|
|
452
|
-
let matched = 0;
|
|
453
|
-
let missed = 0;
|
|
454
|
-
let errors = 0;
|
|
455
|
-
for (const item of result.items) {
|
|
456
|
-
if (item.error !== null) {
|
|
457
|
-
errors += 1;
|
|
458
|
-
continue;
|
|
459
|
-
}
|
|
460
|
-
const equal = valuesEqual(item.result, item.originalOutput);
|
|
461
|
-
if (item.ingestionType === "seeded") {
|
|
462
|
-
if (equal) {
|
|
463
|
-
matched += 1;
|
|
464
|
-
} else {
|
|
465
|
-
missed += 1;
|
|
466
|
-
}
|
|
467
|
-
} else if (equal) {
|
|
468
|
-
same += 1;
|
|
469
|
-
} else {
|
|
470
|
-
changed += 1;
|
|
471
|
-
}
|
|
472
|
-
}
|
|
473
|
-
stderr("\n\u2500\u2500\u2500 Summary \u2500\u2500\u2500");
|
|
474
|
-
stderr(` Pipeline: ${pipeline}`);
|
|
475
|
-
stderr(` Replayed: ${result.items.length}`);
|
|
476
|
-
if (result.attempts > 1) {
|
|
477
|
-
stderr(` Attempts: ${result.attempts}`);
|
|
478
|
-
}
|
|
479
|
-
if (same > 0 || changed > 0 || matched + missed === 0) {
|
|
480
|
-
stderr(` Same: ${same}`);
|
|
481
|
-
stderr(` Changed: ${changed}`);
|
|
482
|
-
}
|
|
483
|
-
if (matched > 0 || missed > 0) {
|
|
484
|
-
stderr(` Matched expected: ${matched}`);
|
|
485
|
-
stderr(` Missed expected: ${missed}`);
|
|
486
|
-
}
|
|
487
|
-
if (errors > 0) {
|
|
488
|
-
stderr(` Errors: ${errors}`);
|
|
489
|
-
}
|
|
490
|
-
stderr(`
|
|
491
|
-
${result.testRunUrl}`);
|
|
492
|
-
}
|
|
493
|
-
function renderDryRun(pipeline, result, stderr) {
|
|
494
|
-
stderr("\n\u2500\u2500\u2500 Dry run \u2500\u2500\u2500");
|
|
495
|
-
stderr(` Pipeline: ${pipeline}`);
|
|
496
|
-
stderr(` Resolved: ${result.items.length} (nothing was executed)`);
|
|
497
|
-
for (const item of result.items) {
|
|
498
|
-
stderr(`
|
|
499
|
-
${item.originalTraceId}`);
|
|
500
|
-
stderr(` args: ${safeJson(item.input)}`);
|
|
501
|
-
}
|
|
502
|
-
stderr(`
|
|
503
|
-
${result.testRunUrl}`);
|
|
504
|
-
}
|
|
505
|
-
function safeJson(value) {
|
|
506
|
-
try {
|
|
507
|
-
return JSON.stringify(value) ?? String(value);
|
|
508
|
-
} catch {
|
|
509
|
-
return String(value);
|
|
510
|
-
}
|
|
511
|
-
}
|
|
512
|
-
function reportReplayError(error, stderr) {
|
|
513
|
-
for (const item of error.items) {
|
|
514
|
-
stderr(
|
|
515
|
-
`${item.originalTraceId}: ${String(item.traceError ?? item.replayError ?? item.error)}`
|
|
516
|
-
);
|
|
517
|
-
}
|
|
518
|
-
}
|
|
519
|
-
async function runReplayCli(registry, argv = typeof process === "undefined" ? [] : process.argv.slice(2), io = {}) {
|
|
520
|
-
const stdout = io.stdout ?? console.log;
|
|
521
|
-
const stderr = io.stderr ?? console.error;
|
|
522
|
-
const args = parseReplayCliArgs(registry, argv);
|
|
523
|
-
const registration = registry[args.pipeline];
|
|
524
|
-
const traceFunctionKey = resolveTraceFunctionKey(registration);
|
|
525
|
-
const readFile = io.readFile ?? defaultReadFile;
|
|
526
|
-
const params = await loadParameters(args.paramsPath, args.params, readFile);
|
|
527
|
-
const dynamicOptions = await registration.optionsFactory?.({ params });
|
|
528
|
-
const registrationOptions = {
|
|
529
|
-
...registration.options,
|
|
530
|
-
...dynamicOptions
|
|
531
|
-
};
|
|
532
|
-
const registrationDatasetIds = resolveRegistrationDatasetIds(registrationOptions);
|
|
533
|
-
if (registrationOptions.traceIds !== void 0 && registrationDatasetIds !== void 0) {
|
|
534
|
-
throw new BitfabError(
|
|
535
|
-
"Replay registry options traceIds and datasetIds select different sources and cannot be used together."
|
|
536
|
-
);
|
|
537
|
-
}
|
|
538
|
-
const selectedDatasetIds = args.datasetIds ?? registrationDatasetIds;
|
|
539
|
-
const codeChange = await loadCodeChange(args.codeChangePath, readFile);
|
|
540
|
-
const options = {
|
|
541
|
-
...registrationOptions,
|
|
542
|
-
...args.name === void 0 ? {} : { name: args.name },
|
|
543
|
-
...args.attempts === void 0 ? {} : { attempts: args.attempts },
|
|
544
|
-
...args.maxConcurrency === void 0 ? {} : { maxConcurrency: args.maxConcurrency },
|
|
545
|
-
...args.experimentGroupId === void 0 ? {} : { experimentGroupId: args.experimentGroupId },
|
|
546
|
-
datasetId: void 0,
|
|
547
|
-
datasetIds: selectedDatasetIds,
|
|
548
|
-
...args.graderIds === void 0 ? {} : { graderIds: args.graderIds },
|
|
549
|
-
...args.onlyWithAssertions === void 0 ? {} : { onlyWithAssertions: args.onlyWithAssertions },
|
|
550
|
-
...args.mock === void 0 ? {} : { mock: args.mock },
|
|
551
|
-
...args.dbBranch === void 0 ? {} : {
|
|
552
|
-
dbBranch: args.dbBranch && registrationOptions.dbBranch !== void 0 && registrationOptions.dbBranch !== false ? registrationOptions.dbBranch : args.dbBranch
|
|
553
|
-
},
|
|
554
|
-
...args.noCodeChange === true ? { codeChangeDescription: null, codeChangeFiles: null } : {},
|
|
555
|
-
...args.dryRun === true ? { dryRun: true } : {},
|
|
556
|
-
...codeChange === void 0 ? {} : {
|
|
557
|
-
codeChangeDescription: codeChange.description,
|
|
558
|
-
codeChangeFiles: codeChange.files
|
|
559
|
-
},
|
|
560
|
-
onItemStart: reportReplayProgress,
|
|
561
|
-
onItemFinish: reportReplayProgress
|
|
562
|
-
};
|
|
563
|
-
if (args.traceIds !== void 0) {
|
|
564
|
-
options.traceIds = args.traceIds;
|
|
565
|
-
options.datasetIds = void 0;
|
|
566
|
-
} else if (args.datasetIds !== void 0) {
|
|
567
|
-
options.traceIds = void 0;
|
|
568
|
-
}
|
|
569
|
-
const bound = args.limit ?? registrationOptions.limit;
|
|
570
|
-
options.limit = void 0;
|
|
571
|
-
if (options.traceIds !== void 0) {
|
|
572
|
-
if (bound !== void 0) {
|
|
573
|
-
options.traceIds = options.traceIds.slice(0, bound);
|
|
574
|
-
}
|
|
575
|
-
} else if (options.datasetIds !== void 0) {
|
|
576
|
-
if (bound !== void 0) {
|
|
577
|
-
const pinned = await pinnedDatasetMembers(
|
|
578
|
-
registration.client,
|
|
579
|
-
options.datasetIds,
|
|
580
|
-
bound
|
|
581
|
-
);
|
|
582
|
-
if (pinned !== void 0) {
|
|
583
|
-
options.traceIds = pinned;
|
|
584
|
-
}
|
|
585
|
-
}
|
|
586
|
-
} else {
|
|
587
|
-
options.limit = bound ?? 10;
|
|
588
|
-
}
|
|
589
|
-
const attemptsSuffix = options.attempts !== void 0 && options.attempts > 1 ? ` (${options.attempts} attempts each)` : "";
|
|
590
|
-
stderr(
|
|
591
|
-
`[replay] ${args.dryRun === true ? "Resolving inputs for" : "Replaying"} ${describeSelection(options)} from "${traceFunctionKey}"${attemptsSuffix}...`
|
|
592
|
-
);
|
|
593
|
-
try {
|
|
594
|
-
const result = await registration.client.replay(
|
|
595
|
-
traceFunctionKey,
|
|
596
|
-
registration.fn,
|
|
597
|
-
options
|
|
598
|
-
);
|
|
599
|
-
if (args.dryRun === true) {
|
|
600
|
-
renderDryRun(args.pipeline, result, stderr);
|
|
601
|
-
} else {
|
|
602
|
-
renderSummary(args.pipeline, result, stderr);
|
|
603
|
-
}
|
|
604
|
-
stdout(serializeReplayResult(result));
|
|
605
|
-
if (result.items.length === 0) {
|
|
606
|
-
throw new BitfabError(
|
|
607
|
-
`No traces matched "${traceFunctionKey}", so nothing was replayed. Seed cases with --seed, or capture a trace first.`
|
|
608
|
-
);
|
|
609
|
-
}
|
|
610
|
-
return result;
|
|
611
|
-
} catch (error) {
|
|
612
|
-
if (error instanceof ReplayError) {
|
|
613
|
-
reportReplayError(error, stderr);
|
|
614
|
-
}
|
|
615
|
-
throw error;
|
|
616
|
-
}
|
|
617
|
-
}
|
|
3
|
+
importRegistry,
|
|
4
|
+
parseRegistryCommandArgs,
|
|
5
|
+
resolveRegistry,
|
|
6
|
+
runIfEntrypoint,
|
|
7
|
+
runRegistryMain,
|
|
8
|
+
runReplayCli,
|
|
9
|
+
runSeedCli
|
|
10
|
+
} from "./chunk-HKTODDFP.js";
|
|
11
|
+
import "./chunk-IG3CPSRQ.js";
|
|
618
12
|
|
|
619
13
|
// src/replayCli.ts
|
|
620
14
|
var USAGE = "Usage: bitfab-replay --registry <path> <pipeline> [options]\n bitfab-replay --registry <path> <pipeline> --seed <cases.jsonl> [--run]\nRun bitfab-replay --registry <path> --help to list replay options.";
|
|
621
|
-
async function parseCommandArgs(argv) {
|
|
622
|
-
if (argv.some((value) => value === "--help" || value === "-h") && !argv.includes("--registry")) {
|
|
623
|
-
throw new ReplayCliHelp(USAGE);
|
|
624
|
-
}
|
|
625
|
-
const registryFlagIndex = argv.indexOf("--registry");
|
|
626
|
-
const registryPath = argv[registryFlagIndex + 1];
|
|
627
|
-
if (registryFlagIndex === -1 || registryPath === void 0 || registryPath.startsWith("--")) {
|
|
628
|
-
throw new BitfabError(USAGE);
|
|
629
|
-
}
|
|
630
|
-
const replayArgs = argv.filter(
|
|
631
|
-
(_value, index) => index !== registryFlagIndex && index !== registryFlagIndex + 1
|
|
632
|
-
);
|
|
633
|
-
if (replayArgs.length === 0) {
|
|
634
|
-
throw new BitfabError(USAGE);
|
|
635
|
-
}
|
|
636
|
-
const { resolve } = await import("path");
|
|
637
|
-
return { registryPath: resolve(registryPath), replayArgs };
|
|
638
|
-
}
|
|
639
|
-
async function importRegistry(path) {
|
|
640
|
-
const { pathToFileURL } = await import("url");
|
|
641
|
-
return tsImport(pathToFileURL(path).href, import.meta.url);
|
|
642
|
-
}
|
|
643
|
-
function resolveRegistry(moduleValue, path) {
|
|
644
|
-
if (typeof moduleValue !== "object" || moduleValue === null) {
|
|
645
|
-
throw new BitfabError(
|
|
646
|
-
`Replay registry '${path}' must export a default registry.`
|
|
647
|
-
);
|
|
648
|
-
}
|
|
649
|
-
const registry = moduleValue.default;
|
|
650
|
-
if (typeof registry !== "object" || registry === null) {
|
|
651
|
-
throw new BitfabError(
|
|
652
|
-
`Replay registry '${path}' must export a default registry.`
|
|
653
|
-
);
|
|
654
|
-
}
|
|
655
|
-
return registry;
|
|
656
|
-
}
|
|
657
15
|
async function runReplayCommand(argv, io = {}, importer = importRegistry) {
|
|
658
|
-
const { registryPath,
|
|
16
|
+
const { registryPath, commandArgs } = await parseRegistryCommandArgs(
|
|
17
|
+
argv,
|
|
18
|
+
USAGE
|
|
19
|
+
);
|
|
659
20
|
const registry = resolveRegistry(await importer(registryPath), registryPath);
|
|
660
|
-
if (
|
|
661
|
-
return runSeedCli(registry,
|
|
21
|
+
if (commandArgs.includes("--seed")) {
|
|
22
|
+
return runSeedCli(registry, commandArgs, io);
|
|
662
23
|
}
|
|
663
|
-
return runReplayCli(registry,
|
|
24
|
+
return runReplayCli(registry, commandArgs, io);
|
|
664
25
|
}
|
|
665
26
|
async function runReplayMain(argv, io = {}) {
|
|
666
|
-
|
|
667
|
-
const stderr = io.stderr ?? console.error;
|
|
668
|
-
try {
|
|
669
|
-
await runReplayCommand(argv, io);
|
|
670
|
-
return 0;
|
|
671
|
-
} catch (error) {
|
|
672
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
673
|
-
if (error instanceof ReplayCliHelp) {
|
|
674
|
-
stdout(message);
|
|
675
|
-
return 0;
|
|
676
|
-
}
|
|
677
|
-
stderr(message);
|
|
678
|
-
return 1;
|
|
679
|
-
}
|
|
27
|
+
return runRegistryMain(() => runReplayCommand(argv, io), io);
|
|
680
28
|
}
|
|
681
|
-
|
|
682
|
-
const executablePath = process.argv[1];
|
|
683
|
-
if (executablePath === void 0) {
|
|
684
|
-
return false;
|
|
685
|
-
}
|
|
686
|
-
const [{ realpath }, { fileURLToPath }] = await Promise.all([
|
|
687
|
-
import("fs/promises"),
|
|
688
|
-
import("url")
|
|
689
|
-
]);
|
|
690
|
-
return await realpath(executablePath) === await realpath(fileURLToPath(import.meta.url));
|
|
691
|
-
}
|
|
692
|
-
void isEntrypoint().then((entrypoint) => {
|
|
693
|
-
if (entrypoint) {
|
|
694
|
-
void runReplayMain(process.argv.slice(2)).then((exitCode) => {
|
|
695
|
-
process.exitCode = exitCode;
|
|
696
|
-
});
|
|
697
|
-
}
|
|
698
|
-
});
|
|
29
|
+
runIfEntrypoint(import.meta.url, () => runReplayMain(process.argv.slice(2)));
|
|
699
30
|
export {
|
|
700
31
|
runReplayCommand,
|
|
701
32
|
runReplayMain
|