bitfab 0.52.2 → 0.52.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-XL5M3ETX.js → chunk-34B7PVAM.js} +79 -28
- package/dist/chunk-34B7PVAM.js.map +1 -0
- package/dist/{chunk-RU7KJE2D.js → chunk-AUROMI67.js} +61 -11
- package/dist/chunk-AUROMI67.js.map +1 -0
- package/dist/chunk-HKTODDFP.js +747 -0
- package/dist/chunk-HKTODDFP.js.map +1 -0
- package/dist/{chunk-ESIEHZHQ.js → chunk-IG3CPSRQ.js} +48 -21
- package/dist/chunk-IG3CPSRQ.js.map +1 -0
- package/dist/{chunk-SIMPLMRG.js → chunk-U2D7ACC4.js} +3 -2
- package/dist/chunk-U2D7ACC4.js.map +1 -0
- package/dist/{http-2NFFBFYV.js → http-F3AXLOJT.js} +2 -2
- package/dist/{http-RULNEWBV.js → http-RFNQ7LSU.js} +2 -2
- package/dist/index.cjs +134 -31
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +36 -2
- package/dist/index.d.ts +36 -2
- package/dist/index.js +5 -3
- package/dist/node.cjs +134 -31
- package/dist/node.cjs.map +1 -1
- package/dist/node.d.cts +1 -1
- package/dist/node.d.ts +1 -1
- package/dist/node.js +5 -3
- package/dist/node.js.map +1 -1
- package/dist/{replay-6SQSBWLX.js → replay-ZOTYMBMD.js} +5 -3
- package/dist/replayCli.js +18 -687
- package/dist/replayCli.js.map +1 -1
- package/dist/seedCli.js +30 -0
- package/dist/seedCli.js.map +1 -0
- package/package.json +3 -2
- package/dist/chunk-ESIEHZHQ.js.map +0 -1
- package/dist/chunk-RU7KJE2D.js.map +0 -1
- package/dist/chunk-SIMPLMRG.js.map +0 -1
- package/dist/chunk-XL5M3ETX.js.map +0 -1
- /package/dist/{http-2NFFBFYV.js.map → http-F3AXLOJT.js.map} +0 -0
- /package/dist/{http-RULNEWBV.js.map → http-RFNQ7LSU.js.map} +0 -0
- /package/dist/{replay-6SQSBWLX.js.map → replay-ZOTYMBMD.js.map} +0 -0
|
@@ -0,0 +1,747 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BitfabError
|
|
3
|
+
} from "./chunk-IG3CPSRQ.js";
|
|
4
|
+
|
|
5
|
+
// src/codeChange.ts
|
|
6
|
+
var NUL = String.fromCharCode(0);
|
|
7
|
+
|
|
8
|
+
// src/serialize.ts
|
|
9
|
+
import superjson from "superjson";
|
|
10
|
+
|
|
11
|
+
// src/replay.ts
|
|
12
|
+
var BITFAB_PROGRESS_PREFIX = "@@bitfab:progress ";
|
|
13
|
+
function reportReplayProgress(progress) {
|
|
14
|
+
const stderr = typeof process !== "undefined" ? process.stderr : void 0;
|
|
15
|
+
if (!stderr) {
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
try {
|
|
19
|
+
stderr.write(
|
|
20
|
+
`${BITFAB_PROGRESS_PREFIX}${JSON.stringify(progress, replayJsonReplacer)}
|
|
21
|
+
`
|
|
22
|
+
);
|
|
23
|
+
} catch {
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
var ReplayError = class extends BitfabError {
|
|
27
|
+
constructor(message, items, testRunId, testRunUrl, cause) {
|
|
28
|
+
super(message, testRunUrl);
|
|
29
|
+
this.items = items;
|
|
30
|
+
this.testRunId = testRunId;
|
|
31
|
+
this.testRunUrl = testRunUrl;
|
|
32
|
+
this.cause = cause;
|
|
33
|
+
this.name = "ReplayError";
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
var DbBranchReplayError = class extends BitfabError {
|
|
37
|
+
constructor(code, message, originalTraceId, cause) {
|
|
38
|
+
super(message);
|
|
39
|
+
this.code = code;
|
|
40
|
+
this.originalTraceId = originalTraceId;
|
|
41
|
+
this.cause = cause;
|
|
42
|
+
this.name = "DbBranchReplayError";
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
function replayJsonReplacer(_key, value) {
|
|
46
|
+
if (value instanceof Error) {
|
|
47
|
+
const serialized = {
|
|
48
|
+
name: value.name,
|
|
49
|
+
message: value.message,
|
|
50
|
+
stack: value.stack
|
|
51
|
+
};
|
|
52
|
+
if (value instanceof DbBranchReplayError) {
|
|
53
|
+
serialized.code = value.code;
|
|
54
|
+
serialized.originalTraceId = value.originalTraceId;
|
|
55
|
+
if (value.cause !== void 0) {
|
|
56
|
+
serialized.cause = value.cause;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return serialized;
|
|
60
|
+
}
|
|
61
|
+
return value;
|
|
62
|
+
}
|
|
63
|
+
function serializeReplayResult(result) {
|
|
64
|
+
return JSON.stringify(result, replayJsonReplacer, 2);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// src/replayRegistry.ts
|
|
68
|
+
var VALUE_FLAGS = /* @__PURE__ */ new Set([
|
|
69
|
+
"--limit",
|
|
70
|
+
"--attempts",
|
|
71
|
+
"--trace-ids",
|
|
72
|
+
"--name",
|
|
73
|
+
"--concurrency",
|
|
74
|
+
"--max-concurrency",
|
|
75
|
+
"--code-change",
|
|
76
|
+
"--experiment-group-id",
|
|
77
|
+
"--dataset-id",
|
|
78
|
+
"--dataset-ids",
|
|
79
|
+
"--grader-ids",
|
|
80
|
+
"--mock",
|
|
81
|
+
"--param",
|
|
82
|
+
"--params"
|
|
83
|
+
]);
|
|
84
|
+
var BOOLEAN_FLAGS = /* @__PURE__ */ new Set([
|
|
85
|
+
"--only-with-assertions",
|
|
86
|
+
"--db-branch",
|
|
87
|
+
"--no-db-branch",
|
|
88
|
+
"--no-code-change",
|
|
89
|
+
"--dry-run"
|
|
90
|
+
]);
|
|
91
|
+
var HELP_FLAGS = /* @__PURE__ */ new Set(["--help", "-h"]);
|
|
92
|
+
var MAX_ATTEMPTS = 100;
|
|
93
|
+
var MAX_PINNED_TRACE_IDS = 100;
|
|
94
|
+
async function pinnedDatasetMembers(client, datasetIds, limit) {
|
|
95
|
+
const members = /* @__PURE__ */ new Set();
|
|
96
|
+
for (const datasetId of datasetIds) {
|
|
97
|
+
const { traceIds } = await client.datasets.listTraces(datasetId);
|
|
98
|
+
for (const traceId of traceIds) {
|
|
99
|
+
members.add(traceId);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const ordered = [...members].sort();
|
|
103
|
+
if (limit >= ordered.length) {
|
|
104
|
+
return void 0;
|
|
105
|
+
}
|
|
106
|
+
if (limit > MAX_PINNED_TRACE_IDS) {
|
|
107
|
+
throw new BitfabError(
|
|
108
|
+
`Bounding a dataset replay to ${limit} names one trace ID per replayed item, and a replay accepts at most ${MAX_PINNED_TRACE_IDS}. Drop the limit to replay every selected dataset in full.`
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
return ordered.slice(0, limit);
|
|
112
|
+
}
|
|
113
|
+
function usage(registry) {
|
|
114
|
+
return `Usage: bitfab-replay --registry <path> <${Object.keys(registry).join("|")}> [options]
|
|
115
|
+
|
|
116
|
+
Options:
|
|
117
|
+
--limit N (bounds a trace ID or dataset selection instead of replacing it)
|
|
118
|
+
--attempts N (replay each trace N times in this run, max ${MAX_ATTEMPTS})
|
|
119
|
+
--trace-ids id1,id2
|
|
120
|
+
--name NAME
|
|
121
|
+
--concurrency N, --max-concurrency N
|
|
122
|
+
--code-change PATH, --no-code-change
|
|
123
|
+
--experiment-group-id UUID
|
|
124
|
+
--dataset-ids uuid1,uuid2 (--dataset-id is the same flag)
|
|
125
|
+
--grader-ids id1,id2
|
|
126
|
+
--only-with-assertions (replay only traces carrying an assertion)
|
|
127
|
+
--mock none|all|marked
|
|
128
|
+
--db-branch, --no-db-branch
|
|
129
|
+
--dry-run (resolve inputs, run nothing)
|
|
130
|
+
--params PATH
|
|
131
|
+
--param name=value (repeatable)
|
|
132
|
+
-h, --help`;
|
|
133
|
+
}
|
|
134
|
+
var ReplayCliHelp = class extends Error {
|
|
135
|
+
};
|
|
136
|
+
async function reseedFromRegistry(registry, pipeline, traceIds) {
|
|
137
|
+
const registration = registry[pipeline];
|
|
138
|
+
if (registration === void 0) {
|
|
139
|
+
throw new BitfabError(
|
|
140
|
+
`Unknown pipeline '${pipeline}'. Registered: ${Object.keys(registry).join(", ")}`
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
const traceFunctionKey = resolveTraceFunctionKey(registration);
|
|
144
|
+
const reseeded = [];
|
|
145
|
+
for (const traceId of traceIds) {
|
|
146
|
+
reseeded.push(
|
|
147
|
+
await registration.client.reseedTrace(traceFunctionKey, registration.fn, {
|
|
148
|
+
traceId
|
|
149
|
+
})
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
return { pipeline, traceFunctionKey, reseeded };
|
|
153
|
+
}
|
|
154
|
+
function parseSeedCases(raw, path, run) {
|
|
155
|
+
const trimmed = raw.trim();
|
|
156
|
+
if (trimmed.length === 0) {
|
|
157
|
+
throw new BitfabError(`Seed file '${path}' is empty.`);
|
|
158
|
+
}
|
|
159
|
+
const values = trimmed.startsWith("[") ? JSON.parse(trimmed) : trimmed.split("\n").map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
|
|
160
|
+
return values.map((value, index) => {
|
|
161
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
162
|
+
throw new BitfabError(
|
|
163
|
+
`Seed case ${index} in '${path}' must be an object with an "input" array.`
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
const { input } = value;
|
|
167
|
+
if (!Array.isArray(input)) {
|
|
168
|
+
throw new BitfabError(
|
|
169
|
+
`Seed case ${index} in '${path}' is missing an "input" array. Wrap a single argument as [arg].`
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
if (run && "expected" in value) {
|
|
173
|
+
throw new BitfabError(
|
|
174
|
+
`Seed case ${index} in '${path}' carries "expected", which --run does not record: the case is run once and its output is what the run produced. Remove the field, or drop --run to record the case without running it.`
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
return value;
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
async function seedFromRegistry(registry, pipeline, cases, options = {}) {
|
|
181
|
+
const registration = registry[pipeline];
|
|
182
|
+
if (registration === void 0) {
|
|
183
|
+
throw new BitfabError(
|
|
184
|
+
`Unknown pipeline '${pipeline}'. Registered: ${Object.keys(registry).join(", ")}`
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
const traceFunctionKey = resolveTraceFunctionKey(registration);
|
|
188
|
+
if (options.run === true) {
|
|
189
|
+
const traceIds2 = [];
|
|
190
|
+
for (const seedCase of cases) {
|
|
191
|
+
traceIds2.push(
|
|
192
|
+
await registration.client.seedTrace(traceFunctionKey, registration.fn, {
|
|
193
|
+
args: seedCase.input,
|
|
194
|
+
metadata: seedCase.metadata,
|
|
195
|
+
sessionId: seedCase.sessionId
|
|
196
|
+
})
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
return { pipeline, traceFunctionKey, traceIds: traceIds2 };
|
|
200
|
+
}
|
|
201
|
+
const traceIds = cases.map(
|
|
202
|
+
(seedCase) => registration.client.seedTrace(traceFunctionKey, {
|
|
203
|
+
input: seedCase.input,
|
|
204
|
+
expected: seedCase.expected,
|
|
205
|
+
fn: registration.fn,
|
|
206
|
+
metadata: seedCase.metadata,
|
|
207
|
+
sessionId: seedCase.sessionId
|
|
208
|
+
})
|
|
209
|
+
);
|
|
210
|
+
const { flushTraces: flushTraces2 } = await import("./http-F3AXLOJT.js");
|
|
211
|
+
await flushTraces2(3e4);
|
|
212
|
+
return { pipeline, traceFunctionKey, traceIds };
|
|
213
|
+
}
|
|
214
|
+
var SEED_USAGE = `Usage: bitfab-seed --registry <path> <pipeline> --from-trace <id>[,<id>...]
|
|
215
|
+
bitfab-seed --registry <path> <pipeline> --cases <cases.jsonl> [--run]
|
|
216
|
+
|
|
217
|
+
--from-trace re-seeds each trace: the registered function runs once on the trace's recorded inputs and the result is recorded under the same trace id, with the previous run kept as history under a new id. Nothing is mocked and no experiment is created. Labels, assertions, and dataset membership stay on the trace.
|
|
218
|
+
|
|
219
|
+
--cases seeds new cases. Each case is a JSON object with an "input" array, plus optional "expected", "metadata", and "sessionId"; a JSON array of those objects works too. Without --run, each case is written as a trace directly ("input" becomes the root span input and "expected" its output) and nothing executes. With --run, each case runs once through the registered function with capture off and the execution is recorded, so the output is what the run produced; cases carrying "expected" are rejected.
|
|
220
|
+
|
|
221
|
+
bitfab-replay --registry <path> <pipeline> --seed <cases.jsonl> [--run] is the older spelling of --cases.`;
|
|
222
|
+
async function runSeedCli(registry, argv, io = {}) {
|
|
223
|
+
const stdout = io.stdout ?? console.log;
|
|
224
|
+
const stderr = io.stderr ?? console.error;
|
|
225
|
+
if (argv.some((value) => HELP_FLAGS.has(value))) {
|
|
226
|
+
throw new ReplayCliHelp(SEED_USAGE);
|
|
227
|
+
}
|
|
228
|
+
const pipeline = argv[0];
|
|
229
|
+
if (pipeline === void 0 || registry[pipeline] === void 0) {
|
|
230
|
+
throw new BitfabError(SEED_USAGE);
|
|
231
|
+
}
|
|
232
|
+
const fromTraceIndex = argv.indexOf("--from-trace");
|
|
233
|
+
if (fromTraceIndex !== -1) {
|
|
234
|
+
const raw = argv[fromTraceIndex + 1];
|
|
235
|
+
if (raw === void 0 || raw.startsWith("--")) {
|
|
236
|
+
throw new BitfabError("--from-trace requires one or more trace ids.");
|
|
237
|
+
}
|
|
238
|
+
const traceIds = commaSeparated("--from-trace", raw);
|
|
239
|
+
stderr(
|
|
240
|
+
`[seed] Re-seeding ${traceIds.length} trace(s) through "${pipeline}"...`
|
|
241
|
+
);
|
|
242
|
+
const result2 = await reseedFromRegistry(registry, pipeline, traceIds);
|
|
243
|
+
for (const entry of result2.reseeded) {
|
|
244
|
+
stderr(
|
|
245
|
+
`[seed] ${entry.traceId} now holds the fresh run; the previous run is kept as ${entry.previousRunTraceId}`
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
stdout(JSON.stringify(result2, null, 2));
|
|
249
|
+
return result2;
|
|
250
|
+
}
|
|
251
|
+
const casesFlagIndex = argv.indexOf("--cases");
|
|
252
|
+
const seedIndex = casesFlagIndex === -1 ? argv.indexOf("--seed") : casesFlagIndex;
|
|
253
|
+
if (seedIndex === -1) {
|
|
254
|
+
throw new BitfabError(SEED_USAGE);
|
|
255
|
+
}
|
|
256
|
+
const casesPath = argv[seedIndex + 1];
|
|
257
|
+
if (casesPath === void 0 || casesPath.startsWith("--")) {
|
|
258
|
+
throw new BitfabError(`${argv[seedIndex]} requires a path to a cases file.`);
|
|
259
|
+
}
|
|
260
|
+
const run = argv.includes("--run");
|
|
261
|
+
const readFile = io.readFile ?? defaultReadFile;
|
|
262
|
+
const cases = parseSeedCases(await readFile(casesPath), casesPath, run);
|
|
263
|
+
stderr(
|
|
264
|
+
run ? `[seed] Running ${cases.length} case(s) through "${pipeline}"...` : `[seed] Seeding ${cases.length} case(s) into "${pipeline}"...`
|
|
265
|
+
);
|
|
266
|
+
const result = await seedFromRegistry(registry, pipeline, cases, { run });
|
|
267
|
+
stderr(
|
|
268
|
+
`[seed] ${run ? "Recorded" : "Wrote"} ${result.traceIds.length} trace(s) for "${result.traceFunctionKey}". Replay them with --trace-ids ${result.traceIds.slice(0, 3).join(",")}${result.traceIds.length > 3 ? ",..." : ""}`
|
|
269
|
+
);
|
|
270
|
+
stdout(JSON.stringify(result, null, 2));
|
|
271
|
+
return result;
|
|
272
|
+
}
|
|
273
|
+
function requirePositiveInteger(flag, raw, max) {
|
|
274
|
+
const value = Number(raw);
|
|
275
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
276
|
+
throw new BitfabError(
|
|
277
|
+
`${flag} must be a positive integer (received '${raw}').`
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
if (max !== void 0 && value > max) {
|
|
281
|
+
throw new BitfabError(`${flag} must be at most ${max} (received '${raw}').`);
|
|
282
|
+
}
|
|
283
|
+
return value;
|
|
284
|
+
}
|
|
285
|
+
function commaSeparated(flag, raw) {
|
|
286
|
+
const values = raw.split(",").map((value) => value.trim()).filter((value) => value.length > 0);
|
|
287
|
+
if (values.length === 0) {
|
|
288
|
+
throw new BitfabError(`${flag} must contain at least one value.`);
|
|
289
|
+
}
|
|
290
|
+
return values;
|
|
291
|
+
}
|
|
292
|
+
function parseReplayCliArgs(registry, argv) {
|
|
293
|
+
if (argv.some((value) => HELP_FLAGS.has(value))) {
|
|
294
|
+
throw new ReplayCliHelp(usage(registry));
|
|
295
|
+
}
|
|
296
|
+
const pipeline = argv[0];
|
|
297
|
+
if (pipeline === void 0 || registry[pipeline] === void 0) {
|
|
298
|
+
throw new BitfabError(usage(registry));
|
|
299
|
+
}
|
|
300
|
+
const parsed = { pipeline, params: [] };
|
|
301
|
+
for (let index = 1; index < argv.length; index += 1) {
|
|
302
|
+
const flag = argv[index];
|
|
303
|
+
if (BOOLEAN_FLAGS.has(flag)) {
|
|
304
|
+
switch (flag) {
|
|
305
|
+
case "--db-branch":
|
|
306
|
+
if (parsed.dbBranch === false) {
|
|
307
|
+
throw new BitfabError(
|
|
308
|
+
"--db-branch and --no-db-branch cannot be used together."
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
parsed.dbBranch = true;
|
|
312
|
+
break;
|
|
313
|
+
case "--no-db-branch":
|
|
314
|
+
if (parsed.dbBranch === true) {
|
|
315
|
+
throw new BitfabError(
|
|
316
|
+
"--db-branch and --no-db-branch cannot be used together."
|
|
317
|
+
);
|
|
318
|
+
}
|
|
319
|
+
parsed.dbBranch = false;
|
|
320
|
+
break;
|
|
321
|
+
case "--only-with-assertions":
|
|
322
|
+
parsed.onlyWithAssertions = true;
|
|
323
|
+
break;
|
|
324
|
+
case "--no-code-change":
|
|
325
|
+
parsed.noCodeChange = true;
|
|
326
|
+
break;
|
|
327
|
+
case "--dry-run":
|
|
328
|
+
parsed.dryRun = true;
|
|
329
|
+
break;
|
|
330
|
+
}
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
if (!VALUE_FLAGS.has(flag)) {
|
|
334
|
+
throw new BitfabError(
|
|
335
|
+
`Unknown replay option '${flag}'.
|
|
336
|
+
${usage(registry)}`
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
const raw = argv[index + 1];
|
|
340
|
+
if (raw === void 0 || raw.startsWith("--")) {
|
|
341
|
+
throw new BitfabError(`${flag} requires a value.`);
|
|
342
|
+
}
|
|
343
|
+
index += 1;
|
|
344
|
+
switch (flag) {
|
|
345
|
+
case "--limit":
|
|
346
|
+
parsed.limit = requirePositiveInteger(flag, raw);
|
|
347
|
+
break;
|
|
348
|
+
case "--attempts":
|
|
349
|
+
parsed.attempts = requirePositiveInteger(flag, raw, MAX_ATTEMPTS);
|
|
350
|
+
break;
|
|
351
|
+
case "--trace-ids":
|
|
352
|
+
parsed.traceIds = commaSeparated(flag, raw);
|
|
353
|
+
break;
|
|
354
|
+
case "--name":
|
|
355
|
+
parsed.name = raw;
|
|
356
|
+
break;
|
|
357
|
+
case "--concurrency":
|
|
358
|
+
case "--max-concurrency":
|
|
359
|
+
parsed.maxConcurrency = requirePositiveInteger(flag, raw);
|
|
360
|
+
break;
|
|
361
|
+
case "--code-change":
|
|
362
|
+
parsed.codeChangePath = raw;
|
|
363
|
+
break;
|
|
364
|
+
case "--experiment-group-id":
|
|
365
|
+
parsed.experimentGroupId = raw;
|
|
366
|
+
break;
|
|
367
|
+
case "--dataset-id":
|
|
368
|
+
case "--dataset-ids":
|
|
369
|
+
parsed.datasetIds = commaSeparated(flag, raw);
|
|
370
|
+
break;
|
|
371
|
+
case "--grader-ids":
|
|
372
|
+
parsed.graderIds = commaSeparated(flag, raw);
|
|
373
|
+
break;
|
|
374
|
+
case "--mock":
|
|
375
|
+
if (raw !== "none" && raw !== "all" && raw !== "marked") {
|
|
376
|
+
throw new BitfabError(
|
|
377
|
+
`--mock must be one of: none, all, marked (received '${raw}').`
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
parsed.mock = raw;
|
|
381
|
+
break;
|
|
382
|
+
case "--params":
|
|
383
|
+
parsed.paramsPath = raw;
|
|
384
|
+
break;
|
|
385
|
+
case "--param":
|
|
386
|
+
parsed.params.push(raw);
|
|
387
|
+
break;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
if (parsed.traceIds !== void 0 && parsed.datasetIds !== void 0) {
|
|
391
|
+
throw new BitfabError(
|
|
392
|
+
"--trace-ids and --dataset-ids select different replay sources and cannot be used together."
|
|
393
|
+
);
|
|
394
|
+
}
|
|
395
|
+
if (parsed.codeChangePath !== void 0 && parsed.noCodeChange === true) {
|
|
396
|
+
throw new BitfabError(
|
|
397
|
+
"--code-change and --no-code-change cannot be used together."
|
|
398
|
+
);
|
|
399
|
+
}
|
|
400
|
+
return parsed;
|
|
401
|
+
}
|
|
402
|
+
function resolveRegistrationDatasetIds(options) {
|
|
403
|
+
if (options.datasetId !== void 0 && options.datasetIds !== void 0) {
|
|
404
|
+
throw new BitfabError(
|
|
405
|
+
"Replay registry options datasetId and datasetIds are the same selector: set one of them."
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
if (options.datasetId !== void 0) {
|
|
409
|
+
return [options.datasetId];
|
|
410
|
+
}
|
|
411
|
+
return options.datasetIds;
|
|
412
|
+
}
|
|
413
|
+
function describeSelection(options) {
|
|
414
|
+
if (options.traceIds !== void 0) {
|
|
415
|
+
return `${options.traceIds.length} traces`;
|
|
416
|
+
}
|
|
417
|
+
if (options.limit !== void 0) {
|
|
418
|
+
return `${options.limit} traces`;
|
|
419
|
+
}
|
|
420
|
+
const datasetCount = options.datasetIds?.length ?? 1;
|
|
421
|
+
return datasetCount > 1 ? `${datasetCount} datasets' traces` : "dataset traces";
|
|
422
|
+
}
|
|
423
|
+
function resolveTraceFunctionKey(registration) {
|
|
424
|
+
const wrappedKey = registration.fn._bitfabTraceFunctionKey;
|
|
425
|
+
const key = registration.traceFunctionKey ?? wrappedKey;
|
|
426
|
+
if (key === void 0) {
|
|
427
|
+
throw new BitfabError(
|
|
428
|
+
"Replay registry entry uses a plain function. Set traceFunctionKey to the key its production handler records."
|
|
429
|
+
);
|
|
430
|
+
}
|
|
431
|
+
return key;
|
|
432
|
+
}
|
|
433
|
+
async function defaultReadFile(path) {
|
|
434
|
+
const fs = await import("fs/promises").catch(() => null);
|
|
435
|
+
if (fs === null) {
|
|
436
|
+
throw new BitfabError(
|
|
437
|
+
"--code-change requires a runtime that can read local files."
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
return fs.readFile(path, "utf8");
|
|
441
|
+
}
|
|
442
|
+
async function loadCodeChange(path, readFile) {
|
|
443
|
+
if (path === void 0) {
|
|
444
|
+
return void 0;
|
|
445
|
+
}
|
|
446
|
+
const value = JSON.parse(await readFile(path));
|
|
447
|
+
if (typeof value.description !== "string" || !Array.isArray(value.files)) {
|
|
448
|
+
throw new BitfabError(
|
|
449
|
+
`Invalid --code-change file '${path}': expected { description, files }.`
|
|
450
|
+
);
|
|
451
|
+
}
|
|
452
|
+
return { description: value.description, files: value.files };
|
|
453
|
+
}
|
|
454
|
+
function parseParameter(raw) {
|
|
455
|
+
const separator = raw.indexOf("=");
|
|
456
|
+
const key = raw.slice(0, separator).trim();
|
|
457
|
+
if (separator < 1 || key.length === 0) {
|
|
458
|
+
throw new BitfabError(
|
|
459
|
+
`Invalid --param '${raw}': expected a non-empty name=value pair.`
|
|
460
|
+
);
|
|
461
|
+
}
|
|
462
|
+
const value = raw.slice(separator + 1);
|
|
463
|
+
try {
|
|
464
|
+
return [key, JSON.parse(value)];
|
|
465
|
+
} catch {
|
|
466
|
+
return [key, value];
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
async function loadParameters(path, rawParameters, readFile) {
|
|
470
|
+
let params = {};
|
|
471
|
+
if (path !== void 0) {
|
|
472
|
+
const value = JSON.parse(await readFile(path));
|
|
473
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
474
|
+
throw new BitfabError(
|
|
475
|
+
`Invalid --params file '${path}': expected a JSON object.`
|
|
476
|
+
);
|
|
477
|
+
}
|
|
478
|
+
params = { ...value };
|
|
479
|
+
}
|
|
480
|
+
for (const raw of rawParameters) {
|
|
481
|
+
const [key, value] = parseParameter(raw);
|
|
482
|
+
params[key] = value;
|
|
483
|
+
}
|
|
484
|
+
return params;
|
|
485
|
+
}
|
|
486
|
+
function valuesEqual(left, right) {
|
|
487
|
+
try {
|
|
488
|
+
return JSON.stringify(left) === JSON.stringify(right);
|
|
489
|
+
} catch {
|
|
490
|
+
return Object.is(left, right);
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
function renderSummary(pipeline, result, stderr) {
|
|
494
|
+
let same = 0;
|
|
495
|
+
let changed = 0;
|
|
496
|
+
let matched = 0;
|
|
497
|
+
let missed = 0;
|
|
498
|
+
let errors = 0;
|
|
499
|
+
for (const item of result.items) {
|
|
500
|
+
if (item.error !== null) {
|
|
501
|
+
errors += 1;
|
|
502
|
+
continue;
|
|
503
|
+
}
|
|
504
|
+
const equal = valuesEqual(item.result, item.originalOutput);
|
|
505
|
+
if (item.ingestionType === "seeded") {
|
|
506
|
+
if (equal) {
|
|
507
|
+
matched += 1;
|
|
508
|
+
} else {
|
|
509
|
+
missed += 1;
|
|
510
|
+
}
|
|
511
|
+
} else if (equal) {
|
|
512
|
+
same += 1;
|
|
513
|
+
} else {
|
|
514
|
+
changed += 1;
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
stderr("\n\u2500\u2500\u2500 Summary \u2500\u2500\u2500");
|
|
518
|
+
stderr(` Pipeline: ${pipeline}`);
|
|
519
|
+
stderr(` Replayed: ${result.items.length}`);
|
|
520
|
+
if (result.attempts > 1) {
|
|
521
|
+
stderr(` Attempts: ${result.attempts}`);
|
|
522
|
+
}
|
|
523
|
+
if (same > 0 || changed > 0 || matched + missed === 0) {
|
|
524
|
+
stderr(` Same: ${same}`);
|
|
525
|
+
stderr(` Changed: ${changed}`);
|
|
526
|
+
}
|
|
527
|
+
if (matched > 0 || missed > 0) {
|
|
528
|
+
stderr(` Matched expected: ${matched}`);
|
|
529
|
+
stderr(` Missed expected: ${missed}`);
|
|
530
|
+
}
|
|
531
|
+
if (errors > 0) {
|
|
532
|
+
stderr(` Errors: ${errors}`);
|
|
533
|
+
}
|
|
534
|
+
stderr(`
|
|
535
|
+
${result.testRunUrl}`);
|
|
536
|
+
}
|
|
537
|
+
function renderDryRun(pipeline, result, stderr) {
|
|
538
|
+
stderr("\n\u2500\u2500\u2500 Dry run \u2500\u2500\u2500");
|
|
539
|
+
stderr(` Pipeline: ${pipeline}`);
|
|
540
|
+
stderr(` Resolved: ${result.items.length} (nothing was executed)`);
|
|
541
|
+
for (const item of result.items) {
|
|
542
|
+
stderr(`
|
|
543
|
+
${item.originalTraceId}`);
|
|
544
|
+
stderr(` args: ${safeJson(item.input)}`);
|
|
545
|
+
}
|
|
546
|
+
stderr(`
|
|
547
|
+
${result.testRunUrl}`);
|
|
548
|
+
}
|
|
549
|
+
function safeJson(value) {
|
|
550
|
+
try {
|
|
551
|
+
return JSON.stringify(value) ?? String(value);
|
|
552
|
+
} catch {
|
|
553
|
+
return String(value);
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
function reportReplayError(error, stderr) {
|
|
557
|
+
for (const item of error.items) {
|
|
558
|
+
stderr(
|
|
559
|
+
`${item.originalTraceId}: ${String(item.traceError ?? item.replayError ?? item.error)}`
|
|
560
|
+
);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
async function runReplayCli(registry, argv = typeof process === "undefined" ? [] : process.argv.slice(2), io = {}) {
|
|
564
|
+
const stdout = io.stdout ?? console.log;
|
|
565
|
+
const stderr = io.stderr ?? console.error;
|
|
566
|
+
const args = parseReplayCliArgs(registry, argv);
|
|
567
|
+
const registration = registry[args.pipeline];
|
|
568
|
+
const traceFunctionKey = resolveTraceFunctionKey(registration);
|
|
569
|
+
const readFile = io.readFile ?? defaultReadFile;
|
|
570
|
+
const params = await loadParameters(args.paramsPath, args.params, readFile);
|
|
571
|
+
const dynamicOptions = await registration.optionsFactory?.({ params });
|
|
572
|
+
const registrationOptions = {
|
|
573
|
+
...registration.options,
|
|
574
|
+
...dynamicOptions
|
|
575
|
+
};
|
|
576
|
+
const registrationDatasetIds = resolveRegistrationDatasetIds(registrationOptions);
|
|
577
|
+
if (registrationOptions.traceIds !== void 0 && registrationDatasetIds !== void 0) {
|
|
578
|
+
throw new BitfabError(
|
|
579
|
+
"Replay registry options traceIds and datasetIds select different sources and cannot be used together."
|
|
580
|
+
);
|
|
581
|
+
}
|
|
582
|
+
const selectedDatasetIds = args.datasetIds ?? registrationDatasetIds;
|
|
583
|
+
const codeChange = await loadCodeChange(args.codeChangePath, readFile);
|
|
584
|
+
const options = {
|
|
585
|
+
...registrationOptions,
|
|
586
|
+
...args.name === void 0 ? {} : { name: args.name },
|
|
587
|
+
...args.attempts === void 0 ? {} : { attempts: args.attempts },
|
|
588
|
+
...args.maxConcurrency === void 0 ? {} : { maxConcurrency: args.maxConcurrency },
|
|
589
|
+
...args.experimentGroupId === void 0 ? {} : { experimentGroupId: args.experimentGroupId },
|
|
590
|
+
datasetId: void 0,
|
|
591
|
+
datasetIds: selectedDatasetIds,
|
|
592
|
+
...args.graderIds === void 0 ? {} : { graderIds: args.graderIds },
|
|
593
|
+
...args.onlyWithAssertions === void 0 ? {} : { onlyWithAssertions: args.onlyWithAssertions },
|
|
594
|
+
...args.mock === void 0 ? {} : { mock: args.mock },
|
|
595
|
+
...args.dbBranch === void 0 ? {} : {
|
|
596
|
+
dbBranch: args.dbBranch && registrationOptions.dbBranch !== void 0 && registrationOptions.dbBranch !== false ? registrationOptions.dbBranch : args.dbBranch
|
|
597
|
+
},
|
|
598
|
+
...args.noCodeChange === true ? { codeChangeDescription: null, codeChangeFiles: null } : {},
|
|
599
|
+
...args.dryRun === true ? { dryRun: true } : {},
|
|
600
|
+
...codeChange === void 0 ? {} : {
|
|
601
|
+
codeChangeDescription: codeChange.description,
|
|
602
|
+
codeChangeFiles: codeChange.files
|
|
603
|
+
},
|
|
604
|
+
onItemStart: reportReplayProgress,
|
|
605
|
+
onItemFinish: reportReplayProgress
|
|
606
|
+
};
|
|
607
|
+
if (args.traceIds !== void 0) {
|
|
608
|
+
options.traceIds = args.traceIds;
|
|
609
|
+
options.datasetIds = void 0;
|
|
610
|
+
} else if (args.datasetIds !== void 0) {
|
|
611
|
+
options.traceIds = void 0;
|
|
612
|
+
}
|
|
613
|
+
const bound = args.limit ?? registrationOptions.limit;
|
|
614
|
+
options.limit = void 0;
|
|
615
|
+
if (options.traceIds !== void 0) {
|
|
616
|
+
if (bound !== void 0) {
|
|
617
|
+
options.traceIds = options.traceIds.slice(0, bound);
|
|
618
|
+
}
|
|
619
|
+
} else if (options.datasetIds !== void 0) {
|
|
620
|
+
if (bound !== void 0) {
|
|
621
|
+
const pinned = await pinnedDatasetMembers(
|
|
622
|
+
registration.client,
|
|
623
|
+
options.datasetIds,
|
|
624
|
+
bound
|
|
625
|
+
);
|
|
626
|
+
if (pinned !== void 0) {
|
|
627
|
+
options.traceIds = pinned;
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
} else {
|
|
631
|
+
options.limit = bound ?? 10;
|
|
632
|
+
}
|
|
633
|
+
const attemptsSuffix = options.attempts !== void 0 && options.attempts > 1 ? ` (${options.attempts} attempts each)` : "";
|
|
634
|
+
stderr(
|
|
635
|
+
`[replay] ${args.dryRun === true ? "Resolving inputs for" : "Replaying"} ${describeSelection(options)} from "${traceFunctionKey}"${attemptsSuffix}...`
|
|
636
|
+
);
|
|
637
|
+
try {
|
|
638
|
+
const result = await registration.client.replay(
|
|
639
|
+
traceFunctionKey,
|
|
640
|
+
registration.fn,
|
|
641
|
+
options
|
|
642
|
+
);
|
|
643
|
+
if (args.dryRun === true) {
|
|
644
|
+
renderDryRun(args.pipeline, result, stderr);
|
|
645
|
+
} else {
|
|
646
|
+
renderSummary(args.pipeline, result, stderr);
|
|
647
|
+
}
|
|
648
|
+
stdout(serializeReplayResult(result));
|
|
649
|
+
if (result.items.length === 0) {
|
|
650
|
+
throw new BitfabError(
|
|
651
|
+
`No traces matched "${traceFunctionKey}", so nothing was replayed. Seed cases with --seed, or capture a trace first.`
|
|
652
|
+
);
|
|
653
|
+
}
|
|
654
|
+
return result;
|
|
655
|
+
} catch (error) {
|
|
656
|
+
if (error instanceof ReplayError) {
|
|
657
|
+
reportReplayError(error, stderr);
|
|
658
|
+
}
|
|
659
|
+
throw error;
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
// src/registryCommand.ts
|
|
664
|
+
import { tsImport } from "tsx/esm/api";
|
|
665
|
+
async function parseRegistryCommandArgs(argv, usage2) {
|
|
666
|
+
if (argv.some((value) => value === "--help" || value === "-h") && !argv.includes("--registry")) {
|
|
667
|
+
throw new ReplayCliHelp(usage2);
|
|
668
|
+
}
|
|
669
|
+
const registryFlagIndex = argv.indexOf("--registry");
|
|
670
|
+
const registryPath = argv[registryFlagIndex + 1];
|
|
671
|
+
if (registryFlagIndex === -1 || registryPath === void 0 || registryPath.startsWith("--")) {
|
|
672
|
+
throw new BitfabError(usage2);
|
|
673
|
+
}
|
|
674
|
+
const commandArgs = argv.filter(
|
|
675
|
+
(_value, index) => index !== registryFlagIndex && index !== registryFlagIndex + 1
|
|
676
|
+
);
|
|
677
|
+
if (commandArgs.length === 0) {
|
|
678
|
+
throw new BitfabError(usage2);
|
|
679
|
+
}
|
|
680
|
+
const { resolve } = await import("path");
|
|
681
|
+
return { registryPath: resolve(registryPath), commandArgs };
|
|
682
|
+
}
|
|
683
|
+
async function importRegistry(path) {
|
|
684
|
+
const { pathToFileURL } = await import("url");
|
|
685
|
+
return tsImport(pathToFileURL(path).href, import.meta.url);
|
|
686
|
+
}
|
|
687
|
+
function resolveRegistry(moduleValue, path) {
|
|
688
|
+
if (typeof moduleValue !== "object" || moduleValue === null) {
|
|
689
|
+
throw new BitfabError(
|
|
690
|
+
`Replay registry '${path}' must export a default registry.`
|
|
691
|
+
);
|
|
692
|
+
}
|
|
693
|
+
const registry = moduleValue.default;
|
|
694
|
+
if (typeof registry !== "object" || registry === null) {
|
|
695
|
+
throw new BitfabError(
|
|
696
|
+
`Replay registry '${path}' must export a default registry.`
|
|
697
|
+
);
|
|
698
|
+
}
|
|
699
|
+
return registry;
|
|
700
|
+
}
|
|
701
|
+
async function runRegistryMain(run, io = {}) {
|
|
702
|
+
const stdout = io.stdout ?? console.log;
|
|
703
|
+
const stderr = io.stderr ?? console.error;
|
|
704
|
+
try {
|
|
705
|
+
await run();
|
|
706
|
+
return 0;
|
|
707
|
+
} catch (error) {
|
|
708
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
709
|
+
if (error instanceof ReplayCliHelp) {
|
|
710
|
+
stdout(message);
|
|
711
|
+
return 0;
|
|
712
|
+
}
|
|
713
|
+
stderr(message);
|
|
714
|
+
return 1;
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
async function isEntrypoint(moduleUrl) {
|
|
718
|
+
const executablePath = process.argv[1];
|
|
719
|
+
if (executablePath === void 0) {
|
|
720
|
+
return false;
|
|
721
|
+
}
|
|
722
|
+
const [{ realpath }, { fileURLToPath }] = await Promise.all([
|
|
723
|
+
import("fs/promises"),
|
|
724
|
+
import("url")
|
|
725
|
+
]);
|
|
726
|
+
return await realpath(executablePath) === await realpath(fileURLToPath(moduleUrl));
|
|
727
|
+
}
|
|
728
|
+
function runIfEntrypoint(moduleUrl, main) {
|
|
729
|
+
void isEntrypoint(moduleUrl).then((entrypoint) => {
|
|
730
|
+
if (entrypoint) {
|
|
731
|
+
void main().then((exitCode) => {
|
|
732
|
+
process.exitCode = exitCode;
|
|
733
|
+
});
|
|
734
|
+
}
|
|
735
|
+
});
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
export {
|
|
739
|
+
runSeedCli,
|
|
740
|
+
runReplayCli,
|
|
741
|
+
parseRegistryCommandArgs,
|
|
742
|
+
importRegistry,
|
|
743
|
+
resolveRegistry,
|
|
744
|
+
runRegistryMain,
|
|
745
|
+
runIfEntrypoint
|
|
746
|
+
};
|
|
747
|
+
//# sourceMappingURL=chunk-HKTODDFP.js.map
|