@bitfab/sdk 0.64.16 → 0.64.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cloudReplay.py +286 -24
- package/dist/{chunk-PAWBCEI7.js → chunk-4JZ6UMDW.js} +2 -2
- package/dist/{chunk-PAWBCEI7.js.map → chunk-4JZ6UMDW.js.map} +1 -1
- package/dist/{chunk-LOIVELPI.js → chunk-6AUI3X66.js} +204 -89
- package/dist/chunk-6AUI3X66.js.map +1 -0
- package/dist/{chunk-I4TFQZCW.js → chunk-7NPHFRQD.js} +16 -2
- package/dist/{chunk-I4TFQZCW.js.map → chunk-7NPHFRQD.js.map} +1 -1
- package/dist/{chunk-NNZAQSTW.js → chunk-ACKEEC7I.js} +2 -2
- package/dist/chunk-NGKTKRCV.js +1905 -0
- package/dist/chunk-NGKTKRCV.js.map +1 -0
- package/dist/{chunk-AXFKZYIY.js → chunk-ZBA4QJWF.js} +11 -6
- package/dist/chunk-ZBA4QJWF.js.map +1 -0
- package/dist/{http-QQ6VVHUZ.js → http-2BXJLZ5N.js} +2 -2
- package/dist/{http-LV7OUBKA.js → http-6VZBSALH.js} +2 -2
- package/dist/index.cjs +6 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +3 -3
- package/dist/node.cjs +6 -1
- package/dist/node.cjs.map +1 -1
- package/dist/node.js +3 -3
- package/dist/{replay-JNKXZPY7.js → replay-OO2DFUAQ.js} +3 -3
- package/dist/replay-Z46UJQVQ.js +28 -0
- package/dist/replay-Z46UJQVQ.js.map +1 -0
- package/dist/replayCli.js +3 -2
- package/dist/replayCli.js.map +1 -1
- package/dist/seedCli.js +3 -2
- package/dist/seedCli.js.map +1 -1
- package/package.json +1 -1
- package/dist/chunk-AXFKZYIY.js.map +0 -1
- package/dist/chunk-LOIVELPI.js.map +0 -1
- /package/dist/{chunk-NNZAQSTW.js.map → chunk-ACKEEC7I.js.map} +0 -0
- /package/dist/{http-LV7OUBKA.js.map → http-2BXJLZ5N.js.map} +0 -0
- /package/dist/{http-QQ6VVHUZ.js.map → http-6VZBSALH.js.map} +0 -0
- /package/dist/{replay-JNKXZPY7.js.map → replay-OO2DFUAQ.js.map} +0 -0
|
@@ -0,0 +1,1905 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BitfabError,
|
|
3
|
+
__version__,
|
|
4
|
+
flushTraces,
|
|
5
|
+
gitRunner,
|
|
6
|
+
readEnv,
|
|
7
|
+
replayContextReady,
|
|
8
|
+
runWithReplayContext,
|
|
9
|
+
unrefTimer,
|
|
10
|
+
warnOnce,
|
|
11
|
+
writeOnce
|
|
12
|
+
} from "./chunk-7NPHFRQD.js";
|
|
13
|
+
|
|
14
|
+
// src/codeChange.ts
|
|
15
|
+
var MAX_FILES = 60;
|
|
16
|
+
var MAX_FILE_BYTES = 5e5;
|
|
17
|
+
var MAX_TOTAL_BYTES = 2e6;
|
|
18
|
+
var NUL = String.fromCharCode(0);
|
|
19
|
+
async function resolveAutoCodeChange(label) {
|
|
20
|
+
if (typeof process === "undefined") {
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
if (process.env?.BITFAB_DISABLE_CODE_CHANGE_CAPTURE) {
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
const fromEnv = await readCodeChangeFile();
|
|
27
|
+
if (fromEnv) {
|
|
28
|
+
return fromEnv;
|
|
29
|
+
}
|
|
30
|
+
return captureCodeChangeFromGit(process.cwd?.() ?? ".", label);
|
|
31
|
+
}
|
|
32
|
+
async function readCodeChangeFile() {
|
|
33
|
+
const path = process.env?.BITFAB_CODE_CHANGE_PATH;
|
|
34
|
+
if (!path) {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
try {
|
|
38
|
+
const { readFile } = await import("fs/promises");
|
|
39
|
+
const parsed = JSON.parse(await readFile(path, "utf8"));
|
|
40
|
+
const files = Array.isArray(parsed?.files) && parsed.files.every(
|
|
41
|
+
(f) => typeof f === "object" && f !== null && !Array.isArray(f)
|
|
42
|
+
) ? parsed.files : void 0;
|
|
43
|
+
const description = typeof parsed?.description === "string" ? parsed.description : void 0;
|
|
44
|
+
if (!files && description === void 0) {
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
return { description, files };
|
|
48
|
+
} catch {
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
async function captureCodeChangeFromGit(cwd, label) {
|
|
53
|
+
let readFile;
|
|
54
|
+
try {
|
|
55
|
+
;
|
|
56
|
+
({ readFile } = await import("fs/promises"));
|
|
57
|
+
} catch {
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
const git = await gitRunner({
|
|
61
|
+
timeoutMs: 3e4,
|
|
62
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
63
|
+
keepProcessAlive: true
|
|
64
|
+
});
|
|
65
|
+
if (!git) {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
try {
|
|
69
|
+
const root = (await git(cwd, ["rev-parse", "--show-toplevel"]))?.trim();
|
|
70
|
+
if (!root) {
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
const resolved = await resolveBase(git, root);
|
|
74
|
+
if (!resolved) {
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
const { base, against } = resolved;
|
|
78
|
+
const blobBytes = async (ref, path) => {
|
|
79
|
+
const out = await git(root, ["cat-file", "-s", `${ref}:${path}`]);
|
|
80
|
+
const n = out ? Number.parseInt(out.trim(), 10) : Number.NaN;
|
|
81
|
+
return Number.isFinite(n) ? n : 0;
|
|
82
|
+
};
|
|
83
|
+
const workingBytes = async (path) => {
|
|
84
|
+
try {
|
|
85
|
+
const { stat } = await import("fs/promises");
|
|
86
|
+
const { join } = await import("path");
|
|
87
|
+
return (await stat(join(root, path))).size;
|
|
88
|
+
} catch {
|
|
89
|
+
return 0;
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
const tracked = await git(root, [
|
|
93
|
+
"diff",
|
|
94
|
+
"--name-status",
|
|
95
|
+
"--find-renames",
|
|
96
|
+
"-z",
|
|
97
|
+
base,
|
|
98
|
+
"--",
|
|
99
|
+
":!.bitfab"
|
|
100
|
+
]);
|
|
101
|
+
const untracked = await git(root, [
|
|
102
|
+
"ls-files",
|
|
103
|
+
"--others",
|
|
104
|
+
"--exclude-standard",
|
|
105
|
+
"-z",
|
|
106
|
+
"--",
|
|
107
|
+
":!.bitfab"
|
|
108
|
+
]);
|
|
109
|
+
const entries = [
|
|
110
|
+
...parseNameStatusZ(tracked ?? ""),
|
|
111
|
+
...(untracked ?? "").split(NUL).filter((p) => p.length > 0).map((path) => ({ status: "A", beforePath: path, path }))
|
|
112
|
+
];
|
|
113
|
+
if (entries.length === 0) {
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
116
|
+
const files = [];
|
|
117
|
+
let totalBytes = 0;
|
|
118
|
+
for (const { status, beforePath, path } of entries) {
|
|
119
|
+
if (files.length >= MAX_FILES) {
|
|
120
|
+
break;
|
|
121
|
+
}
|
|
122
|
+
const beforeBytes = status === "A" ? 0 : await blobBytes(base, beforePath);
|
|
123
|
+
const afterBytes = status === "D" ? 0 : await workingBytes(path);
|
|
124
|
+
if (beforeBytes > MAX_FILE_BYTES || afterBytes > MAX_FILE_BYTES) {
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
const before = (status === "A" ? "" : await git(root, ["show", `${base}:${beforePath}`]) ?? "").replace(/\r\n/g, "\n");
|
|
128
|
+
const after = (status === "D" ? "" : await readWorkingFile(readFile, root, path)).replace(/\r\n/g, "\n");
|
|
129
|
+
if (before === after) {
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
const size = Buffer.byteLength(before, "utf8") + Buffer.byteLength(after, "utf8");
|
|
133
|
+
if (totalBytes + size > MAX_TOTAL_BYTES || looksBinary(before) || looksBinary(after)) {
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
totalBytes += size;
|
|
137
|
+
files.push({ path, before, after });
|
|
138
|
+
}
|
|
139
|
+
if (files.length === 0) {
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
142
|
+
const subject = (await git(root, ["log", "-1", "--format=%s", "HEAD"]))?.trim();
|
|
143
|
+
const fileWord = files.length === 1 ? "file" : "files";
|
|
144
|
+
const head = label?.trim() || subject || "Working-tree change";
|
|
145
|
+
return {
|
|
146
|
+
description: `${head} (${files.length} ${fileWord} changed ${against})`,
|
|
147
|
+
files
|
|
148
|
+
};
|
|
149
|
+
} catch {
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
async function resolveBase(git, root) {
|
|
154
|
+
const forced = process.env?.BITFAB_CODE_CHANGE_BASE;
|
|
155
|
+
if (forced && await refExists(git, root, forced)) {
|
|
156
|
+
const base = (await git(root, ["merge-base", "HEAD", forced]))?.trim() || (await git(root, ["rev-parse", "--verify", forced]))?.trim() || null;
|
|
157
|
+
return base ? { base, against: `vs ${forced}` } : null;
|
|
158
|
+
}
|
|
159
|
+
return await refExists(git, root, "HEAD") ? { base: "HEAD", against: "uncommitted (vs HEAD)" } : null;
|
|
160
|
+
}
|
|
161
|
+
async function refExists(git, root, ref) {
|
|
162
|
+
return await git(root, ["rev-parse", "--verify", `${ref}^{object}`]) !== null;
|
|
163
|
+
}
|
|
164
|
+
async function readWorkingFile(readFile, root, path) {
|
|
165
|
+
try {
|
|
166
|
+
const { join } = await import("path");
|
|
167
|
+
return await readFile(join(root, path), "utf8");
|
|
168
|
+
} catch {
|
|
169
|
+
return "";
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
function parseNameStatusZ(raw) {
|
|
173
|
+
const parts = raw.split(NUL).filter((p) => p.length > 0);
|
|
174
|
+
const out = [];
|
|
175
|
+
let i = 0;
|
|
176
|
+
while (i + 1 < parts.length) {
|
|
177
|
+
const status = parts[i].charAt(0);
|
|
178
|
+
i += 1;
|
|
179
|
+
const beforePath = parts[i];
|
|
180
|
+
i += 1;
|
|
181
|
+
if ((status === "R" || status === "C") && i < parts.length) {
|
|
182
|
+
out.push({ status, beforePath, path: parts[i] });
|
|
183
|
+
i += 1;
|
|
184
|
+
} else {
|
|
185
|
+
out.push({ status, beforePath, path: beforePath });
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
return out;
|
|
189
|
+
}
|
|
190
|
+
function looksBinary(s) {
|
|
191
|
+
return s.slice(0, 8e3).includes(NUL);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// src/serialize.ts
|
|
195
|
+
import superjson from "superjson";
|
|
196
|
+
function deserializeValue(serialized) {
|
|
197
|
+
if (serialized.meta === void 0) {
|
|
198
|
+
return serialized.json;
|
|
199
|
+
}
|
|
200
|
+
return superjson.deserialize({
|
|
201
|
+
json: serialized.json,
|
|
202
|
+
meta: serialized.meta
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// src/integrationFixture.ts
|
|
207
|
+
var INTEGRATION_FIXTURE_CONTEXT_KEY = "__bitfabIntegrationFixture";
|
|
208
|
+
var INTEGRATION_FIXTURE_VERSION = 1;
|
|
209
|
+
function isRecord(value) {
|
|
210
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
211
|
+
}
|
|
212
|
+
function extractIntegrationFixtures(spanData) {
|
|
213
|
+
const contexts = spanData.contexts;
|
|
214
|
+
if (!Array.isArray(contexts)) {
|
|
215
|
+
return void 0;
|
|
216
|
+
}
|
|
217
|
+
const fixtures = /* @__PURE__ */ new Map();
|
|
218
|
+
for (const context of contexts) {
|
|
219
|
+
if (!isRecord(context)) {
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
222
|
+
const fixture = context[INTEGRATION_FIXTURE_CONTEXT_KEY];
|
|
223
|
+
if (!isRecord(fixture) || fixture.version !== INTEGRATION_FIXTURE_VERSION || typeof fixture.name !== "string") {
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
fixtures.set(
|
|
227
|
+
fixture.name,
|
|
228
|
+
deserializeValue({
|
|
229
|
+
json: fixture.value,
|
|
230
|
+
...fixture.valueMeta !== void 0 && { meta: fixture.valueMeta }
|
|
231
|
+
})
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
return fixtures.size > 0 ? fixtures : void 0;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// src/mockOverride.ts
|
|
238
|
+
function normalizeMockOverrides(mockOverride) {
|
|
239
|
+
if (mockOverride === void 0) {
|
|
240
|
+
return [];
|
|
241
|
+
}
|
|
242
|
+
const overrides = Array.isArray(mockOverride) ? mockOverride : [mockOverride];
|
|
243
|
+
return overrides.map(
|
|
244
|
+
(override) => typeof override === "function" ? { match: () => true, value: override } : override
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// src/randomUuid.ts
|
|
249
|
+
function randomUuid() {
|
|
250
|
+
const globalCrypto = globalThis.crypto;
|
|
251
|
+
if (typeof globalCrypto?.randomUUID === "function") {
|
|
252
|
+
try {
|
|
253
|
+
return globalCrypto.randomUUID();
|
|
254
|
+
} catch {
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
warnOnce(
|
|
258
|
+
"crypto-unavailable",
|
|
259
|
+
"global crypto.randomUUID is unavailable; using a non-cryptographic fallback for trace/span ids. Tracing works normally (ids are correlation-only, not security-sensitive)."
|
|
260
|
+
);
|
|
261
|
+
return fallbackUuidV4();
|
|
262
|
+
}
|
|
263
|
+
function fallbackUuidV4() {
|
|
264
|
+
return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, (char) => {
|
|
265
|
+
const rand = Math.random() * 16 | 0;
|
|
266
|
+
const value = char === "x" ? rand : rand & 3 | 8;
|
|
267
|
+
return value.toString(16);
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// src/replayHeartbeat.ts
|
|
272
|
+
var REPLAY_HEARTBEAT_INTERVAL_MS = 6e4;
|
|
273
|
+
function startReplayHeartbeat(send) {
|
|
274
|
+
let stopped = false;
|
|
275
|
+
let inFlight;
|
|
276
|
+
const timer = setInterval(() => {
|
|
277
|
+
if (stopped || inFlight !== void 0) {
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
inFlight = send().catch(() => void 0).finally(() => {
|
|
281
|
+
inFlight = void 0;
|
|
282
|
+
});
|
|
283
|
+
}, REPLAY_HEARTBEAT_INTERVAL_MS);
|
|
284
|
+
unrefTimer(timer);
|
|
285
|
+
return {
|
|
286
|
+
stop: async () => {
|
|
287
|
+
stopped = true;
|
|
288
|
+
clearInterval(timer);
|
|
289
|
+
await inFlight;
|
|
290
|
+
}
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// src/replayInterrupt.ts
|
|
295
|
+
var REPLAY_INTERRUPT_SIGNALS = [
|
|
296
|
+
"SIGINT",
|
|
297
|
+
"SIGTERM"
|
|
298
|
+
];
|
|
299
|
+
var replayListeners = /* @__PURE__ */ new Set();
|
|
300
|
+
function signalCapableProcess() {
|
|
301
|
+
if (typeof process === "undefined" || typeof process.on !== "function") {
|
|
302
|
+
return void 0;
|
|
303
|
+
}
|
|
304
|
+
return process;
|
|
305
|
+
}
|
|
306
|
+
function describeReplayInterrupt(experimentId, markedInterrupted) {
|
|
307
|
+
if (markedInterrupted) {
|
|
308
|
+
return `[replay] Experiment ${experimentId} interrupted.`;
|
|
309
|
+
}
|
|
310
|
+
return `[replay] Experiment ${experimentId} stopped, but Bitfab could not mark it interrupted. It will be marked interrupted within 15 minutes. A resume in the next two minutes may be refused as still running.`;
|
|
311
|
+
}
|
|
312
|
+
function reportInterrupted(experimentId, markedInterrupted) {
|
|
313
|
+
try {
|
|
314
|
+
console.error(describeReplayInterrupt(experimentId, markedInterrupted));
|
|
315
|
+
} catch {
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
function installReplayInterruptHandlers(target) {
|
|
319
|
+
const host = signalCapableProcess();
|
|
320
|
+
if (host === void 0) {
|
|
321
|
+
return { remove: () => {
|
|
322
|
+
}, interrupted: () => false };
|
|
323
|
+
}
|
|
324
|
+
const handlers = /* @__PURE__ */ new Map();
|
|
325
|
+
const removeHandlers = () => {
|
|
326
|
+
for (const [signal, handler] of handlers) {
|
|
327
|
+
host.removeListener(signal, handler);
|
|
328
|
+
replayListeners.delete(handler);
|
|
329
|
+
}
|
|
330
|
+
handlers.clear();
|
|
331
|
+
};
|
|
332
|
+
let interrupting = false;
|
|
333
|
+
const handleSignal = async (signal) => {
|
|
334
|
+
const hostHandlesSignal = host.listeners(signal).some((listener) => !replayListeners.has(listener));
|
|
335
|
+
const markedInterrupted = await target.interrupt();
|
|
336
|
+
removeHandlers();
|
|
337
|
+
if (target.onInterrupt !== void 0) {
|
|
338
|
+
try {
|
|
339
|
+
target.onInterrupt({
|
|
340
|
+
experimentId: target.experimentId,
|
|
341
|
+
experimentUrl: target.experimentUrl,
|
|
342
|
+
signal,
|
|
343
|
+
markedInterrupted
|
|
344
|
+
});
|
|
345
|
+
} catch {
|
|
346
|
+
}
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
reportInterrupted(target.experimentId, markedInterrupted);
|
|
350
|
+
if (!hostHandlesSignal) {
|
|
351
|
+
host.kill(host.pid, signal);
|
|
352
|
+
}
|
|
353
|
+
};
|
|
354
|
+
for (const signal of REPLAY_INTERRUPT_SIGNALS) {
|
|
355
|
+
const handler = () => {
|
|
356
|
+
if (interrupting) {
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
interrupting = true;
|
|
360
|
+
void handleSignal(signal);
|
|
361
|
+
};
|
|
362
|
+
handlers.set(signal, handler);
|
|
363
|
+
replayListeners.add(handler);
|
|
364
|
+
host.on(signal, handler);
|
|
365
|
+
}
|
|
366
|
+
return { remove: removeHandlers, interrupted: () => interrupting };
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
// src/replayInvocation.ts
|
|
370
|
+
var REPLAY_EXECUTION_TARGET_ENV = "BITFAB_REPLAY_EXECUTION_TARGET";
|
|
371
|
+
var HOSTED_EXECUTION_TARGET = "hosted";
|
|
372
|
+
var CI_ENV = "CI";
|
|
373
|
+
var CI_TRUE_VALUES = /* @__PURE__ */ new Set(["true", "1"]);
|
|
374
|
+
var CI_PRESENCE_ENV_VARS = [
|
|
375
|
+
"GITHUB_ACTIONS",
|
|
376
|
+
"GITLAB_CI",
|
|
377
|
+
"CIRCLECI",
|
|
378
|
+
"BUILDKITE",
|
|
379
|
+
"JENKINS_URL",
|
|
380
|
+
"TF_BUILD"
|
|
381
|
+
];
|
|
382
|
+
var MAX_FLAGS = 100;
|
|
383
|
+
var MAX_FLAG_NAME_LENGTH = 100;
|
|
384
|
+
var MAX_FLAG_STRING_LENGTH = 1e3;
|
|
385
|
+
var MAX_FLAG_LIST_LENGTH = 100;
|
|
386
|
+
var SECRET_FLAG_NAME = /key|token|secret|password/i;
|
|
387
|
+
var CLI_FLAGS = /* @__PURE__ */ Symbol("bitfab.replayCliFlags");
|
|
388
|
+
var METADATA_FLAG = "metadata";
|
|
389
|
+
var CLI_FLAG_ALIASES = {
|
|
390
|
+
"max-concurrency": "concurrency",
|
|
391
|
+
"dataset-id": "dataset-ids"
|
|
392
|
+
};
|
|
393
|
+
var OPTION_FLAG_NAMES = {
|
|
394
|
+
maxConcurrency: "concurrency",
|
|
395
|
+
datasetId: "dataset-ids",
|
|
396
|
+
datasetIds: "dataset-ids"
|
|
397
|
+
};
|
|
398
|
+
function replayEnvironment() {
|
|
399
|
+
if (readEnv(REPLAY_EXECUTION_TARGET_ENV) === HOSTED_EXECUTION_TARGET) {
|
|
400
|
+
return "cloud_replay";
|
|
401
|
+
}
|
|
402
|
+
if (CI_TRUE_VALUES.has((readEnv(CI_ENV) ?? "").toLowerCase())) {
|
|
403
|
+
return "ci";
|
|
404
|
+
}
|
|
405
|
+
if (CI_PRESENCE_ENV_VARS.some((name) => (readEnv(name) ?? "") !== "")) {
|
|
406
|
+
return "ci";
|
|
407
|
+
}
|
|
408
|
+
return "local";
|
|
409
|
+
}
|
|
410
|
+
function flagScalar(value) {
|
|
411
|
+
if (typeof value === "string") {
|
|
412
|
+
return value.slice(0, MAX_FLAG_STRING_LENGTH);
|
|
413
|
+
}
|
|
414
|
+
if (typeof value === "number") {
|
|
415
|
+
return Number.isFinite(value) ? value : void 0;
|
|
416
|
+
}
|
|
417
|
+
if (typeof value === "boolean" || value === null) {
|
|
418
|
+
return value;
|
|
419
|
+
}
|
|
420
|
+
return void 0;
|
|
421
|
+
}
|
|
422
|
+
function flagValue(value) {
|
|
423
|
+
if (Array.isArray(value)) {
|
|
424
|
+
const items = value.slice(0, MAX_FLAG_LIST_LENGTH).map(flagScalar);
|
|
425
|
+
return items.every((item) => item !== void 0) ? items : true;
|
|
426
|
+
}
|
|
427
|
+
const scalar = flagScalar(value);
|
|
428
|
+
return scalar === void 0 ? true : scalar;
|
|
429
|
+
}
|
|
430
|
+
function canonicalCliFlag(name) {
|
|
431
|
+
return CLI_FLAG_ALIASES[name] ?? name;
|
|
432
|
+
}
|
|
433
|
+
function optionFlagName(option) {
|
|
434
|
+
return OPTION_FLAG_NAMES[option] ?? option.replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`);
|
|
435
|
+
}
|
|
436
|
+
function metadataKeys(value) {
|
|
437
|
+
const keys = Array.isArray(value) ? value.map((pair) => String(pair).split("=")[0]) : typeof value === "object" && value !== null ? Object.keys(value) : void 0;
|
|
438
|
+
return keys === void 0 ? true : [...new Set(keys)].sort();
|
|
439
|
+
}
|
|
440
|
+
function invocationFlags(entries) {
|
|
441
|
+
const flags = {};
|
|
442
|
+
const seen = /* @__PURE__ */ new Set();
|
|
443
|
+
for (const [name, value] of entries) {
|
|
444
|
+
if (seen.size >= MAX_FLAGS) {
|
|
445
|
+
break;
|
|
446
|
+
}
|
|
447
|
+
if (value === void 0 || seen.has(name) || name.length === 0 || name.length > MAX_FLAG_NAME_LENGTH || SECRET_FLAG_NAME.test(name)) {
|
|
448
|
+
continue;
|
|
449
|
+
}
|
|
450
|
+
flags[name] = flagValue(
|
|
451
|
+
name === METADATA_FLAG ? metadataKeys(value) : value
|
|
452
|
+
);
|
|
453
|
+
seen.add(name);
|
|
454
|
+
}
|
|
455
|
+
return flags;
|
|
456
|
+
}
|
|
457
|
+
function buildReplayInvocation(entries = []) {
|
|
458
|
+
return {
|
|
459
|
+
flags: invocationFlags(entries),
|
|
460
|
+
sdk: { language: "typescript", version: __version__ },
|
|
461
|
+
environment: replayEnvironment()
|
|
462
|
+
};
|
|
463
|
+
}
|
|
464
|
+
function safeReplayInvocation(build) {
|
|
465
|
+
try {
|
|
466
|
+
return build();
|
|
467
|
+
} catch {
|
|
468
|
+
return void 0;
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
function withCliFlags(options, flags) {
|
|
472
|
+
Object.defineProperty(options, CLI_FLAGS, {
|
|
473
|
+
value: flags,
|
|
474
|
+
enumerable: false
|
|
475
|
+
});
|
|
476
|
+
return options;
|
|
477
|
+
}
|
|
478
|
+
function cliFlags(options) {
|
|
479
|
+
return options?.[CLI_FLAGS];
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
// src/selectiveReplay.ts
|
|
483
|
+
function replayNodeKey(node) {
|
|
484
|
+
return JSON.stringify([node.traceFunctionKey, node.spanName]);
|
|
485
|
+
}
|
|
486
|
+
function jsonFingerprint(value) {
|
|
487
|
+
const seen = /* @__PURE__ */ new Set();
|
|
488
|
+
function encode(input) {
|
|
489
|
+
if (input === null || typeof input === "boolean") {
|
|
490
|
+
return JSON.stringify(input);
|
|
491
|
+
}
|
|
492
|
+
if (typeof input === "string" && !input.startsWith("<unserializable")) {
|
|
493
|
+
return JSON.stringify(input);
|
|
494
|
+
}
|
|
495
|
+
if (typeof input === "number" && Number.isFinite(input) && !Object.is(input, -0)) {
|
|
496
|
+
return JSON.stringify(input);
|
|
497
|
+
}
|
|
498
|
+
if (typeof input !== "object" || input === null || seen.has(input)) {
|
|
499
|
+
throw new Error("Not a lossless JSON value");
|
|
500
|
+
}
|
|
501
|
+
seen.add(input);
|
|
502
|
+
const prototype = Object.getPrototypeOf(input);
|
|
503
|
+
if (prototype !== (Array.isArray(input) ? Array.prototype : Object.prototype)) {
|
|
504
|
+
throw new Error("Not a plain JSON object");
|
|
505
|
+
}
|
|
506
|
+
if (Object.getOwnPropertySymbols(input).length > 0) {
|
|
507
|
+
throw new Error("Symbol properties are not recorded");
|
|
508
|
+
}
|
|
509
|
+
const keys = Object.keys(input);
|
|
510
|
+
if (Object.getOwnPropertyNames(input).length !== keys.length + (Array.isArray(input) ? 1 : 0)) {
|
|
511
|
+
throw new Error("Non-enumerable properties are not recorded");
|
|
512
|
+
}
|
|
513
|
+
if (Array.isArray(input) && (keys.length !== input.length || keys.some((key, index) => key !== String(index)))) {
|
|
514
|
+
throw new Error("Not a dense JSON array");
|
|
515
|
+
}
|
|
516
|
+
const properties = keys.map((key) => {
|
|
517
|
+
const descriptor = Object.getOwnPropertyDescriptor(input, key);
|
|
518
|
+
if (!descriptor || !("value" in descriptor)) {
|
|
519
|
+
throw new Error("Accessor properties are not replay inputs");
|
|
520
|
+
}
|
|
521
|
+
return `${Array.isArray(input) ? "" : `${JSON.stringify(key)}:`}${encode(descriptor.value)}`;
|
|
522
|
+
});
|
|
523
|
+
return Array.isArray(input) ? `[${properties.join(",")}]` : `{${properties.join(",")}}`;
|
|
524
|
+
}
|
|
525
|
+
try {
|
|
526
|
+
return encode(value);
|
|
527
|
+
} catch {
|
|
528
|
+
return void 0;
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// src/selectiveReplayRuntime.ts
|
|
533
|
+
function validateSelectiveReplayOptions(options) {
|
|
534
|
+
const validNodes = (nodes) => Array.isArray(nodes) && nodes.length <= 100 && nodes.every(
|
|
535
|
+
(node) => typeof node?.traceFunctionKey === "string" && node.traceFunctionKey.length > 0 && node.traceFunctionKey.length <= 500 && typeof node.spanName === "string" && node.spanName.length > 0 && node.spanName.length <= 500
|
|
536
|
+
);
|
|
537
|
+
if (!options || !validNodes(options.mustRun) || options.mustRun.length === 0 || Object.keys(options).some((key) => key !== "mustRun")) {
|
|
538
|
+
throw new BitfabError(
|
|
539
|
+
"experimentalSelectiveReplay accepts only 1\u2013100 mustRun with traceFunctionKey and spanName. Declare replayReusable on eligible spans, not reusableNodes in the replay request."
|
|
540
|
+
);
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
function policyKey(options) {
|
|
544
|
+
return JSON.stringify(options.mustRun.map(replayNodeKey).sort());
|
|
545
|
+
}
|
|
546
|
+
var SelectiveReplayRuntime = class {
|
|
547
|
+
constructor(plan, options) {
|
|
548
|
+
this.plan = plan;
|
|
549
|
+
this.options = options;
|
|
550
|
+
this.nodes = /* @__PURE__ */ new Map();
|
|
551
|
+
this.children = /* @__PURE__ */ new Map();
|
|
552
|
+
this.runtimeParents = /* @__PURE__ */ new Map();
|
|
553
|
+
this.consumed = /* @__PURE__ */ new Map();
|
|
554
|
+
this.requiredScopes = /* @__PURE__ */ new Set();
|
|
555
|
+
this.recordings = /* @__PURE__ */ new Map();
|
|
556
|
+
this.reached = /* @__PURE__ */ new Set();
|
|
557
|
+
this.decisions = [];
|
|
558
|
+
this.assertionNames = /* @__PURE__ */ new Set();
|
|
559
|
+
this.assertionScopes = /* @__PURE__ */ new Set();
|
|
560
|
+
this.observedNames = /* @__PURE__ */ new Map();
|
|
561
|
+
this.unresolvedAssertions = [];
|
|
562
|
+
validateSelectiveReplayOptions(options);
|
|
563
|
+
if (plan?.version !== 4 || !Array.isArray(plan.nodes) || plan.nodes.length === 0 || plan.nodes.length > 500 || !Array.isArray(plan.unresolvedMustRun) || !Array.isArray(plan.assertions) || !plan.options) {
|
|
564
|
+
throw new BitfabError(
|
|
565
|
+
"Server did not return a compatible selective replay plan. Update the server or disable experimentalSelectiveReplay."
|
|
566
|
+
);
|
|
567
|
+
}
|
|
568
|
+
validateSelectiveReplayOptions(plan.options);
|
|
569
|
+
if (policyKey(plan.options) !== policyKey(options)) {
|
|
570
|
+
throw new BitfabError(
|
|
571
|
+
"Selective replay plan does not match the requested mustRun nodes."
|
|
572
|
+
);
|
|
573
|
+
}
|
|
574
|
+
this.required = new Set(options.mustRun.map(replayNodeKey));
|
|
575
|
+
for (const node of plan.nodes) {
|
|
576
|
+
if (typeof node.id !== "string" || this.nodes.has(node.id) || typeof node.traceFunctionKey !== "string" || typeof node.spanName !== "string" || typeof node.mustRun !== "boolean" || typeof node.reusable !== "boolean") {
|
|
577
|
+
throw new BitfabError("Invalid selective replay node.");
|
|
578
|
+
}
|
|
579
|
+
this.nodes.set(node.id, node);
|
|
580
|
+
if (node.recording !== void 0) {
|
|
581
|
+
this.recordings.set(node.id, {
|
|
582
|
+
input: node.recording.kwargs === void 0 ? jsonFingerprint(node.recording.input) : void 0,
|
|
583
|
+
output: jsonFingerprint(node.recording.output)
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
if (node.parentId !== null) {
|
|
587
|
+
const siblings = this.children.get(node.parentId) ?? [];
|
|
588
|
+
siblings.push(node);
|
|
589
|
+
this.children.set(node.parentId, siblings);
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
const root = this.nodes.get(plan.rootId);
|
|
593
|
+
if (!root || root.parentId !== null || !root.mustRun) {
|
|
594
|
+
throw new BitfabError("Selective replay requires a live recorded root.");
|
|
595
|
+
}
|
|
596
|
+
const assertionIds = /* @__PURE__ */ new Set();
|
|
597
|
+
let wholeTrace = false;
|
|
598
|
+
for (const assertion of plan.assertions) {
|
|
599
|
+
const target = assertion?.target;
|
|
600
|
+
if (typeof assertion?.id !== "string" || assertion.id.length === 0 || assertionIds.has(assertion.id) || target !== null && (typeof target !== "object" || target.kind !== "output" && target.kind !== "span")) {
|
|
601
|
+
throw new BitfabError(
|
|
602
|
+
"Invalid selective replay assertion requirements."
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
assertionIds.add(assertion.id);
|
|
606
|
+
if (target === null) {
|
|
607
|
+
wholeTrace = true;
|
|
608
|
+
} else if (target.kind === "span") {
|
|
609
|
+
if (typeof target.name !== "string" || target.name.length === 0 || target.occurrence !== void 0 && target.occurrence !== "first" && target.occurrence !== "last" && !(Number.isInteger(target.occurrence) && typeof target.occurrence === "number" && target.occurrence >= 0)) {
|
|
610
|
+
throw new BitfabError("Invalid selective replay assertion target.");
|
|
611
|
+
}
|
|
612
|
+
this.assertionNames.add(target.name);
|
|
613
|
+
const count = plan.nodes.filter(
|
|
614
|
+
(node) => node.spanName === target.name
|
|
615
|
+
).length;
|
|
616
|
+
if (count <= (typeof target.occurrence === "number" ? target.occurrence : 0)) {
|
|
617
|
+
this.unresolvedAssertions.push(assertion.id);
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
this.fullTraceForAssertions = wholeTrace || this.unresolvedAssertions.length > 0;
|
|
622
|
+
const recordedKeys = new Set(plan.nodes.map(replayNodeKey));
|
|
623
|
+
const unresolved = options.mustRun.filter(
|
|
624
|
+
(node) => !recordedKeys.has(replayNodeKey(node))
|
|
625
|
+
);
|
|
626
|
+
if (JSON.stringify(unresolved.map(replayNodeKey).sort()) !== JSON.stringify(plan.unresolvedMustRun.map(replayNodeKey).sort())) {
|
|
627
|
+
throw new BitfabError(
|
|
628
|
+
"Selective replay plan has incomplete mustRun coverage."
|
|
629
|
+
);
|
|
630
|
+
}
|
|
631
|
+
for (const node of plan.nodes) {
|
|
632
|
+
const ancestors = /* @__PURE__ */ new Set([node.id]);
|
|
633
|
+
let requiredByManifest = this.required.has(replayNodeKey(node));
|
|
634
|
+
let requiredForAssertion = this.fullTraceForAssertions || this.assertionNames.has(node.spanName);
|
|
635
|
+
let parentId = node.parentId;
|
|
636
|
+
while (parentId !== null) {
|
|
637
|
+
const parent = this.nodes.get(parentId);
|
|
638
|
+
if (!parent || ancestors.has(parentId) || node.mustRun && !parent.mustRun) {
|
|
639
|
+
throw new BitfabError("Invalid selective replay ancestry.");
|
|
640
|
+
}
|
|
641
|
+
ancestors.add(parentId);
|
|
642
|
+
requiredByManifest ||= this.required.has(replayNodeKey(parent));
|
|
643
|
+
requiredForAssertion ||= this.assertionNames.has(parent.spanName);
|
|
644
|
+
parentId = parent.parentId;
|
|
645
|
+
}
|
|
646
|
+
if (!ancestors.has(plan.rootId) || (requiredByManifest || unresolved.length > 0) && !node.mustRun) {
|
|
647
|
+
throw new BitfabError("Selective replay plan could hide required code.");
|
|
648
|
+
}
|
|
649
|
+
if (requiredForAssertion && !node.mustRun) {
|
|
650
|
+
throw new BitfabError(
|
|
651
|
+
"Selective replay plan could hide assertion evidence."
|
|
652
|
+
);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
enter(params) {
|
|
657
|
+
const identity = {
|
|
658
|
+
traceFunctionKey: params.traceFunctionKey,
|
|
659
|
+
spanName: params.spanName
|
|
660
|
+
};
|
|
661
|
+
const key = replayNodeKey(identity);
|
|
662
|
+
const fingerprint = jsonFingerprint(params.inputs);
|
|
663
|
+
let node;
|
|
664
|
+
let matchReason = "unmatched-call";
|
|
665
|
+
if (params.parentSpanId === null) {
|
|
666
|
+
node = this.nodes.get(this.plan.rootId);
|
|
667
|
+
} else {
|
|
668
|
+
const recordedParent = this.runtimeParents.get(params.parentSpanId);
|
|
669
|
+
const consumed = this.consumed.get(params.parentSpanId) ?? /* @__PURE__ */ new Set();
|
|
670
|
+
this.consumed.set(params.parentSpanId, consumed);
|
|
671
|
+
const candidates = recordedParent ? (this.children.get(recordedParent) ?? []).filter(
|
|
672
|
+
(candidate) => replayNodeKey(candidate) === key && !consumed.has(candidate.id)
|
|
673
|
+
) : [];
|
|
674
|
+
const exact = fingerprint === void 0 ? [] : candidates.filter(
|
|
675
|
+
(candidate) => this.recordings.get(candidate.id)?.input === fingerprint
|
|
676
|
+
);
|
|
677
|
+
node = exact.length === 1 ? exact[0] : exact.length === 0 && candidates.length === 1 ? candidates[0] : void 0;
|
|
678
|
+
if (node) {
|
|
679
|
+
consumed.add(node.id);
|
|
680
|
+
} else if (candidates.length > 0) {
|
|
681
|
+
matchReason = "ambiguous-call";
|
|
682
|
+
for (const candidate of candidates) {
|
|
683
|
+
consumed.add(candidate.id);
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
this.runtimeParents.set(params.spanId, node?.id ?? null);
|
|
688
|
+
const requiredByManifest = this.required.has(key) || params.parentSpanId !== null && this.requiredScopes.has(params.parentSpanId);
|
|
689
|
+
if (requiredByManifest) {
|
|
690
|
+
this.requiredScopes.add(params.spanId);
|
|
691
|
+
}
|
|
692
|
+
const requiredForAssertion = this.fullTraceForAssertions || this.assertionNames.has(params.spanName) || params.parentSpanId !== null && this.assertionScopes.has(params.parentSpanId);
|
|
693
|
+
if (requiredForAssertion) {
|
|
694
|
+
this.assertionScopes.add(params.spanId);
|
|
695
|
+
}
|
|
696
|
+
const mustRun = params.parentSpanId === null || requiredByManifest || node?.mustRun === true || requiredForAssertion || this.plan.unresolvedMustRun.length > 0;
|
|
697
|
+
const recording = node === void 0 ? void 0 : this.recordings.get(node.id);
|
|
698
|
+
const equalInputs = fingerprint !== void 0 && recording?.input === fingerprint;
|
|
699
|
+
const reusable = recording?.output !== void 0 && equalInputs && params.canReuseOutput;
|
|
700
|
+
const reason = mustRun ? requiredByManifest ? "must-run" : requiredForAssertion ? "assertion" : node?.reason ?? "unresolved-must-run" : !node ? matchReason : !equalInputs ? "input-drift-or-missing-recording" : !params.canReuseOutput ? "unsupported-output-boundary" : !params.safetyMock && !params.reuseAllowed ? "reuse-not-declared" : node.reason;
|
|
701
|
+
const decision = (action, why) => {
|
|
702
|
+
this.decisions.push({
|
|
703
|
+
...identity,
|
|
704
|
+
originalSpanId: node?.id ?? null,
|
|
705
|
+
action,
|
|
706
|
+
reason: why
|
|
707
|
+
});
|
|
708
|
+
};
|
|
709
|
+
if (params.safetyMock && (mustRun || !reusable)) {
|
|
710
|
+
decision("blocked", `safety-conflict:${reason}`);
|
|
711
|
+
this.failure = new BitfabError(
|
|
712
|
+
`Selective replay cannot execute or safely reuse safety-mocked node ${params.spanName} (${reason}).`
|
|
713
|
+
);
|
|
714
|
+
throw this.failure;
|
|
715
|
+
}
|
|
716
|
+
if (!mustRun && reusable && (params.safetyMock || node?.reusable === true && params.reuseAllowed)) {
|
|
717
|
+
decision("mock", params.safetyMock ? "safety-mock" : "unchanged-inputs");
|
|
718
|
+
return {
|
|
719
|
+
mock: true,
|
|
720
|
+
output: JSON.parse(recording.output)
|
|
721
|
+
};
|
|
722
|
+
}
|
|
723
|
+
decision("run", reason);
|
|
724
|
+
this.observedNames.set(
|
|
725
|
+
params.spanName,
|
|
726
|
+
(this.observedNames.get(params.spanName) ?? 0) + 1
|
|
727
|
+
);
|
|
728
|
+
if (this.required.has(key)) {
|
|
729
|
+
this.reached.add(key);
|
|
730
|
+
}
|
|
731
|
+
return { mock: false };
|
|
732
|
+
}
|
|
733
|
+
assertNoSafetyConflict() {
|
|
734
|
+
if (this.failure) {
|
|
735
|
+
throw this.failure;
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
report() {
|
|
739
|
+
return {
|
|
740
|
+
decisions: this.decisions,
|
|
741
|
+
mustRunNotReached: this.options.mustRun.filter(
|
|
742
|
+
(node) => !this.reached.has(replayNodeKey(node))
|
|
743
|
+
),
|
|
744
|
+
unresolvedMustRun: this.plan.unresolvedMustRun,
|
|
745
|
+
assertionIds: this.plan.assertions.map((assertion) => assertion.id),
|
|
746
|
+
unresolvedAssertions: this.unresolvedAssertions,
|
|
747
|
+
assertionTargetsNotReached: this.plan.assertions.flatMap(
|
|
748
|
+
({ id, target }) => target?.kind === "span" && (this.observedNames.get(target.name) ?? 0) <= (typeof target.occurrence === "number" ? target.occurrence : 0) ? [id] : []
|
|
749
|
+
)
|
|
750
|
+
};
|
|
751
|
+
}
|
|
752
|
+
};
|
|
753
|
+
|
|
754
|
+
// src/replay.ts
|
|
755
|
+
var REPLAY_PERSISTENCE_TIMEOUT_MS = 3e4;
|
|
756
|
+
var MAX_REPLAY_ATTEMPTS = 100;
|
|
757
|
+
function expandReplayWork(serverItems, attempts) {
|
|
758
|
+
return Array.from(
|
|
759
|
+
{ length: attempts },
|
|
760
|
+
(_, attempt) => serverItems.map((serverItem) => ({ serverItem, attempt }))
|
|
761
|
+
).flat();
|
|
762
|
+
}
|
|
763
|
+
var RESUME_CONFLICTING_OPTIONS = [
|
|
764
|
+
"traceIds",
|
|
765
|
+
"limit",
|
|
766
|
+
"datasetId",
|
|
767
|
+
"datasetIds",
|
|
768
|
+
"attempts",
|
|
769
|
+
"graderIds",
|
|
770
|
+
"onlyWithAssertions",
|
|
771
|
+
"dryRun",
|
|
772
|
+
"experimentalSelectiveReplay"
|
|
773
|
+
];
|
|
774
|
+
var CARRIED_OVER_ERROR = "This item errored in an earlier run of this experiment, before it was resumed.";
|
|
775
|
+
function validateResume(resume, options) {
|
|
776
|
+
if (resume.trim().length === 0) {
|
|
777
|
+
throw new BitfabError("resume must be the ID of the experiment to resume.");
|
|
778
|
+
}
|
|
779
|
+
const conflicting = RESUME_CONFLICTING_OPTIONS.filter(
|
|
780
|
+
(name) => options[name] !== void 0 && options[name] !== false
|
|
781
|
+
);
|
|
782
|
+
if (conflicting.length > 0) {
|
|
783
|
+
throw new BitfabError(
|
|
784
|
+
`resume continues experiment ${resume} with the traces, attempts, and graders it started with, so it cannot be combined with ${conflicting.join(", ")}. Drop those options, or start a new replay without resume.`
|
|
785
|
+
);
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
async function resumeReplaySession(httpClient, experimentId, traceFunctionKey, codeChangeFiles, force, requestedDbBranch) {
|
|
789
|
+
const response = await httpClient.resumeReplay(
|
|
790
|
+
experimentId,
|
|
791
|
+
traceFunctionKey,
|
|
792
|
+
codeChangeFiles,
|
|
793
|
+
force
|
|
794
|
+
);
|
|
795
|
+
const work = response.work.map(({ itemIndex, attempt }) => {
|
|
796
|
+
const serverItem = response.items[itemIndex];
|
|
797
|
+
if (serverItem === void 0) {
|
|
798
|
+
throw new BitfabError(
|
|
799
|
+
`Replay resume for experiment ${response.experimentId} named item ${itemIndex}, but the server returned only ${response.items.length} item(s).`
|
|
800
|
+
);
|
|
801
|
+
}
|
|
802
|
+
return { serverItem, attempt };
|
|
803
|
+
});
|
|
804
|
+
if (response.skippedOriginalTraceIds.length > 0) {
|
|
805
|
+
try {
|
|
806
|
+
console.warn(
|
|
807
|
+
`Bitfab: ${response.skippedOriginalTraceIds.length} trace(s) selected when experiment ${response.experimentId} started no longer exist, so resume skips them: ${response.skippedOriginalTraceIds.join(", ")}`
|
|
808
|
+
);
|
|
809
|
+
} catch {
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
if (requestedDbBranch !== void 0 && dbBranchEnabled(requestedDbBranch) !== response.includeDbBranchLease) {
|
|
813
|
+
try {
|
|
814
|
+
console.warn(
|
|
815
|
+
`Bitfab: experiment ${response.experimentId} started with database branching ${response.includeDbBranchLease ? "on" : "off"}, so resume keeps it ${response.includeDbBranchLease ? "on" : "off"} and ignores the dbBranch option passed here.`
|
|
816
|
+
);
|
|
817
|
+
} catch {
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
return {
|
|
821
|
+
experimentId: response.experimentId,
|
|
822
|
+
experimentUrl: response.experimentUrl,
|
|
823
|
+
attempts: response.attempts,
|
|
824
|
+
work,
|
|
825
|
+
finished: response.finished,
|
|
826
|
+
includeDbBranchLease: response.includeDbBranchLease,
|
|
827
|
+
dbBranchSettings: response.dbBranchSettings ?? void 0
|
|
828
|
+
};
|
|
829
|
+
}
|
|
830
|
+
function carriedOverReplayItem(finished) {
|
|
831
|
+
return {
|
|
832
|
+
traceId: finished.traceId,
|
|
833
|
+
originalTraceId: finished.originalTraceId,
|
|
834
|
+
originalSpanId: "",
|
|
835
|
+
sourceTraceId: finished.originalTraceId,
|
|
836
|
+
sourceSpanId: "",
|
|
837
|
+
attempt: finished.attempt,
|
|
838
|
+
input: [],
|
|
839
|
+
result: void 0,
|
|
840
|
+
originalOutput: void 0,
|
|
841
|
+
error: finished.hasError ? CARRIED_OVER_ERROR : null,
|
|
842
|
+
traceError: null,
|
|
843
|
+
replayError: null,
|
|
844
|
+
durationMs: null,
|
|
845
|
+
originalDurationMs: null,
|
|
846
|
+
originalTokens: null,
|
|
847
|
+
originalModel: null,
|
|
848
|
+
tokens: null,
|
|
849
|
+
model: null,
|
|
850
|
+
dbSnapshotRef: null,
|
|
851
|
+
dbBranchTimings: null,
|
|
852
|
+
carriedOver: true
|
|
853
|
+
};
|
|
854
|
+
}
|
|
855
|
+
function resolveAttempts(attempts) {
|
|
856
|
+
if (attempts === void 0) {
|
|
857
|
+
return 1;
|
|
858
|
+
}
|
|
859
|
+
if (!Number.isInteger(attempts) || attempts < 1 || attempts > MAX_REPLAY_ATTEMPTS) {
|
|
860
|
+
throw new BitfabError(
|
|
861
|
+
`attempts must be an integer from 1 to ${MAX_REPLAY_ATTEMPTS} (got ${attempts}).`
|
|
862
|
+
);
|
|
863
|
+
}
|
|
864
|
+
return attempts;
|
|
865
|
+
}
|
|
866
|
+
function resolveDatasetIds(options) {
|
|
867
|
+
if (options?.datasetId !== void 0 && options?.datasetIds !== void 0) {
|
|
868
|
+
throw new BitfabError(
|
|
869
|
+
"datasetId and datasetIds are the same selector: pass one dataset through datasetId, or several through datasetIds."
|
|
870
|
+
);
|
|
871
|
+
}
|
|
872
|
+
if (options?.datasetId !== void 0) {
|
|
873
|
+
return [options.datasetId];
|
|
874
|
+
}
|
|
875
|
+
if (options?.datasetIds === void 0) {
|
|
876
|
+
return void 0;
|
|
877
|
+
}
|
|
878
|
+
if (options.datasetIds.length === 0) {
|
|
879
|
+
throw new BitfabError("datasetIds must contain at least one dataset ID.");
|
|
880
|
+
}
|
|
881
|
+
return [...new Set(options.datasetIds)];
|
|
882
|
+
}
|
|
883
|
+
function dbBranchEnabled(dbBranch) {
|
|
884
|
+
return dbBranch !== void 0 && dbBranch !== false;
|
|
885
|
+
}
|
|
886
|
+
function resolveDbBranchSettings(dbBranch) {
|
|
887
|
+
if (!dbBranch || dbBranch === true) {
|
|
888
|
+
return void 0;
|
|
889
|
+
}
|
|
890
|
+
const { minCu, maxCu, warmupSql } = dbBranch;
|
|
891
|
+
const settings = {
|
|
892
|
+
...minCu === void 0 ? {} : { minCu },
|
|
893
|
+
...maxCu === void 0 ? {} : { maxCu },
|
|
894
|
+
...warmupSql === void 0 ? {} : { warmupSql }
|
|
895
|
+
};
|
|
896
|
+
return Object.keys(settings).length === 0 ? void 0 : settings;
|
|
897
|
+
}
|
|
898
|
+
var REPLAY_OPTION_NAMES = [
|
|
899
|
+
"experimentalSelectiveReplay",
|
|
900
|
+
"limit",
|
|
901
|
+
"attempts",
|
|
902
|
+
"traceIds",
|
|
903
|
+
"name",
|
|
904
|
+
"notes",
|
|
905
|
+
"metadata",
|
|
906
|
+
"maxConcurrency",
|
|
907
|
+
"codeChangeDescription",
|
|
908
|
+
"codeChangeFiles",
|
|
909
|
+
"mock",
|
|
910
|
+
"mockOverride",
|
|
911
|
+
"dbBranch",
|
|
912
|
+
"experimentGroupId",
|
|
913
|
+
"datasetId",
|
|
914
|
+
"datasetIds",
|
|
915
|
+
"graderIds",
|
|
916
|
+
"onlyWithAssertions",
|
|
917
|
+
"skipAssertionJudging",
|
|
918
|
+
"judgeAssertions",
|
|
919
|
+
"adaptInputs",
|
|
920
|
+
"dryRun",
|
|
921
|
+
"onItemFinish",
|
|
922
|
+
"onProgress",
|
|
923
|
+
"onItemStart",
|
|
924
|
+
"onExperimentStart",
|
|
925
|
+
"onInterrupt",
|
|
926
|
+
"resume",
|
|
927
|
+
"force"
|
|
928
|
+
];
|
|
929
|
+
function explicitReplayOptions(options) {
|
|
930
|
+
const explicit = {};
|
|
931
|
+
for (const name of REPLAY_OPTION_NAMES) {
|
|
932
|
+
if (options?.[name] !== void 0) {
|
|
933
|
+
explicit[optionFlagName(name)] ??= options[name];
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
return explicit;
|
|
937
|
+
}
|
|
938
|
+
var BITFAB_PROGRESS_PREFIX = "@@bitfab:progress ";
|
|
939
|
+
function reportReplayProgress(progress) {
|
|
940
|
+
const stderr = typeof process !== "undefined" ? process.stderr : void 0;
|
|
941
|
+
if (!stderr) {
|
|
942
|
+
return;
|
|
943
|
+
}
|
|
944
|
+
try {
|
|
945
|
+
stderr.write(
|
|
946
|
+
`${BITFAB_PROGRESS_PREFIX}${JSON.stringify(progress, replayJsonReplacer)}
|
|
947
|
+
`
|
|
948
|
+
);
|
|
949
|
+
} catch {
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
var ReplayError = class extends BitfabError {
|
|
953
|
+
constructor(message, items, experimentId, experimentUrl, cause) {
|
|
954
|
+
super(message, experimentUrl);
|
|
955
|
+
this.items = items;
|
|
956
|
+
this.experimentId = experimentId;
|
|
957
|
+
this.experimentUrl = experimentUrl;
|
|
958
|
+
this.cause = cause;
|
|
959
|
+
this.name = "ReplayError";
|
|
960
|
+
this.testRunId = experimentId;
|
|
961
|
+
this.testRunUrl = experimentUrl;
|
|
962
|
+
}
|
|
963
|
+
};
|
|
964
|
+
var DbBranchReplayError = class extends BitfabError {
|
|
965
|
+
constructor(code, message, originalTraceId, cause) {
|
|
966
|
+
super(message);
|
|
967
|
+
this.code = code;
|
|
968
|
+
this.originalTraceId = originalTraceId;
|
|
969
|
+
this.cause = cause;
|
|
970
|
+
this.name = "DbBranchReplayError";
|
|
971
|
+
}
|
|
972
|
+
};
|
|
973
|
+
function isPlainRecord(value) {
|
|
974
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
975
|
+
}
|
|
976
|
+
function errorMessage(error) {
|
|
977
|
+
return error instanceof Error ? error.message : String(error);
|
|
978
|
+
}
|
|
979
|
+
function replayItemErrorMessage(error) {
|
|
980
|
+
if (error instanceof DbBranchReplayError) {
|
|
981
|
+
return `Replay requested a database branch for trace ${error.originalTraceId} but it could not be resolved (${error.code}): ${error.message}. The function was not run, because replaying it against the live database would produce a result that looks valid but did not use the historical data you asked for.`;
|
|
982
|
+
}
|
|
983
|
+
return errorMessage(error);
|
|
984
|
+
}
|
|
985
|
+
function replayJsonReplacer(_key, value) {
|
|
986
|
+
if (value instanceof Error) {
|
|
987
|
+
const serialized = {
|
|
988
|
+
name: value.name,
|
|
989
|
+
message: value.message,
|
|
990
|
+
stack: value.stack
|
|
991
|
+
};
|
|
992
|
+
if (value instanceof DbBranchReplayError) {
|
|
993
|
+
serialized.code = value.code;
|
|
994
|
+
serialized.originalTraceId = value.originalTraceId;
|
|
995
|
+
if (value.cause !== void 0) {
|
|
996
|
+
serialized.cause = value.cause;
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
return serialized;
|
|
1000
|
+
}
|
|
1001
|
+
return value;
|
|
1002
|
+
}
|
|
1003
|
+
function serializeReplayResult(result) {
|
|
1004
|
+
return JSON.stringify(result, replayJsonReplacer, 2);
|
|
1005
|
+
}
|
|
1006
|
+
async function preserveReplayFailure(operation, items, experimentId, experimentUrl) {
|
|
1007
|
+
try {
|
|
1008
|
+
return await operation();
|
|
1009
|
+
} catch (cause) {
|
|
1010
|
+
if (cause instanceof ReplayError) {
|
|
1011
|
+
throw cause;
|
|
1012
|
+
}
|
|
1013
|
+
throw new ReplayError(
|
|
1014
|
+
errorMessage(cause),
|
|
1015
|
+
items,
|
|
1016
|
+
experimentId,
|
|
1017
|
+
experimentUrl,
|
|
1018
|
+
cause
|
|
1019
|
+
);
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
function deserializeInputs(spanData) {
|
|
1023
|
+
const inputMeta = spanData.input_meta;
|
|
1024
|
+
const rawInput = spanData.input;
|
|
1025
|
+
if (inputMeta !== void 0 && inputMeta !== null) {
|
|
1026
|
+
const deserialized = deserializeValue({ json: rawInput, meta: inputMeta });
|
|
1027
|
+
if (Array.isArray(deserialized)) {
|
|
1028
|
+
return deserialized;
|
|
1029
|
+
}
|
|
1030
|
+
return deserialized !== void 0 && deserialized !== null ? [deserialized] : [];
|
|
1031
|
+
}
|
|
1032
|
+
if (Array.isArray(rawInput)) {
|
|
1033
|
+
return rawInput;
|
|
1034
|
+
}
|
|
1035
|
+
return rawInput !== void 0 && rawInput !== null ? [rawInput] : [];
|
|
1036
|
+
}
|
|
1037
|
+
function deserializeOutput(spanData) {
|
|
1038
|
+
const outputMeta = spanData.output_meta;
|
|
1039
|
+
const rawOutput = spanData.output;
|
|
1040
|
+
if (outputMeta !== void 0 && outputMeta !== null) {
|
|
1041
|
+
return deserializeValue({ json: rawOutput, meta: outputMeta });
|
|
1042
|
+
}
|
|
1043
|
+
return rawOutput;
|
|
1044
|
+
}
|
|
1045
|
+
function buildMockTree(rootNode) {
|
|
1046
|
+
const spans = /* @__PURE__ */ new Map();
|
|
1047
|
+
const counters = /* @__PURE__ */ new Map();
|
|
1048
|
+
function walk(node) {
|
|
1049
|
+
const key = node.traceFunctionKey;
|
|
1050
|
+
if (key) {
|
|
1051
|
+
const name = node.spanName || key;
|
|
1052
|
+
const counterKey = `${key}:${name}`;
|
|
1053
|
+
const index = counters.get(counterKey) ?? 0;
|
|
1054
|
+
counters.set(counterKey, index + 1);
|
|
1055
|
+
spans.set(`${counterKey}:${index}`, {
|
|
1056
|
+
sourceSpanId: node.sourceSpanId,
|
|
1057
|
+
externalSpanId: node.externalSpanId,
|
|
1058
|
+
output: node.output,
|
|
1059
|
+
outputMeta: node.outputMeta
|
|
1060
|
+
});
|
|
1061
|
+
}
|
|
1062
|
+
for (const child of node.children) {
|
|
1063
|
+
walk(child);
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
for (const child of rootNode.children) {
|
|
1067
|
+
walk(child);
|
|
1068
|
+
}
|
|
1069
|
+
return { spans };
|
|
1070
|
+
}
|
|
1071
|
+
async function processItem(httpClient, serverItem, fn, experimentId, mockStrategy, resolvedOverrides, replayedTraceId, attempt, includeDbBranchLease, dbBranchSettings, adaptInputs, dryRun, selectiveOptions) {
|
|
1072
|
+
let lease = includeDbBranchLease ? serverItem.dbBranchLease : void 0;
|
|
1073
|
+
let leaseError = includeDbBranchLease ? serverItem.dbBranchLeaseError : void 0;
|
|
1074
|
+
let dbSnapshotRef = serverItem.dbSnapshotRef;
|
|
1075
|
+
let dbBranchTimings = includeDbBranchLease ? serverItem.dbBranchTimings : void 0;
|
|
1076
|
+
let inputs = [];
|
|
1077
|
+
let originalOutput;
|
|
1078
|
+
let result;
|
|
1079
|
+
let error = null;
|
|
1080
|
+
let traceError = null;
|
|
1081
|
+
let replayError = null;
|
|
1082
|
+
let replayDurationMs = null;
|
|
1083
|
+
let replayStarted = null;
|
|
1084
|
+
let selectiveReplay;
|
|
1085
|
+
const originalTraceId = serverItem.originalTraceId ?? serverItem.sourceTraceId;
|
|
1086
|
+
const originalSpanId = serverItem.originalSpanId ?? serverItem.sourceSpanId;
|
|
1087
|
+
try {
|
|
1088
|
+
if (includeDbBranchLease && experimentId !== null && !lease && !leaseError) {
|
|
1089
|
+
let resolved;
|
|
1090
|
+
try {
|
|
1091
|
+
resolved = await httpClient.resolveDbBranchLease(
|
|
1092
|
+
experimentId,
|
|
1093
|
+
originalTraceId,
|
|
1094
|
+
dbBranchSettings,
|
|
1095
|
+
attempt
|
|
1096
|
+
);
|
|
1097
|
+
} catch (cause) {
|
|
1098
|
+
throw new DbBranchReplayError(
|
|
1099
|
+
"lease_request_failed",
|
|
1100
|
+
`Bitfab could not request the database branch: ${errorMessage(cause)}`,
|
|
1101
|
+
originalTraceId,
|
|
1102
|
+
cause
|
|
1103
|
+
);
|
|
1104
|
+
}
|
|
1105
|
+
lease = resolved.lease ?? void 0;
|
|
1106
|
+
leaseError = resolved.leaseError ?? void 0;
|
|
1107
|
+
dbSnapshotRef = resolved.dbSnapshotRef ?? dbSnapshotRef;
|
|
1108
|
+
dbBranchTimings = resolved.timings ?? dbBranchTimings;
|
|
1109
|
+
}
|
|
1110
|
+
if (leaseError) {
|
|
1111
|
+
throw new DbBranchReplayError(
|
|
1112
|
+
leaseError.code,
|
|
1113
|
+
leaseError.message,
|
|
1114
|
+
originalTraceId
|
|
1115
|
+
);
|
|
1116
|
+
}
|
|
1117
|
+
if (selectiveOptions !== void 0) {
|
|
1118
|
+
if (serverItem.selectiveReplayPlan === void 0) {
|
|
1119
|
+
throw new BitfabError(
|
|
1120
|
+
"Server did not return a selective replay plan. Update the server or disable experimentalSelectiveReplay."
|
|
1121
|
+
);
|
|
1122
|
+
}
|
|
1123
|
+
selectiveReplay = new SelectiveReplayRuntime(
|
|
1124
|
+
serverItem.selectiveReplayPlan,
|
|
1125
|
+
selectiveOptions
|
|
1126
|
+
);
|
|
1127
|
+
}
|
|
1128
|
+
const span = await httpClient.getExternalSpan(originalSpanId, {
|
|
1129
|
+
view: "replay"
|
|
1130
|
+
});
|
|
1131
|
+
const spanData = span.rawData?.span_data ?? {};
|
|
1132
|
+
inputs = deserializeInputs(spanData);
|
|
1133
|
+
originalOutput = deserializeOutput(spanData);
|
|
1134
|
+
const integrationFixtures = extractIntegrationFixtures(spanData);
|
|
1135
|
+
if (adaptInputs) {
|
|
1136
|
+
const originalMetadata = serverItem.originalMetadata;
|
|
1137
|
+
inputs = adaptInputs(inputs, {
|
|
1138
|
+
originalTraceId,
|
|
1139
|
+
originalSpanId,
|
|
1140
|
+
// Deprecated aliases for originalTraceId/originalSpanId.
|
|
1141
|
+
sourceTraceId: originalTraceId,
|
|
1142
|
+
sourceSpanId: originalSpanId,
|
|
1143
|
+
metadata: isPlainRecord(originalMetadata) ? { ...originalMetadata } : {}
|
|
1144
|
+
});
|
|
1145
|
+
}
|
|
1146
|
+
const hasOverrides = resolvedOverrides.length > 0;
|
|
1147
|
+
const needTree = selectiveReplay === void 0 && (mockStrategy === "all" || mockStrategy === "marked" || hasOverrides);
|
|
1148
|
+
const includeOutputs = mockStrategy === "all";
|
|
1149
|
+
let mockTree;
|
|
1150
|
+
if (needTree) {
|
|
1151
|
+
try {
|
|
1152
|
+
const treeResponse = await httpClient.getSpanTree(originalSpanId, {
|
|
1153
|
+
includeOutputs,
|
|
1154
|
+
includeRootOutput: false
|
|
1155
|
+
});
|
|
1156
|
+
if (!treeResponse.root) {
|
|
1157
|
+
throw new BitfabError(
|
|
1158
|
+
`Replay mock strategy "${mockStrategy}"${hasOverrides ? " with overrides" : ""} requires a span tree root for original span ${originalSpanId}.`
|
|
1159
|
+
);
|
|
1160
|
+
}
|
|
1161
|
+
mockTree = buildMockTree(treeResponse.root);
|
|
1162
|
+
} catch (error2) {
|
|
1163
|
+
if (mockStrategy !== "marked" || hasOverrides) {
|
|
1164
|
+
throw error2;
|
|
1165
|
+
}
|
|
1166
|
+
mockTree = { spans: /* @__PURE__ */ new Map() };
|
|
1167
|
+
}
|
|
1168
|
+
}
|
|
1169
|
+
const outputCache = /* @__PURE__ */ new Map();
|
|
1170
|
+
const fetchSpanOutput = mockTree && !includeOutputs ? (externalSpanId) => {
|
|
1171
|
+
let pending = outputCache.get(externalSpanId);
|
|
1172
|
+
if (!pending) {
|
|
1173
|
+
pending = httpClient.getExternalSpan(externalSpanId, { view: "replay" }).then(
|
|
1174
|
+
(s) => deserializeOutput(
|
|
1175
|
+
s.rawData?.span_data ?? {}
|
|
1176
|
+
)
|
|
1177
|
+
);
|
|
1178
|
+
outputCache.set(externalSpanId, pending);
|
|
1179
|
+
}
|
|
1180
|
+
return pending;
|
|
1181
|
+
} : void 0;
|
|
1182
|
+
if (dryRun || experimentId === null) {
|
|
1183
|
+
return {
|
|
1184
|
+
traceId: null,
|
|
1185
|
+
originalTraceId,
|
|
1186
|
+
originalSpanId,
|
|
1187
|
+
sourceTraceId: originalTraceId,
|
|
1188
|
+
sourceSpanId: originalSpanId,
|
|
1189
|
+
attempt,
|
|
1190
|
+
input: inputs,
|
|
1191
|
+
result: void 0,
|
|
1192
|
+
originalOutput,
|
|
1193
|
+
...serverItem.ingestionType && {
|
|
1194
|
+
ingestionType: serverItem.ingestionType
|
|
1195
|
+
},
|
|
1196
|
+
error: null,
|
|
1197
|
+
traceError: null,
|
|
1198
|
+
replayError: null,
|
|
1199
|
+
durationMs: null,
|
|
1200
|
+
originalDurationMs: serverItem.originalDurationMs ?? serverItem.durationMs ?? null,
|
|
1201
|
+
originalTokens: serverItem.originalTokens ?? serverItem.tokens ?? null,
|
|
1202
|
+
originalModel: serverItem.originalModel ?? serverItem.model ?? null,
|
|
1203
|
+
tokens: null,
|
|
1204
|
+
model: serverItem.originalModel ?? serverItem.model ?? null,
|
|
1205
|
+
dbSnapshotRef: dbSnapshotRef ?? null,
|
|
1206
|
+
dbBranchTimings: dbBranchTimings ?? null
|
|
1207
|
+
};
|
|
1208
|
+
}
|
|
1209
|
+
try {
|
|
1210
|
+
replayStarted = performance.now();
|
|
1211
|
+
const maybePromise = runWithReplayContext(
|
|
1212
|
+
{
|
|
1213
|
+
experimentId,
|
|
1214
|
+
traceId: replayedTraceId,
|
|
1215
|
+
inputSourceSpanId: span.id,
|
|
1216
|
+
inputSourceTraceId: span.externalTraceId,
|
|
1217
|
+
sourceBitfabTraceId: originalTraceId,
|
|
1218
|
+
replayAttempt: attempt,
|
|
1219
|
+
mockTree,
|
|
1220
|
+
selectiveReplay,
|
|
1221
|
+
callCounters: mockTree ? /* @__PURE__ */ new Map() : void 0,
|
|
1222
|
+
mockStrategy,
|
|
1223
|
+
mockOverrides: hasOverrides ? resolvedOverrides : void 0,
|
|
1224
|
+
fetchSpanOutput,
|
|
1225
|
+
integrationFixtures,
|
|
1226
|
+
dbBranchLease: lease,
|
|
1227
|
+
dbBranchTimings
|
|
1228
|
+
},
|
|
1229
|
+
() => fn(...inputs)
|
|
1230
|
+
);
|
|
1231
|
+
result = maybePromise instanceof Promise ? await maybePromise : maybePromise;
|
|
1232
|
+
selectiveReplay?.assertNoSafetyConflict();
|
|
1233
|
+
replayDurationMs = Math.round(performance.now() - replayStarted);
|
|
1234
|
+
} catch (e) {
|
|
1235
|
+
if (replayStarted !== null) {
|
|
1236
|
+
replayDurationMs = Math.round(performance.now() - replayStarted);
|
|
1237
|
+
}
|
|
1238
|
+
traceError = e;
|
|
1239
|
+
error = errorMessage(e);
|
|
1240
|
+
}
|
|
1241
|
+
} catch (e) {
|
|
1242
|
+
if (replayStarted !== null) {
|
|
1243
|
+
replayDurationMs = Math.round(performance.now() - replayStarted);
|
|
1244
|
+
}
|
|
1245
|
+
replayError = e;
|
|
1246
|
+
error = replayItemErrorMessage(e);
|
|
1247
|
+
} finally {
|
|
1248
|
+
if (lease) {
|
|
1249
|
+
try {
|
|
1250
|
+
await httpClient.releaseDbBranchLease(lease.neonBranchId);
|
|
1251
|
+
} catch (e) {
|
|
1252
|
+
try {
|
|
1253
|
+
console.warn(
|
|
1254
|
+
`Bitfab: failed to release DB branch ${lease.neonBranchId} (TTL janitor will catch it): ${e instanceof Error ? e.message : String(e)}`
|
|
1255
|
+
);
|
|
1256
|
+
} catch {
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
const originalDurationMs = serverItem.originalDurationMs ?? serverItem.durationMs ?? null;
|
|
1262
|
+
const originalModel = serverItem.originalModel ?? serverItem.model ?? null;
|
|
1263
|
+
return {
|
|
1264
|
+
// Written in by replay() from the complete-replay response once the server
|
|
1265
|
+
// has minted this replay trace's row. Null until then: the client-side
|
|
1266
|
+
// correlation id (replayedTraceId) is never surfaced as the item's traceId.
|
|
1267
|
+
traceId: null,
|
|
1268
|
+
originalTraceId,
|
|
1269
|
+
originalSpanId,
|
|
1270
|
+
// Deprecated aliases for originalTraceId/originalSpanId.
|
|
1271
|
+
sourceTraceId: originalTraceId,
|
|
1272
|
+
sourceSpanId: originalSpanId,
|
|
1273
|
+
attempt,
|
|
1274
|
+
input: inputs,
|
|
1275
|
+
result,
|
|
1276
|
+
originalOutput,
|
|
1277
|
+
...serverItem.ingestionType && {
|
|
1278
|
+
ingestionType: serverItem.ingestionType
|
|
1279
|
+
},
|
|
1280
|
+
error,
|
|
1281
|
+
traceError,
|
|
1282
|
+
replayError,
|
|
1283
|
+
durationMs: replayDurationMs,
|
|
1284
|
+
...selectiveReplay === void 0 ? {} : { selectiveReplay: selectiveReplay.report() },
|
|
1285
|
+
originalDurationMs,
|
|
1286
|
+
originalTokens: serverItem.originalTokens ?? serverItem.tokens ?? null,
|
|
1287
|
+
originalModel,
|
|
1288
|
+
// Filled in by replay() from the complete-replay response once the
|
|
1289
|
+
// replay traces are persisted and their spans aggregated server-side.
|
|
1290
|
+
// Null here (and on older servers) means "replay tokens not known".
|
|
1291
|
+
tokens: null,
|
|
1292
|
+
model: originalModel,
|
|
1293
|
+
dbSnapshotRef: dbSnapshotRef ?? null,
|
|
1294
|
+
dbBranchTimings: dbBranchTimings ?? null
|
|
1295
|
+
};
|
|
1296
|
+
}
|
|
1297
|
+
async function waitForReplayPersistence(httpClient, experimentId, replayedTraceIds, replayLabels = {}, warn = console.warn) {
|
|
1298
|
+
const deferredSettled = await httpClient.settleDeferredWork(
|
|
1299
|
+
REPLAY_PERSISTENCE_TIMEOUT_MS
|
|
1300
|
+
);
|
|
1301
|
+
if (!deferredSettled) {
|
|
1302
|
+
httpClient.takeTraceDeliveries(replayedTraceIds);
|
|
1303
|
+
throw new BitfabError(
|
|
1304
|
+
`Replay could not settle deferred span work before the deadline, so the expected span counts are incomplete (experiment ${experimentId}).`
|
|
1305
|
+
);
|
|
1306
|
+
}
|
|
1307
|
+
if (!httpClient.hasClosedDeliveries(replayedTraceIds)) {
|
|
1308
|
+
httpClient.takeTraceDeliveries(replayedTraceIds);
|
|
1309
|
+
return {};
|
|
1310
|
+
}
|
|
1311
|
+
const flushed = await flushTraces(REPLAY_PERSISTENCE_TIMEOUT_MS);
|
|
1312
|
+
const deliveries = httpClient.takeTraceDeliveries(replayedTraceIds);
|
|
1313
|
+
const expectedSpanCounts = {};
|
|
1314
|
+
const readBackTraceIds = {};
|
|
1315
|
+
let allDelivered = true;
|
|
1316
|
+
for (const [traceId, delivery] of Object.entries(deliveries)) {
|
|
1317
|
+
if (delivery.serverTraceId !== void 0) {
|
|
1318
|
+
readBackTraceIds[traceId] = delivery.serverTraceId;
|
|
1319
|
+
}
|
|
1320
|
+
if (!delivery.closed) {
|
|
1321
|
+
continue;
|
|
1322
|
+
}
|
|
1323
|
+
expectedSpanCounts[traceId] = delivery.spanCount;
|
|
1324
|
+
allDelivered = allDelivered && delivery.delivered;
|
|
1325
|
+
}
|
|
1326
|
+
if (allDelivered) {
|
|
1327
|
+
return readBackTraceIds;
|
|
1328
|
+
}
|
|
1329
|
+
const deadline = Date.now() + REPLAY_PERSISTENCE_TIMEOUT_MS;
|
|
1330
|
+
let missing = Object.keys(expectedSpanCounts);
|
|
1331
|
+
while (true) {
|
|
1332
|
+
const status = await httpClient.getReplayStatus(
|
|
1333
|
+
experimentId,
|
|
1334
|
+
expectedSpanCounts
|
|
1335
|
+
);
|
|
1336
|
+
const ready = status.traceIds ?? {};
|
|
1337
|
+
Object.assign(readBackTraceIds, ready);
|
|
1338
|
+
missing = Object.keys(expectedSpanCounts).filter(
|
|
1339
|
+
(traceId) => ready[traceId] === void 0
|
|
1340
|
+
);
|
|
1341
|
+
if (missing.length === 0) {
|
|
1342
|
+
return readBackTraceIds;
|
|
1343
|
+
}
|
|
1344
|
+
if (Date.now() >= deadline) {
|
|
1345
|
+
break;
|
|
1346
|
+
}
|
|
1347
|
+
await sleepForReplayPersistence(
|
|
1348
|
+
Math.min(100, Math.max(0, deadline - Date.now()))
|
|
1349
|
+
);
|
|
1350
|
+
}
|
|
1351
|
+
for (const traceId of missing) {
|
|
1352
|
+
warn(
|
|
1353
|
+
unconfirmedReplayWarning(
|
|
1354
|
+
replayLabels[traceId] ?? traceId,
|
|
1355
|
+
REPLAY_PERSISTENCE_TIMEOUT_MS / 1e3,
|
|
1356
|
+
flushed
|
|
1357
|
+
)
|
|
1358
|
+
);
|
|
1359
|
+
}
|
|
1360
|
+
return readBackTraceIds;
|
|
1361
|
+
}
|
|
1362
|
+
function unconfirmedReplayWarning(label, seconds, flushed) {
|
|
1363
|
+
return `[bitfab] replay ${label} could not confirm all spans reached Bitfab within ${seconds}s. The run continues; Bitfab will show this replay as capture incomplete.${flushed ? "" : " Some spans likely never left this process."}`;
|
|
1364
|
+
}
|
|
1365
|
+
function sleepForReplayPersistence(ms) {
|
|
1366
|
+
return new Promise((resolve) => {
|
|
1367
|
+
setTimeout(resolve, ms);
|
|
1368
|
+
});
|
|
1369
|
+
}
|
|
1370
|
+
async function mapWithConcurrency(tasks, maxConcurrency, onSettled, onStarted, shouldStop) {
|
|
1371
|
+
const results = new Array(tasks.length);
|
|
1372
|
+
let nextIndex = 0;
|
|
1373
|
+
async function worker() {
|
|
1374
|
+
while (nextIndex < tasks.length && shouldStop?.() !== true) {
|
|
1375
|
+
const index = nextIndex++;
|
|
1376
|
+
onStarted?.(index);
|
|
1377
|
+
const result = await tasks[index]();
|
|
1378
|
+
results[index] = result;
|
|
1379
|
+
await onSettled?.(result, index);
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
const workers = Array.from(
|
|
1383
|
+
{ length: Math.min(maxConcurrency, tasks.length) },
|
|
1384
|
+
() => worker()
|
|
1385
|
+
);
|
|
1386
|
+
await Promise.all(workers);
|
|
1387
|
+
return results.slice(0, nextIndex);
|
|
1388
|
+
}
|
|
1389
|
+
var JUDGE_ASSERTIONS_DEPRECATION_WARNING = "Bitfab: judgeAssertions is deprecated. Assertion judging is on by default; pass skipAssertionJudging: true (--skip-assertion-judging) to turn it off.";
|
|
1390
|
+
function warnJudgeAssertionsDeprecated(write = console.warn) {
|
|
1391
|
+
writeOnce(
|
|
1392
|
+
"judgeAssertions-deprecated",
|
|
1393
|
+
JUDGE_ASSERTIONS_DEPRECATION_WARNING,
|
|
1394
|
+
write
|
|
1395
|
+
);
|
|
1396
|
+
}
|
|
1397
|
+
async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, registeredOverrides = []) {
|
|
1398
|
+
if (options?.judgeAssertions !== void 0) {
|
|
1399
|
+
warnJudgeAssertionsDeprecated();
|
|
1400
|
+
}
|
|
1401
|
+
if (options?.resume !== void 0) {
|
|
1402
|
+
validateResume(options.resume, options);
|
|
1403
|
+
} else if (options?.force === true) {
|
|
1404
|
+
throw new BitfabError(
|
|
1405
|
+
"force only applies when resuming an experiment. Pass it together with resume."
|
|
1406
|
+
);
|
|
1407
|
+
}
|
|
1408
|
+
if (options?.experimentalSelectiveReplay !== void 0) {
|
|
1409
|
+
validateSelectiveReplayOptions(options.experimentalSelectiveReplay);
|
|
1410
|
+
if ((options.mock ?? "marked") !== "marked" || normalizeMockOverrides(options.mockOverride).length > 0 || registeredOverrides.length > 0) {
|
|
1411
|
+
throw new BitfabError(
|
|
1412
|
+
"experimentalSelectiveReplay requires mock: 'marked' and cannot be combined with mock overrides."
|
|
1413
|
+
);
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
if (options?.traceIds !== void 0) {
|
|
1417
|
+
if (options.traceIds.length === 0) {
|
|
1418
|
+
throw new BitfabError("traceIds must contain at least one trace ID.");
|
|
1419
|
+
}
|
|
1420
|
+
if (options.traceIds.length > 100) {
|
|
1421
|
+
throw new BitfabError(
|
|
1422
|
+
`traceIds supports at most 100 trace IDs per replay (got ${options.traceIds.length}).`
|
|
1423
|
+
);
|
|
1424
|
+
}
|
|
1425
|
+
}
|
|
1426
|
+
const datasetIds = resolveDatasetIds(options);
|
|
1427
|
+
if (options?.limit !== void 0 && options?.traceIds !== void 0) {
|
|
1428
|
+
try {
|
|
1429
|
+
console.warn(
|
|
1430
|
+
"Bitfab: limit is ignored when traceIds is passed: the explicit trace ID list already determines how many traces replay."
|
|
1431
|
+
);
|
|
1432
|
+
} catch {
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
const requestedAttempts = resolveAttempts(options?.attempts);
|
|
1436
|
+
const maxConcurrency = options?.maxConcurrency ?? 10;
|
|
1437
|
+
if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {
|
|
1438
|
+
throw new BitfabError(
|
|
1439
|
+
`maxConcurrency must be a positive integer (got ${maxConcurrency}).`
|
|
1440
|
+
);
|
|
1441
|
+
}
|
|
1442
|
+
await replayContextReady;
|
|
1443
|
+
let codeChangeDescription = options?.codeChangeDescription;
|
|
1444
|
+
let codeChangeFiles = options?.codeChangeFiles;
|
|
1445
|
+
if (codeChangeFiles === void 0) {
|
|
1446
|
+
const captured = await resolveAutoCodeChange(options?.name);
|
|
1447
|
+
if (captured) {
|
|
1448
|
+
codeChangeFiles = captured.files;
|
|
1449
|
+
if (codeChangeDescription === void 0) {
|
|
1450
|
+
codeChangeDescription = captured.description;
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
const includeDbBranchLease = dbBranchEnabled(options?.dbBranch) && options?.dryRun !== true;
|
|
1455
|
+
let session;
|
|
1456
|
+
if (options?.resume === void 0) {
|
|
1457
|
+
const started = await httpClient.startReplay(
|
|
1458
|
+
traceFunctionKey,
|
|
1459
|
+
// limit is meaningless with explicit traceIds (the ID list determines
|
|
1460
|
+
// the count), so it's omitted from the request entirely.
|
|
1461
|
+
options?.traceIds ? void 0 : options?.limit ?? 5,
|
|
1462
|
+
options?.traceIds,
|
|
1463
|
+
options?.name,
|
|
1464
|
+
codeChangeDescription,
|
|
1465
|
+
codeChangeFiles,
|
|
1466
|
+
// A dry run executes nothing, so a branch would be provisioned (and billed)
|
|
1467
|
+
// for code that never runs, and a seeded source's refusal would fail the item
|
|
1468
|
+
// before it could report its resolved inputs.
|
|
1469
|
+
includeDbBranchLease,
|
|
1470
|
+
options?.experimentGroupId,
|
|
1471
|
+
datasetIds,
|
|
1472
|
+
options?.graderIds,
|
|
1473
|
+
resolveDbBranchSettings(options?.dbBranch),
|
|
1474
|
+
requestedAttempts,
|
|
1475
|
+
options?.adaptInputs !== void 0,
|
|
1476
|
+
options?.onlyWithAssertions,
|
|
1477
|
+
options?.skipAssertionJudging === true || options?.judgeAssertions === false,
|
|
1478
|
+
options?.experimentalSelectiveReplay,
|
|
1479
|
+
options?.notes,
|
|
1480
|
+
options?.metadata,
|
|
1481
|
+
options?.dryRun === true,
|
|
1482
|
+
safeReplayInvocation(
|
|
1483
|
+
() => buildReplayInvocation(
|
|
1484
|
+
Object.entries(cliFlags(options) ?? explicitReplayOptions(options))
|
|
1485
|
+
)
|
|
1486
|
+
)
|
|
1487
|
+
);
|
|
1488
|
+
if (started.items.length === 0) {
|
|
1489
|
+
try {
|
|
1490
|
+
console.warn(
|
|
1491
|
+
`Bitfab: no traces matched "${traceFunctionKey}", so this replay ran nothing. Capture a trace, or seed one with seedTrace, before replaying.`
|
|
1492
|
+
);
|
|
1493
|
+
} catch {
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
session = {
|
|
1497
|
+
experimentId: started.experimentId,
|
|
1498
|
+
experimentUrl: started.experimentUrl,
|
|
1499
|
+
attempts: requestedAttempts,
|
|
1500
|
+
work: expandReplayWork(started.items, requestedAttempts),
|
|
1501
|
+
finished: [],
|
|
1502
|
+
includeDbBranchLease,
|
|
1503
|
+
dbBranchSettings: resolveDbBranchSettings(options?.dbBranch)
|
|
1504
|
+
};
|
|
1505
|
+
} else {
|
|
1506
|
+
session = await resumeReplaySession(
|
|
1507
|
+
httpClient,
|
|
1508
|
+
options.resume,
|
|
1509
|
+
traceFunctionKey,
|
|
1510
|
+
codeChangeFiles,
|
|
1511
|
+
options.force,
|
|
1512
|
+
options.dbBranch
|
|
1513
|
+
);
|
|
1514
|
+
}
|
|
1515
|
+
const { experimentId } = session;
|
|
1516
|
+
if (experimentId === null || session.experimentUrl === null) {
|
|
1517
|
+
return runReplaySession(
|
|
1518
|
+
httpClient,
|
|
1519
|
+
fn,
|
|
1520
|
+
session,
|
|
1521
|
+
null,
|
|
1522
|
+
maxConcurrency,
|
|
1523
|
+
() => {
|
|
1524
|
+
},
|
|
1525
|
+
() => {
|
|
1526
|
+
},
|
|
1527
|
+
() => false,
|
|
1528
|
+
options,
|
|
1529
|
+
registeredOverrides
|
|
1530
|
+
);
|
|
1531
|
+
}
|
|
1532
|
+
const fullExperimentUrl = `${serviceUrl}${session.experimentUrl}`;
|
|
1533
|
+
const heartbeat = startReplayHeartbeat(
|
|
1534
|
+
() => httpClient.heartbeatReplay(experimentId)
|
|
1535
|
+
);
|
|
1536
|
+
const interruptReplay = async () => {
|
|
1537
|
+
await heartbeat.stop();
|
|
1538
|
+
return httpClient.interruptReplay(experimentId);
|
|
1539
|
+
};
|
|
1540
|
+
const interruptHandlers = installReplayInterruptHandlers({
|
|
1541
|
+
experimentId,
|
|
1542
|
+
experimentUrl: fullExperimentUrl,
|
|
1543
|
+
interrupt: interruptReplay,
|
|
1544
|
+
onInterrupt: options?.onInterrupt
|
|
1545
|
+
});
|
|
1546
|
+
let completeReturned = false;
|
|
1547
|
+
const markCompleteReturned = () => {
|
|
1548
|
+
completeReturned = true;
|
|
1549
|
+
void heartbeat.stop();
|
|
1550
|
+
};
|
|
1551
|
+
try {
|
|
1552
|
+
return await runReplaySession(
|
|
1553
|
+
httpClient,
|
|
1554
|
+
fn,
|
|
1555
|
+
session,
|
|
1556
|
+
fullExperimentUrl,
|
|
1557
|
+
maxConcurrency,
|
|
1558
|
+
interruptHandlers.remove,
|
|
1559
|
+
markCompleteReturned,
|
|
1560
|
+
interruptHandlers.interrupted,
|
|
1561
|
+
options,
|
|
1562
|
+
registeredOverrides
|
|
1563
|
+
);
|
|
1564
|
+
} catch (error) {
|
|
1565
|
+
if (!completeReturned && !interruptHandlers.interrupted()) {
|
|
1566
|
+
await interruptReplay();
|
|
1567
|
+
}
|
|
1568
|
+
throw error;
|
|
1569
|
+
} finally {
|
|
1570
|
+
interruptHandlers.remove();
|
|
1571
|
+
await heartbeat.stop();
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
function throwIfInterrupted(interrupted, items, experimentId, experimentUrl) {
|
|
1575
|
+
if (!interrupted()) {
|
|
1576
|
+
return;
|
|
1577
|
+
}
|
|
1578
|
+
const cause = new BitfabError(
|
|
1579
|
+
`Experiment ${experimentId} was interrupted, so it was not completed or graded. Resume it by passing { resume: "${experimentId}" } to replay, or with bitfab-replay --resume ${experimentId}.`,
|
|
1580
|
+
experimentUrl
|
|
1581
|
+
);
|
|
1582
|
+
throw new ReplayError(
|
|
1583
|
+
cause.message,
|
|
1584
|
+
items,
|
|
1585
|
+
experimentId,
|
|
1586
|
+
experimentUrl,
|
|
1587
|
+
cause
|
|
1588
|
+
);
|
|
1589
|
+
}
|
|
1590
|
+
async function runReplaySession(httpClient, fn, session, fullExperimentUrl, maxConcurrency, stopInterruptingOnSignal, markCompleteReturned, interrupted, options, registeredOverrides) {
|
|
1591
|
+
const { experimentId, attempts } = session;
|
|
1592
|
+
const mockStrategy = options?.mock ?? "marked";
|
|
1593
|
+
if (experimentId !== null && fullExperimentUrl !== null) {
|
|
1594
|
+
try {
|
|
1595
|
+
options?.onExperimentStart?.({
|
|
1596
|
+
experimentId,
|
|
1597
|
+
experimentUrl: fullExperimentUrl
|
|
1598
|
+
});
|
|
1599
|
+
} catch {
|
|
1600
|
+
}
|
|
1601
|
+
}
|
|
1602
|
+
const resolvedOverrides = [
|
|
1603
|
+
...normalizeMockOverrides(options?.mockOverride),
|
|
1604
|
+
...registeredOverrides
|
|
1605
|
+
];
|
|
1606
|
+
const workItems = session.work;
|
|
1607
|
+
const carriedOverItems = session.finished.map(
|
|
1608
|
+
(finished) => carriedOverReplayItem(finished)
|
|
1609
|
+
);
|
|
1610
|
+
const replayedTraceIds = workItems.map(() => randomUuid());
|
|
1611
|
+
httpClient.trackTraceDeliveries(replayedTraceIds);
|
|
1612
|
+
const tasks = workItems.map(
|
|
1613
|
+
({ serverItem, attempt }, index) => () => processItem(
|
|
1614
|
+
httpClient,
|
|
1615
|
+
serverItem,
|
|
1616
|
+
fn,
|
|
1617
|
+
experimentId,
|
|
1618
|
+
mockStrategy,
|
|
1619
|
+
resolvedOverrides,
|
|
1620
|
+
replayedTraceIds[index],
|
|
1621
|
+
attempt,
|
|
1622
|
+
session.includeDbBranchLease,
|
|
1623
|
+
session.dbBranchSettings,
|
|
1624
|
+
options?.adaptInputs,
|
|
1625
|
+
options?.dryRun === true,
|
|
1626
|
+
options?.experimentalSelectiveReplay
|
|
1627
|
+
)
|
|
1628
|
+
);
|
|
1629
|
+
const total = tasks.length + carriedOverItems.length;
|
|
1630
|
+
const onItemFinish = options?.onItemFinish ?? options?.onProgress;
|
|
1631
|
+
let completed = carriedOverItems.length;
|
|
1632
|
+
let started = carriedOverItems.length;
|
|
1633
|
+
let errored = carriedOverItems.filter((item) => item.error !== null).length;
|
|
1634
|
+
let succeeded = completed - errored;
|
|
1635
|
+
let flushInFlight = null;
|
|
1636
|
+
let flushPending = false;
|
|
1637
|
+
const flushFinishedItemTrace = () => {
|
|
1638
|
+
flushPending = true;
|
|
1639
|
+
if (!flushInFlight) {
|
|
1640
|
+
flushInFlight = (async () => {
|
|
1641
|
+
try {
|
|
1642
|
+
while (flushPending) {
|
|
1643
|
+
flushPending = false;
|
|
1644
|
+
await httpClient.settleDeferredWork(REPLAY_PERSISTENCE_TIMEOUT_MS);
|
|
1645
|
+
await flushTraces(REPLAY_PERSISTENCE_TIMEOUT_MS);
|
|
1646
|
+
}
|
|
1647
|
+
} finally {
|
|
1648
|
+
flushInFlight = null;
|
|
1649
|
+
}
|
|
1650
|
+
})();
|
|
1651
|
+
}
|
|
1652
|
+
return flushInFlight;
|
|
1653
|
+
};
|
|
1654
|
+
const ranItems = await mapWithConcurrency(
|
|
1655
|
+
tasks,
|
|
1656
|
+
maxConcurrency,
|
|
1657
|
+
async (item, index) => {
|
|
1658
|
+
let serverTraceId = null;
|
|
1659
|
+
try {
|
|
1660
|
+
await flushFinishedItemTrace();
|
|
1661
|
+
serverTraceId = httpClient.peekServerTraceId(replayedTraceIds[index]) ?? null;
|
|
1662
|
+
} catch {
|
|
1663
|
+
}
|
|
1664
|
+
item.traceId = serverTraceId;
|
|
1665
|
+
completed += 1;
|
|
1666
|
+
if (item.error === null) {
|
|
1667
|
+
succeeded += 1;
|
|
1668
|
+
} else {
|
|
1669
|
+
errored += 1;
|
|
1670
|
+
}
|
|
1671
|
+
try {
|
|
1672
|
+
onItemFinish?.({
|
|
1673
|
+
experimentId,
|
|
1674
|
+
testRunId: experimentId,
|
|
1675
|
+
completed,
|
|
1676
|
+
total,
|
|
1677
|
+
succeeded,
|
|
1678
|
+
errored,
|
|
1679
|
+
item: {
|
|
1680
|
+
// The server's assigned traces.id, read back off the ingest
|
|
1681
|
+
// response and surfaced as this item finishes. Null only if the
|
|
1682
|
+
// per-item flush could not confirm delivery in time; the end-of-run
|
|
1683
|
+
// barrier then fills the returned ReplayItem. The client-side
|
|
1684
|
+
// placeholder is never surfaced.
|
|
1685
|
+
traceId: serverTraceId,
|
|
1686
|
+
originalTraceId: item.originalTraceId ?? null,
|
|
1687
|
+
originalSpanId: item.originalSpanId ?? null,
|
|
1688
|
+
// Deprecated aliases for originalTraceId/originalSpanId.
|
|
1689
|
+
sourceTraceId: item.originalTraceId ?? null,
|
|
1690
|
+
sourceSpanId: item.originalSpanId ?? null,
|
|
1691
|
+
attempt: item.attempt,
|
|
1692
|
+
input: item.input,
|
|
1693
|
+
result: item.result,
|
|
1694
|
+
originalOutput: item.originalOutput,
|
|
1695
|
+
error: item.error,
|
|
1696
|
+
traceError: item.traceError,
|
|
1697
|
+
replayError: item.replayError,
|
|
1698
|
+
durationMs: item.durationMs,
|
|
1699
|
+
...item.selectiveReplay === void 0 ? {} : { selectiveReplay: item.selectiveReplay },
|
|
1700
|
+
originalDurationMs: item.originalDurationMs,
|
|
1701
|
+
originalTokens: item.originalTokens,
|
|
1702
|
+
originalModel: item.originalModel,
|
|
1703
|
+
ingestionType: item.ingestionType,
|
|
1704
|
+
tokens: item.tokens,
|
|
1705
|
+
model: item.model,
|
|
1706
|
+
dbSnapshotRef: item.dbSnapshotRef,
|
|
1707
|
+
dbBranchTimings: item.dbBranchTimings
|
|
1708
|
+
}
|
|
1709
|
+
});
|
|
1710
|
+
} catch {
|
|
1711
|
+
}
|
|
1712
|
+
},
|
|
1713
|
+
options?.onItemStart ? (index) => {
|
|
1714
|
+
started += 1;
|
|
1715
|
+
const { serverItem, attempt } = workItems[index];
|
|
1716
|
+
const originalTraceId = serverItem.originalTraceId ?? serverItem.sourceTraceId;
|
|
1717
|
+
const originalSpanId = serverItem.originalSpanId ?? serverItem.sourceSpanId;
|
|
1718
|
+
try {
|
|
1719
|
+
options.onItemStart?.({
|
|
1720
|
+
type: "started",
|
|
1721
|
+
experimentId,
|
|
1722
|
+
testRunId: experimentId,
|
|
1723
|
+
started,
|
|
1724
|
+
completed,
|
|
1725
|
+
total,
|
|
1726
|
+
succeeded,
|
|
1727
|
+
errored,
|
|
1728
|
+
item: {
|
|
1729
|
+
originalTraceId,
|
|
1730
|
+
originalSpanId,
|
|
1731
|
+
sourceTraceId: originalTraceId,
|
|
1732
|
+
sourceSpanId: originalSpanId,
|
|
1733
|
+
attempt
|
|
1734
|
+
}
|
|
1735
|
+
});
|
|
1736
|
+
} catch {
|
|
1737
|
+
}
|
|
1738
|
+
} : void 0,
|
|
1739
|
+
interrupted
|
|
1740
|
+
);
|
|
1741
|
+
const resultItems = [...carriedOverItems, ...ranItems];
|
|
1742
|
+
if (options?.dryRun === true || experimentId === null || fullExperimentUrl === null) {
|
|
1743
|
+
if (experimentId !== null && fullExperimentUrl !== null) {
|
|
1744
|
+
throwIfInterrupted(
|
|
1745
|
+
interrupted,
|
|
1746
|
+
resultItems,
|
|
1747
|
+
experimentId,
|
|
1748
|
+
fullExperimentUrl
|
|
1749
|
+
);
|
|
1750
|
+
stopInterruptingOnSignal();
|
|
1751
|
+
await httpClient.completeReplay(experimentId).catch(() => void 0);
|
|
1752
|
+
markCompleteReturned();
|
|
1753
|
+
}
|
|
1754
|
+
const dryResult = {
|
|
1755
|
+
items: resultItems,
|
|
1756
|
+
experimentId,
|
|
1757
|
+
experimentUrl: fullExperimentUrl,
|
|
1758
|
+
testRunId: experimentId,
|
|
1759
|
+
testRunUrl: fullExperimentUrl,
|
|
1760
|
+
attempts
|
|
1761
|
+
};
|
|
1762
|
+
await writeReplayResultFile(dryResult);
|
|
1763
|
+
return dryResult;
|
|
1764
|
+
}
|
|
1765
|
+
throwIfInterrupted(interrupted, resultItems, experimentId, fullExperimentUrl);
|
|
1766
|
+
const deliveredTraceIds = await preserveReplayFailure(
|
|
1767
|
+
() => waitForReplayPersistence(
|
|
1768
|
+
httpClient,
|
|
1769
|
+
experimentId,
|
|
1770
|
+
replayedTraceIds,
|
|
1771
|
+
Object.fromEntries(
|
|
1772
|
+
workItems.map(({ serverItem, attempt }, index) => [
|
|
1773
|
+
replayedTraceIds[index],
|
|
1774
|
+
`${serverItem.originalTraceId ?? serverItem.sourceTraceId}#${attempt}`
|
|
1775
|
+
])
|
|
1776
|
+
)
|
|
1777
|
+
),
|
|
1778
|
+
resultItems,
|
|
1779
|
+
experimentId,
|
|
1780
|
+
fullExperimentUrl
|
|
1781
|
+
);
|
|
1782
|
+
for (let index = 0; index < ranItems.length; index += 1) {
|
|
1783
|
+
const localId = replayedTraceIds[index];
|
|
1784
|
+
const readBack = localId ? deliveredTraceIds[localId] : void 0;
|
|
1785
|
+
if (readBack !== void 0) {
|
|
1786
|
+
ranItems[index].traceId = readBack;
|
|
1787
|
+
}
|
|
1788
|
+
}
|
|
1789
|
+
throwIfInterrupted(interrupted, resultItems, experimentId, fullExperimentUrl);
|
|
1790
|
+
stopInterruptingOnSignal();
|
|
1791
|
+
const completeResult = await preserveReplayFailure(
|
|
1792
|
+
() => httpClient.completeReplay(experimentId),
|
|
1793
|
+
resultItems,
|
|
1794
|
+
experimentId,
|
|
1795
|
+
fullExperimentUrl
|
|
1796
|
+
);
|
|
1797
|
+
markCompleteReturned();
|
|
1798
|
+
const serverTraceIds = completeResult.traceIds;
|
|
1799
|
+
const replayTokens = completeResult.tokens;
|
|
1800
|
+
if (serverTraceIds !== void 0) {
|
|
1801
|
+
const missing = [];
|
|
1802
|
+
let completedCount = 0;
|
|
1803
|
+
for (let index = 0; index < ranItems.length; index += 1) {
|
|
1804
|
+
const item = ranItems[index];
|
|
1805
|
+
const localId = replayedTraceIds[index];
|
|
1806
|
+
const mapped = localId ? serverTraceIds[localId] : void 0;
|
|
1807
|
+
item.traceId = item.traceId ?? mapped ?? null;
|
|
1808
|
+
if (item.error === null) {
|
|
1809
|
+
completedCount += 1;
|
|
1810
|
+
if (mapped === void 0) {
|
|
1811
|
+
missing.push(localId ?? item.originalTraceId);
|
|
1812
|
+
}
|
|
1813
|
+
}
|
|
1814
|
+
if (mapped !== void 0) {
|
|
1815
|
+
item.tokens = replayTokens?.[mapped] ?? null;
|
|
1816
|
+
}
|
|
1817
|
+
}
|
|
1818
|
+
if (completedCount > 0 && missing.length === completedCount) {
|
|
1819
|
+
const serverCount = completeResult.traceCount !== void 0 ? ` The server persisted ${completeResult.traceCount} trace(s) for this run.` : "";
|
|
1820
|
+
const cause = new BitfabError(
|
|
1821
|
+
`Replay completed but the server has no persisted trace for any of the ${completedCount} completed item(s) (experiment ${experimentId}).${serverCount} Trace uploads were awaited, so either the uploads failed (check for "Bitfab: Failed to create" errors above) or the replayed function is not wrapped with withSpan.`
|
|
1822
|
+
);
|
|
1823
|
+
throw new ReplayError(
|
|
1824
|
+
cause.message,
|
|
1825
|
+
resultItems,
|
|
1826
|
+
experimentId,
|
|
1827
|
+
fullExperimentUrl,
|
|
1828
|
+
cause
|
|
1829
|
+
);
|
|
1830
|
+
}
|
|
1831
|
+
if (missing.length > 0) {
|
|
1832
|
+
try {
|
|
1833
|
+
console.error(
|
|
1834
|
+
`Bitfab: server has no persisted trace for ${missing.length} of ${completedCount} completed replay item(s) (experiment ${experimentId}). Their replay token usage is unavailable and they cannot be labeled.`
|
|
1835
|
+
);
|
|
1836
|
+
} catch {
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1840
|
+
const result = {
|
|
1841
|
+
items: resultItems,
|
|
1842
|
+
experimentId,
|
|
1843
|
+
experimentUrl: fullExperimentUrl,
|
|
1844
|
+
testRunId: experimentId,
|
|
1845
|
+
testRunUrl: fullExperimentUrl,
|
|
1846
|
+
attempts
|
|
1847
|
+
};
|
|
1848
|
+
await writeReplayResultFile(result);
|
|
1849
|
+
if (!options?.onItemFinish) {
|
|
1850
|
+
try {
|
|
1851
|
+
options?.onProgress?.({
|
|
1852
|
+
type: "complete",
|
|
1853
|
+
experimentId,
|
|
1854
|
+
testRunId: experimentId,
|
|
1855
|
+
completed: total,
|
|
1856
|
+
total,
|
|
1857
|
+
succeeded,
|
|
1858
|
+
errored,
|
|
1859
|
+
result
|
|
1860
|
+
});
|
|
1861
|
+
} catch {
|
|
1862
|
+
}
|
|
1863
|
+
}
|
|
1864
|
+
return result;
|
|
1865
|
+
}
|
|
1866
|
+
async function writeReplayResultFile(result) {
|
|
1867
|
+
const resultPath = typeof process !== "undefined" ? process.env?.BITFAB_REPLAY_RESULT_PATH : void 0;
|
|
1868
|
+
if (!resultPath) {
|
|
1869
|
+
return;
|
|
1870
|
+
}
|
|
1871
|
+
try {
|
|
1872
|
+
const [{ dirname }, { mkdir, writeFile }] = await Promise.all([
|
|
1873
|
+
import("path"),
|
|
1874
|
+
import("fs/promises")
|
|
1875
|
+
]);
|
|
1876
|
+
await mkdir(dirname(resultPath), { recursive: true });
|
|
1877
|
+
await writeFile(resultPath, `${serializeReplayResult(result)}
|
|
1878
|
+
`);
|
|
1879
|
+
} catch (err) {
|
|
1880
|
+
try {
|
|
1881
|
+
console.warn(
|
|
1882
|
+
`Bitfab: failed to write replay result to BITFAB_REPLAY_RESULT_PATH (${resultPath}): ${err instanceof Error ? err.message : String(err)}`
|
|
1883
|
+
);
|
|
1884
|
+
} catch {
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
export {
|
|
1890
|
+
describeReplayInterrupt,
|
|
1891
|
+
canonicalCliFlag,
|
|
1892
|
+
withCliFlags,
|
|
1893
|
+
BITFAB_PROGRESS_PREFIX,
|
|
1894
|
+
reportReplayProgress,
|
|
1895
|
+
ReplayError,
|
|
1896
|
+
DbBranchReplayError,
|
|
1897
|
+
serializeReplayResult,
|
|
1898
|
+
deserializeInputs,
|
|
1899
|
+
waitForReplayPersistence,
|
|
1900
|
+
sleepForReplayPersistence,
|
|
1901
|
+
JUDGE_ASSERTIONS_DEPRECATION_WARNING,
|
|
1902
|
+
warnJudgeAssertionsDeprecated,
|
|
1903
|
+
replay
|
|
1904
|
+
};
|
|
1905
|
+
//# sourceMappingURL=chunk-NGKTKRCV.js.map
|