bitfab 0.61.2 → 0.61.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/autoTrace.cjs.map +1 -1
- package/dist/autoTrace.d.cts +1 -0
- package/dist/autoTrace.d.ts +1 -0
- package/dist/autoTrace.js +1 -1
- package/dist/{chunk-INA6W7OB.js → chunk-54EGXUZ3.js} +32 -3
- package/dist/chunk-54EGXUZ3.js.map +1 -0
- package/dist/{chunk-BMJNDIVA.js → chunk-K4XCWQBC.js} +1 -1
- package/dist/chunk-K4XCWQBC.js.map +1 -0
- package/dist/{chunk-A4UVO6BQ.js → chunk-PSAEQ4OW.js} +480 -15
- package/dist/chunk-PSAEQ4OW.js.map +1 -0
- package/dist/{chunk-ONZFMRRF.js → chunk-VWRCLZCA.js} +30 -3
- package/dist/chunk-VWRCLZCA.js.map +1 -0
- package/dist/{chunk-MTK6PZHA.js → chunk-WPYBJGTG.js} +21 -3
- package/dist/chunk-WPYBJGTG.js.map +1 -0
- package/dist/{chunk-LUOCT4HG.js → chunk-XGRQMQ2R.js} +38 -7
- package/dist/chunk-XGRQMQ2R.js.map +1 -0
- package/dist/{http-2R6MSRVA.js → http-C2TCBBQV.js} +2 -2
- package/dist/{http-KM6QWKAP.js → http-ONLJPP6W.js} +2 -2
- package/dist/index.cjs +568 -15
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +131 -3
- package/dist/index.d.ts +131 -3
- package/dist/index.js +4 -4
- package/dist/node.cjs +568 -15
- package/dist/node.cjs.map +1 -1
- package/dist/node.d.cts +1 -1
- package/dist/node.d.ts +1 -1
- package/dist/node.js +4 -4
- package/dist/{replay-6UUBJBLK.js → replay-RM24QKEA.js} +3 -3
- package/dist/replayCli.js +2 -2
- package/dist/seedCli.js +2 -2
- package/package.json +1 -1
- package/dist/chunk-A4UVO6BQ.js.map +0 -1
- package/dist/chunk-BMJNDIVA.js.map +0 -1
- package/dist/chunk-INA6W7OB.js.map +0 -1
- package/dist/chunk-LUOCT4HG.js.map +0 -1
- package/dist/chunk-MTK6PZHA.js.map +0 -1
- package/dist/chunk-ONZFMRRF.js.map +0 -1
- /package/dist/{http-2R6MSRVA.js.map → http-C2TCBBQV.js.map} +0 -0
- /package/dist/{http-KM6QWKAP.js.map → http-ONLJPP6W.js.map} +0 -0
- /package/dist/{replay-6UUBJBLK.js.map → replay-RM24QKEA.js.map} +0 -0
|
@@ -4,8 +4,9 @@ import {
|
|
|
4
4
|
flushTraces,
|
|
5
5
|
gitRunner,
|
|
6
6
|
replayContextReady,
|
|
7
|
-
runWithReplayContext
|
|
8
|
-
|
|
7
|
+
runWithReplayContext,
|
|
8
|
+
unrefTimer
|
|
9
|
+
} from "./chunk-54EGXUZ3.js";
|
|
9
10
|
import {
|
|
10
11
|
runWithCaptureSuppressed,
|
|
11
12
|
warnOnce
|
|
@@ -437,6 +438,373 @@ function fallbackUuidV4() {
|
|
|
437
438
|
});
|
|
438
439
|
}
|
|
439
440
|
|
|
441
|
+
// src/replayHeartbeat.ts
|
|
442
|
+
var REPLAY_HEARTBEAT_INTERVAL_MS = 6e4;
|
|
443
|
+
function startReplayHeartbeat(send) {
|
|
444
|
+
let stopped = false;
|
|
445
|
+
let inFlight;
|
|
446
|
+
const timer = setInterval(() => {
|
|
447
|
+
if (stopped || inFlight !== void 0) {
|
|
448
|
+
return;
|
|
449
|
+
}
|
|
450
|
+
inFlight = send().catch(() => void 0).finally(() => {
|
|
451
|
+
inFlight = void 0;
|
|
452
|
+
});
|
|
453
|
+
}, REPLAY_HEARTBEAT_INTERVAL_MS);
|
|
454
|
+
unrefTimer(timer);
|
|
455
|
+
return {
|
|
456
|
+
stop: async () => {
|
|
457
|
+
stopped = true;
|
|
458
|
+
clearInterval(timer);
|
|
459
|
+
await inFlight;
|
|
460
|
+
}
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
// src/replayInterrupt.ts
|
|
465
|
+
var REPLAY_INTERRUPT_SIGNALS = [
|
|
466
|
+
"SIGINT",
|
|
467
|
+
"SIGTERM"
|
|
468
|
+
];
|
|
469
|
+
function signalCapableProcess() {
|
|
470
|
+
if (typeof process === "undefined" || typeof process.on !== "function") {
|
|
471
|
+
return void 0;
|
|
472
|
+
}
|
|
473
|
+
return process;
|
|
474
|
+
}
|
|
475
|
+
function describeReplayInterrupt(experimentId, markedInterrupted) {
|
|
476
|
+
if (markedInterrupted) {
|
|
477
|
+
return `[replay] Experiment ${experimentId} interrupted.`;
|
|
478
|
+
}
|
|
479
|
+
return `[replay] Experiment ${experimentId} stopped, but Bitfab could not mark it interrupted. It will be marked interrupted within 15 minutes. A resume in the next two minutes may be refused as still running.`;
|
|
480
|
+
}
|
|
481
|
+
function reportInterrupted(experimentId, markedInterrupted) {
|
|
482
|
+
try {
|
|
483
|
+
console.error(describeReplayInterrupt(experimentId, markedInterrupted));
|
|
484
|
+
} catch {
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
function installReplayInterruptHandlers(target) {
|
|
488
|
+
const host = signalCapableProcess();
|
|
489
|
+
if (host === void 0) {
|
|
490
|
+
return { remove: () => {
|
|
491
|
+
}, interrupted: () => false };
|
|
492
|
+
}
|
|
493
|
+
const handlers = /* @__PURE__ */ new Map();
|
|
494
|
+
const removeHandlers = () => {
|
|
495
|
+
for (const [signal, handler] of handlers) {
|
|
496
|
+
host.removeListener(signal, handler);
|
|
497
|
+
}
|
|
498
|
+
handlers.clear();
|
|
499
|
+
};
|
|
500
|
+
let interrupting = false;
|
|
501
|
+
const handleSignal = async (signal) => {
|
|
502
|
+
const hostHandlesSignal = host.listenerCount(signal) > 1;
|
|
503
|
+
const markedInterrupted = await target.interrupt();
|
|
504
|
+
removeHandlers();
|
|
505
|
+
if (target.onInterrupt !== void 0) {
|
|
506
|
+
try {
|
|
507
|
+
target.onInterrupt({
|
|
508
|
+
experimentId: target.experimentId,
|
|
509
|
+
experimentUrl: target.experimentUrl,
|
|
510
|
+
signal,
|
|
511
|
+
markedInterrupted
|
|
512
|
+
});
|
|
513
|
+
} catch {
|
|
514
|
+
}
|
|
515
|
+
return;
|
|
516
|
+
}
|
|
517
|
+
reportInterrupted(target.experimentId, markedInterrupted);
|
|
518
|
+
if (!hostHandlesSignal) {
|
|
519
|
+
host.kill(host.pid, signal);
|
|
520
|
+
}
|
|
521
|
+
};
|
|
522
|
+
for (const signal of REPLAY_INTERRUPT_SIGNALS) {
|
|
523
|
+
const handler = () => {
|
|
524
|
+
if (interrupting) {
|
|
525
|
+
return;
|
|
526
|
+
}
|
|
527
|
+
interrupting = true;
|
|
528
|
+
void handleSignal(signal);
|
|
529
|
+
};
|
|
530
|
+
handlers.set(signal, handler);
|
|
531
|
+
host.on(signal, handler);
|
|
532
|
+
}
|
|
533
|
+
return { remove: removeHandlers, interrupted: () => interrupting };
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
// src/selectiveReplay.ts
|
|
537
|
+
function replayNodeKey(node) {
|
|
538
|
+
return JSON.stringify([node.traceFunctionKey, node.spanName]);
|
|
539
|
+
}
|
|
540
|
+
function jsonFingerprint(value) {
|
|
541
|
+
const seen = /* @__PURE__ */ new Set();
|
|
542
|
+
function encode(input) {
|
|
543
|
+
if (input === null || typeof input === "boolean") {
|
|
544
|
+
return JSON.stringify(input);
|
|
545
|
+
}
|
|
546
|
+
if (typeof input === "string" && !input.startsWith("<unserializable")) {
|
|
547
|
+
return JSON.stringify(input);
|
|
548
|
+
}
|
|
549
|
+
if (typeof input === "number" && Number.isFinite(input) && !Object.is(input, -0)) {
|
|
550
|
+
return JSON.stringify(input);
|
|
551
|
+
}
|
|
552
|
+
if (typeof input !== "object" || input === null || seen.has(input)) {
|
|
553
|
+
throw new Error("Not a lossless JSON value");
|
|
554
|
+
}
|
|
555
|
+
seen.add(input);
|
|
556
|
+
const prototype = Object.getPrototypeOf(input);
|
|
557
|
+
if (prototype !== (Array.isArray(input) ? Array.prototype : Object.prototype)) {
|
|
558
|
+
throw new Error("Not a plain JSON object");
|
|
559
|
+
}
|
|
560
|
+
if (Object.getOwnPropertySymbols(input).length > 0) {
|
|
561
|
+
throw new Error("Symbol properties are not recorded");
|
|
562
|
+
}
|
|
563
|
+
const keys = Object.keys(input);
|
|
564
|
+
if (Object.getOwnPropertyNames(input).length !== keys.length + (Array.isArray(input) ? 1 : 0)) {
|
|
565
|
+
throw new Error("Non-enumerable properties are not recorded");
|
|
566
|
+
}
|
|
567
|
+
if (Array.isArray(input) && (keys.length !== input.length || keys.some((key, index) => key !== String(index)))) {
|
|
568
|
+
throw new Error("Not a dense JSON array");
|
|
569
|
+
}
|
|
570
|
+
const properties = keys.map((key) => {
|
|
571
|
+
const descriptor = Object.getOwnPropertyDescriptor(input, key);
|
|
572
|
+
if (!descriptor || !("value" in descriptor)) {
|
|
573
|
+
throw new Error("Accessor properties are not replay inputs");
|
|
574
|
+
}
|
|
575
|
+
return `${Array.isArray(input) ? "" : `${JSON.stringify(key)}:`}${encode(descriptor.value)}`;
|
|
576
|
+
});
|
|
577
|
+
return Array.isArray(input) ? `[${properties.join(",")}]` : `{${properties.join(",")}}`;
|
|
578
|
+
}
|
|
579
|
+
try {
|
|
580
|
+
return encode(value);
|
|
581
|
+
} catch {
|
|
582
|
+
return void 0;
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
// src/selectiveReplayRuntime.ts
|
|
587
|
+
function validateSelectiveReplayOptions(options) {
|
|
588
|
+
const validNodes = (nodes) => Array.isArray(nodes) && nodes.length <= 100 && nodes.every(
|
|
589
|
+
(node) => typeof node?.traceFunctionKey === "string" && node.traceFunctionKey.length > 0 && node.traceFunctionKey.length <= 500 && typeof node.spanName === "string" && node.spanName.length > 0 && node.spanName.length <= 500
|
|
590
|
+
);
|
|
591
|
+
if (!options || !validNodes(options.mustRun) || options.mustRun.length === 0 || Object.keys(options).some((key) => key !== "mustRun")) {
|
|
592
|
+
throw new BitfabError(
|
|
593
|
+
"experimentalSelectiveReplay accepts only 1\u2013100 mustRun with traceFunctionKey and spanName. Declare replayReusable on eligible spans, not reusableNodes in the replay request."
|
|
594
|
+
);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
function policyKey(options) {
|
|
598
|
+
return JSON.stringify(options.mustRun.map(replayNodeKey).sort());
|
|
599
|
+
}
|
|
600
|
+
var SelectiveReplayRuntime = class {
|
|
601
|
+
constructor(plan, options) {
|
|
602
|
+
this.plan = plan;
|
|
603
|
+
this.options = options;
|
|
604
|
+
this.nodes = /* @__PURE__ */ new Map();
|
|
605
|
+
this.children = /* @__PURE__ */ new Map();
|
|
606
|
+
this.runtimeParents = /* @__PURE__ */ new Map();
|
|
607
|
+
this.consumed = /* @__PURE__ */ new Map();
|
|
608
|
+
this.requiredScopes = /* @__PURE__ */ new Set();
|
|
609
|
+
this.recordings = /* @__PURE__ */ new Map();
|
|
610
|
+
this.reached = /* @__PURE__ */ new Set();
|
|
611
|
+
this.decisions = [];
|
|
612
|
+
this.assertionNames = /* @__PURE__ */ new Set();
|
|
613
|
+
this.assertionScopes = /* @__PURE__ */ new Set();
|
|
614
|
+
this.observedNames = /* @__PURE__ */ new Map();
|
|
615
|
+
this.unresolvedAssertions = [];
|
|
616
|
+
validateSelectiveReplayOptions(options);
|
|
617
|
+
if (plan?.version !== 4 || !Array.isArray(plan.nodes) || plan.nodes.length === 0 || plan.nodes.length > 500 || !Array.isArray(plan.unresolvedMustRun) || !Array.isArray(plan.assertions) || !plan.options) {
|
|
618
|
+
throw new BitfabError(
|
|
619
|
+
"Server did not return a compatible selective replay plan. Update the server or disable experimentalSelectiveReplay."
|
|
620
|
+
);
|
|
621
|
+
}
|
|
622
|
+
validateSelectiveReplayOptions(plan.options);
|
|
623
|
+
if (policyKey(plan.options) !== policyKey(options)) {
|
|
624
|
+
throw new BitfabError(
|
|
625
|
+
"Selective replay plan does not match the requested mustRun nodes."
|
|
626
|
+
);
|
|
627
|
+
}
|
|
628
|
+
this.required = new Set(options.mustRun.map(replayNodeKey));
|
|
629
|
+
for (const node of plan.nodes) {
|
|
630
|
+
if (typeof node.id !== "string" || this.nodes.has(node.id) || typeof node.traceFunctionKey !== "string" || typeof node.spanName !== "string" || typeof node.mustRun !== "boolean" || typeof node.reusable !== "boolean") {
|
|
631
|
+
throw new BitfabError("Invalid selective replay node.");
|
|
632
|
+
}
|
|
633
|
+
this.nodes.set(node.id, node);
|
|
634
|
+
if (node.recording !== void 0) {
|
|
635
|
+
this.recordings.set(node.id, {
|
|
636
|
+
input: node.recording.kwargs === void 0 ? jsonFingerprint(node.recording.input) : void 0,
|
|
637
|
+
output: jsonFingerprint(node.recording.output)
|
|
638
|
+
});
|
|
639
|
+
}
|
|
640
|
+
if (node.parentId !== null) {
|
|
641
|
+
const siblings = this.children.get(node.parentId) ?? [];
|
|
642
|
+
siblings.push(node);
|
|
643
|
+
this.children.set(node.parentId, siblings);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
const root = this.nodes.get(plan.rootId);
|
|
647
|
+
if (!root || root.parentId !== null || !root.mustRun) {
|
|
648
|
+
throw new BitfabError("Selective replay requires a live recorded root.");
|
|
649
|
+
}
|
|
650
|
+
const assertionIds = /* @__PURE__ */ new Set();
|
|
651
|
+
let wholeTrace = false;
|
|
652
|
+
for (const assertion of plan.assertions) {
|
|
653
|
+
const target = assertion?.target;
|
|
654
|
+
if (typeof assertion?.id !== "string" || assertion.id.length === 0 || assertionIds.has(assertion.id) || target !== null && (typeof target !== "object" || target.kind !== "output" && target.kind !== "span")) {
|
|
655
|
+
throw new BitfabError(
|
|
656
|
+
"Invalid selective replay assertion requirements."
|
|
657
|
+
);
|
|
658
|
+
}
|
|
659
|
+
assertionIds.add(assertion.id);
|
|
660
|
+
if (target === null) {
|
|
661
|
+
wholeTrace = true;
|
|
662
|
+
} else if (target.kind === "span") {
|
|
663
|
+
if (typeof target.name !== "string" || target.name.length === 0 || target.occurrence !== void 0 && target.occurrence !== "first" && target.occurrence !== "last" && !(Number.isInteger(target.occurrence) && typeof target.occurrence === "number" && target.occurrence >= 0)) {
|
|
664
|
+
throw new BitfabError("Invalid selective replay assertion target.");
|
|
665
|
+
}
|
|
666
|
+
this.assertionNames.add(target.name);
|
|
667
|
+
const count = plan.nodes.filter(
|
|
668
|
+
(node) => node.spanName === target.name
|
|
669
|
+
).length;
|
|
670
|
+
if (count <= (typeof target.occurrence === "number" ? target.occurrence : 0)) {
|
|
671
|
+
this.unresolvedAssertions.push(assertion.id);
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
this.fullTraceForAssertions = wholeTrace || this.unresolvedAssertions.length > 0;
|
|
676
|
+
const recordedKeys = new Set(plan.nodes.map(replayNodeKey));
|
|
677
|
+
const unresolved = options.mustRun.filter(
|
|
678
|
+
(node) => !recordedKeys.has(replayNodeKey(node))
|
|
679
|
+
);
|
|
680
|
+
if (JSON.stringify(unresolved.map(replayNodeKey).sort()) !== JSON.stringify(plan.unresolvedMustRun.map(replayNodeKey).sort())) {
|
|
681
|
+
throw new BitfabError(
|
|
682
|
+
"Selective replay plan has incomplete mustRun coverage."
|
|
683
|
+
);
|
|
684
|
+
}
|
|
685
|
+
for (const node of plan.nodes) {
|
|
686
|
+
const ancestors = /* @__PURE__ */ new Set([node.id]);
|
|
687
|
+
let requiredByManifest = this.required.has(replayNodeKey(node));
|
|
688
|
+
let requiredForAssertion = this.fullTraceForAssertions || this.assertionNames.has(node.spanName);
|
|
689
|
+
let parentId = node.parentId;
|
|
690
|
+
while (parentId !== null) {
|
|
691
|
+
const parent = this.nodes.get(parentId);
|
|
692
|
+
if (!parent || ancestors.has(parentId) || node.mustRun && !parent.mustRun) {
|
|
693
|
+
throw new BitfabError("Invalid selective replay ancestry.");
|
|
694
|
+
}
|
|
695
|
+
ancestors.add(parentId);
|
|
696
|
+
requiredByManifest || (requiredByManifest = this.required.has(replayNodeKey(parent)));
|
|
697
|
+
requiredForAssertion || (requiredForAssertion = this.assertionNames.has(parent.spanName));
|
|
698
|
+
parentId = parent.parentId;
|
|
699
|
+
}
|
|
700
|
+
if (!ancestors.has(plan.rootId) || (requiredByManifest || unresolved.length > 0) && !node.mustRun) {
|
|
701
|
+
throw new BitfabError("Selective replay plan could hide required code.");
|
|
702
|
+
}
|
|
703
|
+
if (requiredForAssertion && !node.mustRun) {
|
|
704
|
+
throw new BitfabError(
|
|
705
|
+
"Selective replay plan could hide assertion evidence."
|
|
706
|
+
);
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
enter(params) {
|
|
711
|
+
const identity = {
|
|
712
|
+
traceFunctionKey: params.traceFunctionKey,
|
|
713
|
+
spanName: params.spanName
|
|
714
|
+
};
|
|
715
|
+
const key = replayNodeKey(identity);
|
|
716
|
+
const fingerprint = jsonFingerprint(params.inputs);
|
|
717
|
+
let node;
|
|
718
|
+
let matchReason = "unmatched-call";
|
|
719
|
+
if (params.parentSpanId === null) {
|
|
720
|
+
node = this.nodes.get(this.plan.rootId);
|
|
721
|
+
} else {
|
|
722
|
+
const recordedParent = this.runtimeParents.get(params.parentSpanId);
|
|
723
|
+
const consumed = this.consumed.get(params.parentSpanId) ?? /* @__PURE__ */ new Set();
|
|
724
|
+
this.consumed.set(params.parentSpanId, consumed);
|
|
725
|
+
const candidates = recordedParent ? (this.children.get(recordedParent) ?? []).filter(
|
|
726
|
+
(candidate) => replayNodeKey(candidate) === key && !consumed.has(candidate.id)
|
|
727
|
+
) : [];
|
|
728
|
+
const exact = fingerprint === void 0 ? [] : candidates.filter(
|
|
729
|
+
(candidate) => this.recordings.get(candidate.id)?.input === fingerprint
|
|
730
|
+
);
|
|
731
|
+
node = exact.length === 1 ? exact[0] : exact.length === 0 && candidates.length === 1 ? candidates[0] : void 0;
|
|
732
|
+
if (node) {
|
|
733
|
+
consumed.add(node.id);
|
|
734
|
+
} else if (candidates.length > 0) {
|
|
735
|
+
matchReason = "ambiguous-call";
|
|
736
|
+
for (const candidate of candidates) {
|
|
737
|
+
consumed.add(candidate.id);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
this.runtimeParents.set(params.spanId, node?.id ?? null);
|
|
742
|
+
const requiredByManifest = this.required.has(key) || params.parentSpanId !== null && this.requiredScopes.has(params.parentSpanId);
|
|
743
|
+
if (requiredByManifest) {
|
|
744
|
+
this.requiredScopes.add(params.spanId);
|
|
745
|
+
}
|
|
746
|
+
const requiredForAssertion = this.fullTraceForAssertions || this.assertionNames.has(params.spanName) || params.parentSpanId !== null && this.assertionScopes.has(params.parentSpanId);
|
|
747
|
+
if (requiredForAssertion) {
|
|
748
|
+
this.assertionScopes.add(params.spanId);
|
|
749
|
+
}
|
|
750
|
+
const mustRun = params.parentSpanId === null || requiredByManifest || node?.mustRun === true || requiredForAssertion || this.plan.unresolvedMustRun.length > 0;
|
|
751
|
+
const recording = node === void 0 ? void 0 : this.recordings.get(node.id);
|
|
752
|
+
const equalInputs = fingerprint !== void 0 && recording?.input === fingerprint;
|
|
753
|
+
const reusable = recording?.output !== void 0 && equalInputs && params.canReuseOutput;
|
|
754
|
+
const reason = mustRun ? requiredByManifest ? "must-run" : requiredForAssertion ? "assertion" : node?.reason ?? "unresolved-must-run" : !node ? matchReason : !equalInputs ? "input-drift-or-missing-recording" : !params.canReuseOutput ? "unsupported-output-boundary" : !params.safetyMock && !params.reuseAllowed ? "reuse-not-declared" : node.reason;
|
|
755
|
+
const decision = (action, why) => {
|
|
756
|
+
this.decisions.push({
|
|
757
|
+
...identity,
|
|
758
|
+
originalSpanId: node?.id ?? null,
|
|
759
|
+
action,
|
|
760
|
+
reason: why
|
|
761
|
+
});
|
|
762
|
+
};
|
|
763
|
+
if (params.safetyMock && (mustRun || !reusable)) {
|
|
764
|
+
decision("blocked", `safety-conflict:${reason}`);
|
|
765
|
+
this.failure = new BitfabError(
|
|
766
|
+
`Selective replay cannot execute or safely reuse safety-mocked node ${params.spanName} (${reason}).`
|
|
767
|
+
);
|
|
768
|
+
throw this.failure;
|
|
769
|
+
}
|
|
770
|
+
if (!mustRun && reusable && (params.safetyMock || node?.reusable === true && params.reuseAllowed)) {
|
|
771
|
+
decision("mock", params.safetyMock ? "safety-mock" : "unchanged-inputs");
|
|
772
|
+
return {
|
|
773
|
+
mock: true,
|
|
774
|
+
output: JSON.parse(recording.output)
|
|
775
|
+
};
|
|
776
|
+
}
|
|
777
|
+
decision("run", reason);
|
|
778
|
+
this.observedNames.set(
|
|
779
|
+
params.spanName,
|
|
780
|
+
(this.observedNames.get(params.spanName) ?? 0) + 1
|
|
781
|
+
);
|
|
782
|
+
if (this.required.has(key)) {
|
|
783
|
+
this.reached.add(key);
|
|
784
|
+
}
|
|
785
|
+
return { mock: false };
|
|
786
|
+
}
|
|
787
|
+
assertNoSafetyConflict() {
|
|
788
|
+
if (this.failure) {
|
|
789
|
+
throw this.failure;
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
report() {
|
|
793
|
+
return {
|
|
794
|
+
decisions: this.decisions,
|
|
795
|
+
mustRunNotReached: this.options.mustRun.filter(
|
|
796
|
+
(node) => !this.reached.has(replayNodeKey(node))
|
|
797
|
+
),
|
|
798
|
+
unresolvedMustRun: this.plan.unresolvedMustRun,
|
|
799
|
+
assertionIds: this.plan.assertions.map((assertion) => assertion.id),
|
|
800
|
+
unresolvedAssertions: this.unresolvedAssertions,
|
|
801
|
+
assertionTargetsNotReached: this.plan.assertions.flatMap(
|
|
802
|
+
({ id, target }) => target?.kind === "span" && (this.observedNames.get(target.name) ?? 0) <= (typeof target.occurrence === "number" ? target.occurrence : 0) ? [id] : []
|
|
803
|
+
)
|
|
804
|
+
};
|
|
805
|
+
}
|
|
806
|
+
};
|
|
807
|
+
|
|
440
808
|
// src/replay.ts
|
|
441
809
|
var REPLAY_PERSISTENCE_TIMEOUT_MS = 3e4;
|
|
442
810
|
var MAX_REPLAY_ATTEMPTS = 100;
|
|
@@ -454,7 +822,8 @@ var RESUME_CONFLICTING_OPTIONS = [
|
|
|
454
822
|
"attempts",
|
|
455
823
|
"graderIds",
|
|
456
824
|
"onlyWithAssertions",
|
|
457
|
-
"dryRun"
|
|
825
|
+
"dryRun",
|
|
826
|
+
"experimentalSelectiveReplay"
|
|
458
827
|
];
|
|
459
828
|
var CARRIED_OVER_ERROR = "This item errored in an earlier run of this experiment, before it was resumed.";
|
|
460
829
|
function validateResume(resume, options) {
|
|
@@ -715,7 +1084,7 @@ function buildMockTree(rootNode) {
|
|
|
715
1084
|
}
|
|
716
1085
|
return { spans };
|
|
717
1086
|
}
|
|
718
|
-
async function processItem(httpClient, serverItem, fn, experimentId, mockStrategy, resolvedOverrides, replayedTraceId, attempt, includeDbBranchLease, dbBranchSettings, adaptInputs, dryRun) {
|
|
1087
|
+
async function processItem(httpClient, serverItem, fn, experimentId, mockStrategy, resolvedOverrides, replayedTraceId, attempt, includeDbBranchLease, dbBranchSettings, adaptInputs, dryRun, selectiveOptions) {
|
|
719
1088
|
let lease = includeDbBranchLease ? serverItem.dbBranchLease : void 0;
|
|
720
1089
|
let leaseError = includeDbBranchLease ? serverItem.dbBranchLeaseError : void 0;
|
|
721
1090
|
let dbSnapshotRef = serverItem.dbSnapshotRef;
|
|
@@ -728,6 +1097,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
|
|
|
728
1097
|
let replayError = null;
|
|
729
1098
|
let replayDurationMs = null;
|
|
730
1099
|
let replayStarted = null;
|
|
1100
|
+
let selectiveReplay;
|
|
731
1101
|
const originalTraceId = serverItem.originalTraceId ?? serverItem.sourceTraceId;
|
|
732
1102
|
const originalSpanId = serverItem.originalSpanId ?? serverItem.sourceSpanId;
|
|
733
1103
|
try {
|
|
@@ -760,6 +1130,17 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
|
|
|
760
1130
|
originalTraceId
|
|
761
1131
|
);
|
|
762
1132
|
}
|
|
1133
|
+
if (selectiveOptions !== void 0) {
|
|
1134
|
+
if (serverItem.selectiveReplayPlan === void 0) {
|
|
1135
|
+
throw new BitfabError(
|
|
1136
|
+
"Server did not return a selective replay plan. Update the server or disable experimentalSelectiveReplay."
|
|
1137
|
+
);
|
|
1138
|
+
}
|
|
1139
|
+
selectiveReplay = new SelectiveReplayRuntime(
|
|
1140
|
+
serverItem.selectiveReplayPlan,
|
|
1141
|
+
selectiveOptions
|
|
1142
|
+
);
|
|
1143
|
+
}
|
|
763
1144
|
const span = await httpClient.getExternalSpan(originalSpanId, {
|
|
764
1145
|
view: "replay"
|
|
765
1146
|
});
|
|
@@ -779,7 +1160,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
|
|
|
779
1160
|
});
|
|
780
1161
|
}
|
|
781
1162
|
const hasOverrides = resolvedOverrides.length > 0;
|
|
782
|
-
const needTree = mockStrategy === "all" || mockStrategy === "marked" || hasOverrides;
|
|
1163
|
+
const needTree = selectiveReplay === void 0 && (mockStrategy === "all" || mockStrategy === "marked" || hasOverrides);
|
|
783
1164
|
const includeOutputs = mockStrategy === "all";
|
|
784
1165
|
let mockTree;
|
|
785
1166
|
if (needTree) {
|
|
@@ -854,6 +1235,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
|
|
|
854
1235
|
sourceBitfabTraceId: originalTraceId,
|
|
855
1236
|
replayAttempt: attempt,
|
|
856
1237
|
mockTree,
|
|
1238
|
+
selectiveReplay,
|
|
857
1239
|
callCounters: mockTree ? /* @__PURE__ */ new Map() : void 0,
|
|
858
1240
|
mockStrategy,
|
|
859
1241
|
mockOverrides: hasOverrides ? resolvedOverrides : void 0,
|
|
@@ -865,6 +1247,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
|
|
|
865
1247
|
() => fn(...inputs)
|
|
866
1248
|
);
|
|
867
1249
|
result = maybePromise instanceof Promise ? await maybePromise : maybePromise;
|
|
1250
|
+
selectiveReplay?.assertNoSafetyConflict();
|
|
868
1251
|
replayDurationMs = Math.round(performance.now() - replayStarted);
|
|
869
1252
|
} catch (e) {
|
|
870
1253
|
if (replayStarted !== null) {
|
|
@@ -916,6 +1299,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
|
|
|
916
1299
|
traceError,
|
|
917
1300
|
replayError,
|
|
918
1301
|
durationMs: replayDurationMs,
|
|
1302
|
+
...selectiveReplay === void 0 ? {} : { selectiveReplay: selectiveReplay.report() },
|
|
919
1303
|
originalDurationMs,
|
|
920
1304
|
originalTokens: serverItem.originalTokens ?? serverItem.tokens ?? null,
|
|
921
1305
|
originalModel,
|
|
@@ -1020,6 +1404,14 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1020
1404
|
"force only applies when resuming an experiment. Pass it together with resume."
|
|
1021
1405
|
);
|
|
1022
1406
|
}
|
|
1407
|
+
if (options?.experimentalSelectiveReplay !== void 0) {
|
|
1408
|
+
validateSelectiveReplayOptions(options.experimentalSelectiveReplay);
|
|
1409
|
+
if ((options.mock ?? "marked") !== "marked" || normalizeMockOverrides(options.mockOverride).length > 0 || registeredOverrides.length > 0) {
|
|
1410
|
+
throw new BitfabError(
|
|
1411
|
+
"experimentalSelectiveReplay requires mock: 'marked' and cannot be combined with mock overrides."
|
|
1412
|
+
);
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1023
1415
|
if (options?.traceIds !== void 0) {
|
|
1024
1416
|
if (options.traceIds.length === 0) {
|
|
1025
1417
|
throw new BitfabError("traceIds must contain at least one trace ID.");
|
|
@@ -1061,7 +1453,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1061
1453
|
const includeDbBranchLease = dbBranchEnabled(options?.dbBranch) && options?.dryRun !== true;
|
|
1062
1454
|
let session;
|
|
1063
1455
|
if (options?.resume === void 0) {
|
|
1064
|
-
const
|
|
1456
|
+
const started = await httpClient.startReplay(
|
|
1065
1457
|
traceFunctionKey,
|
|
1066
1458
|
// limit is meaningless with explicit traceIds (the ID list determines
|
|
1067
1459
|
// the count), so it's omitted from the request entirely.
|
|
@@ -1081,9 +1473,10 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1081
1473
|
requestedAttempts,
|
|
1082
1474
|
options?.adaptInputs !== void 0,
|
|
1083
1475
|
options?.onlyWithAssertions,
|
|
1084
|
-
options?.judgeAssertions
|
|
1476
|
+
options?.judgeAssertions,
|
|
1477
|
+
options?.experimentalSelectiveReplay
|
|
1085
1478
|
);
|
|
1086
|
-
if (
|
|
1479
|
+
if (started.items.length === 0) {
|
|
1087
1480
|
try {
|
|
1088
1481
|
console.warn(
|
|
1089
1482
|
`Bitfab: no traces matched "${traceFunctionKey}", so this replay ran nothing. Capture a trace, or seed one with seedTrace, before replaying.`
|
|
@@ -1092,10 +1485,10 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1092
1485
|
}
|
|
1093
1486
|
}
|
|
1094
1487
|
session = {
|
|
1095
|
-
experimentId:
|
|
1096
|
-
experimentUrl:
|
|
1488
|
+
experimentId: started.experimentId,
|
|
1489
|
+
experimentUrl: started.experimentUrl,
|
|
1097
1490
|
attempts: requestedAttempts,
|
|
1098
|
-
work: expandReplayWork(
|
|
1491
|
+
work: expandReplayWork(started.items, requestedAttempts),
|
|
1099
1492
|
finished: [],
|
|
1100
1493
|
includeDbBranchLease,
|
|
1101
1494
|
dbBranchSettings: resolveDbBranchSettings(options?.dbBranch)
|
|
@@ -1110,9 +1503,68 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1110
1503
|
options.dbBranch
|
|
1111
1504
|
);
|
|
1112
1505
|
}
|
|
1113
|
-
const { experimentId
|
|
1506
|
+
const { experimentId } = session;
|
|
1507
|
+
const fullExperimentUrl = `${serviceUrl}${session.experimentUrl}`;
|
|
1508
|
+
const heartbeat = startReplayHeartbeat(
|
|
1509
|
+
() => httpClient.heartbeatReplay(experimentId)
|
|
1510
|
+
);
|
|
1511
|
+
const interruptReplay = async () => {
|
|
1512
|
+
await heartbeat.stop();
|
|
1513
|
+
return httpClient.interruptReplay(experimentId);
|
|
1514
|
+
};
|
|
1515
|
+
const interruptHandlers = installReplayInterruptHandlers({
|
|
1516
|
+
experimentId,
|
|
1517
|
+
experimentUrl: fullExperimentUrl,
|
|
1518
|
+
interrupt: interruptReplay,
|
|
1519
|
+
onInterrupt: options?.onInterrupt
|
|
1520
|
+
});
|
|
1521
|
+
let completeReturned = false;
|
|
1522
|
+
const markCompleteReturned = () => {
|
|
1523
|
+
completeReturned = true;
|
|
1524
|
+
void heartbeat.stop();
|
|
1525
|
+
};
|
|
1526
|
+
try {
|
|
1527
|
+
return await runReplaySession(
|
|
1528
|
+
httpClient,
|
|
1529
|
+
fn,
|
|
1530
|
+
session,
|
|
1531
|
+
fullExperimentUrl,
|
|
1532
|
+
maxConcurrency,
|
|
1533
|
+
interruptHandlers.remove,
|
|
1534
|
+
markCompleteReturned,
|
|
1535
|
+
interruptHandlers.interrupted,
|
|
1536
|
+
options,
|
|
1537
|
+
registeredOverrides
|
|
1538
|
+
);
|
|
1539
|
+
} catch (error) {
|
|
1540
|
+
if (!completeReturned && !interruptHandlers.interrupted()) {
|
|
1541
|
+
await interruptReplay();
|
|
1542
|
+
}
|
|
1543
|
+
throw error;
|
|
1544
|
+
} finally {
|
|
1545
|
+
interruptHandlers.remove();
|
|
1546
|
+
await heartbeat.stop();
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1549
|
+
function throwIfInterrupted(interrupted, items, experimentId, experimentUrl) {
|
|
1550
|
+
if (!interrupted()) {
|
|
1551
|
+
return;
|
|
1552
|
+
}
|
|
1553
|
+
const cause = new BitfabError(
|
|
1554
|
+
`Experiment ${experimentId} was interrupted, so it was not completed or graded. Resume it by passing { resume: "${experimentId}" } to replay, or with bitfab-replay --resume ${experimentId}.`,
|
|
1555
|
+
experimentUrl
|
|
1556
|
+
);
|
|
1557
|
+
throw new ReplayError(
|
|
1558
|
+
cause.message,
|
|
1559
|
+
items,
|
|
1560
|
+
experimentId,
|
|
1561
|
+
experimentUrl,
|
|
1562
|
+
cause
|
|
1563
|
+
);
|
|
1564
|
+
}
|
|
1565
|
+
async function runReplaySession(httpClient, fn, session, fullExperimentUrl, maxConcurrency, stopInterruptingOnSignal, markCompleteReturned, interrupted, options, registeredOverrides) {
|
|
1566
|
+
const { experimentId, attempts } = session;
|
|
1114
1567
|
const mockStrategy = options?.mock ?? "marked";
|
|
1115
|
-
const fullExperimentUrl = `${serviceUrl}${experimentUrl}`;
|
|
1116
1568
|
try {
|
|
1117
1569
|
options?.onExperimentStart?.({
|
|
1118
1570
|
experimentId,
|
|
@@ -1143,7 +1595,8 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1143
1595
|
session.includeDbBranchLease,
|
|
1144
1596
|
session.dbBranchSettings,
|
|
1145
1597
|
options?.adaptInputs,
|
|
1146
|
-
options?.dryRun === true
|
|
1598
|
+
options?.dryRun === true,
|
|
1599
|
+
options?.experimentalSelectiveReplay
|
|
1147
1600
|
)
|
|
1148
1601
|
);
|
|
1149
1602
|
const total = tasks.length + carriedOverItems.length;
|
|
@@ -1216,6 +1669,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1216
1669
|
traceError: item.traceError,
|
|
1217
1670
|
replayError: item.replayError,
|
|
1218
1671
|
durationMs: item.durationMs,
|
|
1672
|
+
...item.selectiveReplay === void 0 ? {} : { selectiveReplay: item.selectiveReplay },
|
|
1219
1673
|
originalDurationMs: item.originalDurationMs,
|
|
1220
1674
|
originalTokens: item.originalTokens,
|
|
1221
1675
|
originalModel: item.originalModel,
|
|
@@ -1260,7 +1714,15 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1260
1714
|
);
|
|
1261
1715
|
const resultItems = [...carriedOverItems, ...ranItems];
|
|
1262
1716
|
if (options?.dryRun === true) {
|
|
1717
|
+
throwIfInterrupted(
|
|
1718
|
+
interrupted,
|
|
1719
|
+
resultItems,
|
|
1720
|
+
experimentId,
|
|
1721
|
+
fullExperimentUrl
|
|
1722
|
+
);
|
|
1723
|
+
stopInterruptingOnSignal();
|
|
1263
1724
|
await httpClient.completeReplay(experimentId).catch(() => void 0);
|
|
1725
|
+
markCompleteReturned();
|
|
1264
1726
|
const dryResult = {
|
|
1265
1727
|
items: resultItems,
|
|
1266
1728
|
experimentId,
|
|
@@ -1285,12 +1747,15 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1285
1747
|
ranItems[index].traceId = readBack;
|
|
1286
1748
|
}
|
|
1287
1749
|
}
|
|
1750
|
+
throwIfInterrupted(interrupted, resultItems, experimentId, fullExperimentUrl);
|
|
1751
|
+
stopInterruptingOnSignal();
|
|
1288
1752
|
const completeResult = await preserveReplayFailure(
|
|
1289
1753
|
() => httpClient.completeReplay(experimentId),
|
|
1290
1754
|
resultItems,
|
|
1291
1755
|
experimentId,
|
|
1292
1756
|
fullExperimentUrl
|
|
1293
1757
|
);
|
|
1758
|
+
markCompleteReturned();
|
|
1294
1759
|
const serverTraceIds = completeResult.traceIds;
|
|
1295
1760
|
const replayTokens = completeResult.tokens;
|
|
1296
1761
|
if (serverTraceIds !== void 0) {
|
|
@@ -1403,4 +1868,4 @@ export {
|
|
|
1403
1868
|
sleepForReplayPersistence,
|
|
1404
1869
|
replay
|
|
1405
1870
|
};
|
|
1406
|
-
//# sourceMappingURL=chunk-
|
|
1871
|
+
//# sourceMappingURL=chunk-PSAEQ4OW.js.map
|