bitfab 0.61.1 → 0.61.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/autoTrace.cjs.map +1 -1
- package/dist/autoTrace.d.cts +1 -0
- package/dist/autoTrace.d.ts +1 -0
- package/dist/autoTrace.js +1 -1
- package/dist/{chunk-HKUHPLAM.js → chunk-6IMUONDN.js} +47 -3
- package/dist/chunk-6IMUONDN.js.map +1 -0
- package/dist/{chunk-I5J2FDKH.js → chunk-HVEZHLVQ.js} +46 -3
- package/dist/chunk-HVEZHLVQ.js.map +1 -0
- package/dist/{chunk-VKGLQL7V.js → chunk-IZUZ5NBB.js} +473 -51
- package/dist/chunk-IZUZ5NBB.js.map +1 -0
- package/dist/{chunk-BMJNDIVA.js → chunk-K4XCWQBC.js} +1 -1
- package/dist/chunk-K4XCWQBC.js.map +1 -0
- package/dist/{chunk-ENHRFXCR.js → chunk-PAUXHTNF.js} +38 -7
- package/dist/chunk-PAUXHTNF.js.map +1 -0
- package/dist/{chunk-S4WE6U2Y.js → chunk-WOHIAKZF.js} +122 -52
- package/dist/chunk-WOHIAKZF.js.map +1 -0
- package/dist/{http-5DA23DKC.js → http-3IYU5Z54.js} +2 -2
- package/dist/{http-NY6M2AK3.js → http-BYICBOWP.js} +2 -2
- package/dist/index.cjs +563 -52
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +146 -3
- package/dist/index.d.ts +146 -3
- package/dist/index.js +4 -4
- package/dist/node.cjs +563 -52
- package/dist/node.cjs.map +1 -1
- package/dist/node.d.cts +1 -1
- package/dist/node.d.ts +1 -1
- package/dist/node.js +4 -4
- package/dist/{replay-KVENJRCX.js → replay-SXHX5BBU.js} +3 -3
- package/dist/replayCli.js +4 -3
- package/dist/replayCli.js.map +1 -1
- package/dist/seedCli.js +2 -2
- package/package.json +1 -1
- package/dist/chunk-BMJNDIVA.js.map +0 -1
- package/dist/chunk-ENHRFXCR.js.map +0 -1
- package/dist/chunk-HKUHPLAM.js.map +0 -1
- package/dist/chunk-I5J2FDKH.js.map +0 -1
- package/dist/chunk-S4WE6U2Y.js.map +0 -1
- package/dist/chunk-VKGLQL7V.js.map +0 -1
- /package/dist/{http-5DA23DKC.js.map → http-3IYU5Z54.js.map} +0 -0
- /package/dist/{http-NY6M2AK3.js.map → http-BYICBOWP.js.map} +0 -0
- /package/dist/{replay-KVENJRCX.js.map → replay-SXHX5BBU.js.map} +0 -0
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
gitRunner,
|
|
6
6
|
replayContextReady,
|
|
7
7
|
runWithReplayContext
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-6IMUONDN.js";
|
|
9
9
|
import {
|
|
10
10
|
runWithCaptureSuppressed,
|
|
11
11
|
warnOnce
|
|
@@ -437,6 +437,278 @@ function fallbackUuidV4() {
|
|
|
437
437
|
});
|
|
438
438
|
}
|
|
439
439
|
|
|
440
|
+
// src/selectiveReplay.ts
|
|
441
|
+
function replayNodeKey(node) {
|
|
442
|
+
return JSON.stringify([node.traceFunctionKey, node.spanName]);
|
|
443
|
+
}
|
|
444
|
+
function jsonFingerprint(value) {
|
|
445
|
+
const seen = /* @__PURE__ */ new Set();
|
|
446
|
+
function encode(input) {
|
|
447
|
+
if (input === null || typeof input === "boolean") {
|
|
448
|
+
return JSON.stringify(input);
|
|
449
|
+
}
|
|
450
|
+
if (typeof input === "string" && !input.startsWith("<unserializable")) {
|
|
451
|
+
return JSON.stringify(input);
|
|
452
|
+
}
|
|
453
|
+
if (typeof input === "number" && Number.isFinite(input) && !Object.is(input, -0)) {
|
|
454
|
+
return JSON.stringify(input);
|
|
455
|
+
}
|
|
456
|
+
if (typeof input !== "object" || input === null || seen.has(input)) {
|
|
457
|
+
throw new Error("Not a lossless JSON value");
|
|
458
|
+
}
|
|
459
|
+
seen.add(input);
|
|
460
|
+
const prototype = Object.getPrototypeOf(input);
|
|
461
|
+
if (prototype !== (Array.isArray(input) ? Array.prototype : Object.prototype)) {
|
|
462
|
+
throw new Error("Not a plain JSON object");
|
|
463
|
+
}
|
|
464
|
+
if (Object.getOwnPropertySymbols(input).length > 0) {
|
|
465
|
+
throw new Error("Symbol properties are not recorded");
|
|
466
|
+
}
|
|
467
|
+
const keys = Object.keys(input);
|
|
468
|
+
if (Object.getOwnPropertyNames(input).length !== keys.length + (Array.isArray(input) ? 1 : 0)) {
|
|
469
|
+
throw new Error("Non-enumerable properties are not recorded");
|
|
470
|
+
}
|
|
471
|
+
if (Array.isArray(input) && (keys.length !== input.length || keys.some((key, index) => key !== String(index)))) {
|
|
472
|
+
throw new Error("Not a dense JSON array");
|
|
473
|
+
}
|
|
474
|
+
const properties = keys.map((key) => {
|
|
475
|
+
const descriptor = Object.getOwnPropertyDescriptor(input, key);
|
|
476
|
+
if (!descriptor || !("value" in descriptor)) {
|
|
477
|
+
throw new Error("Accessor properties are not replay inputs");
|
|
478
|
+
}
|
|
479
|
+
return `${Array.isArray(input) ? "" : `${JSON.stringify(key)}:`}${encode(descriptor.value)}`;
|
|
480
|
+
});
|
|
481
|
+
return Array.isArray(input) ? `[${properties.join(",")}]` : `{${properties.join(",")}}`;
|
|
482
|
+
}
|
|
483
|
+
try {
|
|
484
|
+
return encode(value);
|
|
485
|
+
} catch {
|
|
486
|
+
return void 0;
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
// src/selectiveReplayRuntime.ts
|
|
491
|
+
function validateSelectiveReplayOptions(options) {
|
|
492
|
+
const validNodes = (nodes) => Array.isArray(nodes) && nodes.length <= 100 && nodes.every(
|
|
493
|
+
(node) => typeof node?.traceFunctionKey === "string" && node.traceFunctionKey.length > 0 && node.traceFunctionKey.length <= 500 && typeof node.spanName === "string" && node.spanName.length > 0 && node.spanName.length <= 500
|
|
494
|
+
);
|
|
495
|
+
if (!options || !validNodes(options.mustRun) || options.mustRun.length === 0 || Object.keys(options).some((key) => key !== "mustRun")) {
|
|
496
|
+
throw new BitfabError(
|
|
497
|
+
"experimentalSelectiveReplay accepts only 1\u2013100 mustRun with traceFunctionKey and spanName. Declare replayReusable on eligible spans, not reusableNodes in the replay request."
|
|
498
|
+
);
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
function policyKey(options) {
|
|
502
|
+
return JSON.stringify(options.mustRun.map(replayNodeKey).sort());
|
|
503
|
+
}
|
|
504
|
+
var SelectiveReplayRuntime = class {
|
|
505
|
+
constructor(plan, options) {
|
|
506
|
+
this.plan = plan;
|
|
507
|
+
this.options = options;
|
|
508
|
+
this.nodes = /* @__PURE__ */ new Map();
|
|
509
|
+
this.children = /* @__PURE__ */ new Map();
|
|
510
|
+
this.runtimeParents = /* @__PURE__ */ new Map();
|
|
511
|
+
this.consumed = /* @__PURE__ */ new Map();
|
|
512
|
+
this.requiredScopes = /* @__PURE__ */ new Set();
|
|
513
|
+
this.recordings = /* @__PURE__ */ new Map();
|
|
514
|
+
this.reached = /* @__PURE__ */ new Set();
|
|
515
|
+
this.decisions = [];
|
|
516
|
+
this.assertionNames = /* @__PURE__ */ new Set();
|
|
517
|
+
this.assertionScopes = /* @__PURE__ */ new Set();
|
|
518
|
+
this.observedNames = /* @__PURE__ */ new Map();
|
|
519
|
+
this.unresolvedAssertions = [];
|
|
520
|
+
validateSelectiveReplayOptions(options);
|
|
521
|
+
if (plan?.version !== 4 || !Array.isArray(plan.nodes) || plan.nodes.length === 0 || plan.nodes.length > 500 || !Array.isArray(plan.unresolvedMustRun) || !Array.isArray(plan.assertions) || !plan.options) {
|
|
522
|
+
throw new BitfabError(
|
|
523
|
+
"Server did not return a compatible selective replay plan. Update the server or disable experimentalSelectiveReplay."
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
validateSelectiveReplayOptions(plan.options);
|
|
527
|
+
if (policyKey(plan.options) !== policyKey(options)) {
|
|
528
|
+
throw new BitfabError(
|
|
529
|
+
"Selective replay plan does not match the requested mustRun nodes."
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
this.required = new Set(options.mustRun.map(replayNodeKey));
|
|
533
|
+
for (const node of plan.nodes) {
|
|
534
|
+
if (typeof node.id !== "string" || this.nodes.has(node.id) || typeof node.traceFunctionKey !== "string" || typeof node.spanName !== "string" || typeof node.mustRun !== "boolean" || typeof node.reusable !== "boolean") {
|
|
535
|
+
throw new BitfabError("Invalid selective replay node.");
|
|
536
|
+
}
|
|
537
|
+
this.nodes.set(node.id, node);
|
|
538
|
+
if (node.recording !== void 0) {
|
|
539
|
+
this.recordings.set(node.id, {
|
|
540
|
+
input: node.recording.kwargs === void 0 ? jsonFingerprint(node.recording.input) : void 0,
|
|
541
|
+
output: jsonFingerprint(node.recording.output)
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
if (node.parentId !== null) {
|
|
545
|
+
const siblings = this.children.get(node.parentId) ?? [];
|
|
546
|
+
siblings.push(node);
|
|
547
|
+
this.children.set(node.parentId, siblings);
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
const root = this.nodes.get(plan.rootId);
|
|
551
|
+
if (!root || root.parentId !== null || !root.mustRun) {
|
|
552
|
+
throw new BitfabError("Selective replay requires a live recorded root.");
|
|
553
|
+
}
|
|
554
|
+
const assertionIds = /* @__PURE__ */ new Set();
|
|
555
|
+
let wholeTrace = false;
|
|
556
|
+
for (const assertion of plan.assertions) {
|
|
557
|
+
const target = assertion?.target;
|
|
558
|
+
if (typeof assertion?.id !== "string" || assertion.id.length === 0 || assertionIds.has(assertion.id) || target !== null && (typeof target !== "object" || target.kind !== "output" && target.kind !== "span")) {
|
|
559
|
+
throw new BitfabError(
|
|
560
|
+
"Invalid selective replay assertion requirements."
|
|
561
|
+
);
|
|
562
|
+
}
|
|
563
|
+
assertionIds.add(assertion.id);
|
|
564
|
+
if (target === null) {
|
|
565
|
+
wholeTrace = true;
|
|
566
|
+
} else if (target.kind === "span") {
|
|
567
|
+
if (typeof target.name !== "string" || target.name.length === 0 || target.occurrence !== void 0 && target.occurrence !== "first" && target.occurrence !== "last" && !(Number.isInteger(target.occurrence) && typeof target.occurrence === "number" && target.occurrence >= 0)) {
|
|
568
|
+
throw new BitfabError("Invalid selective replay assertion target.");
|
|
569
|
+
}
|
|
570
|
+
this.assertionNames.add(target.name);
|
|
571
|
+
const count = plan.nodes.filter(
|
|
572
|
+
(node) => node.spanName === target.name
|
|
573
|
+
).length;
|
|
574
|
+
if (count <= (typeof target.occurrence === "number" ? target.occurrence : 0)) {
|
|
575
|
+
this.unresolvedAssertions.push(assertion.id);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
this.fullTraceForAssertions = wholeTrace || this.unresolvedAssertions.length > 0;
|
|
580
|
+
const recordedKeys = new Set(plan.nodes.map(replayNodeKey));
|
|
581
|
+
const unresolved = options.mustRun.filter(
|
|
582
|
+
(node) => !recordedKeys.has(replayNodeKey(node))
|
|
583
|
+
);
|
|
584
|
+
if (JSON.stringify(unresolved.map(replayNodeKey).sort()) !== JSON.stringify(plan.unresolvedMustRun.map(replayNodeKey).sort())) {
|
|
585
|
+
throw new BitfabError(
|
|
586
|
+
"Selective replay plan has incomplete mustRun coverage."
|
|
587
|
+
);
|
|
588
|
+
}
|
|
589
|
+
for (const node of plan.nodes) {
|
|
590
|
+
const ancestors = /* @__PURE__ */ new Set([node.id]);
|
|
591
|
+
let requiredByManifest = this.required.has(replayNodeKey(node));
|
|
592
|
+
let requiredForAssertion = this.fullTraceForAssertions || this.assertionNames.has(node.spanName);
|
|
593
|
+
let parentId = node.parentId;
|
|
594
|
+
while (parentId !== null) {
|
|
595
|
+
const parent = this.nodes.get(parentId);
|
|
596
|
+
if (!parent || ancestors.has(parentId) || node.mustRun && !parent.mustRun) {
|
|
597
|
+
throw new BitfabError("Invalid selective replay ancestry.");
|
|
598
|
+
}
|
|
599
|
+
ancestors.add(parentId);
|
|
600
|
+
requiredByManifest || (requiredByManifest = this.required.has(replayNodeKey(parent)));
|
|
601
|
+
requiredForAssertion || (requiredForAssertion = this.assertionNames.has(parent.spanName));
|
|
602
|
+
parentId = parent.parentId;
|
|
603
|
+
}
|
|
604
|
+
if (!ancestors.has(plan.rootId) || (requiredByManifest || unresolved.length > 0) && !node.mustRun) {
|
|
605
|
+
throw new BitfabError("Selective replay plan could hide required code.");
|
|
606
|
+
}
|
|
607
|
+
if (requiredForAssertion && !node.mustRun) {
|
|
608
|
+
throw new BitfabError(
|
|
609
|
+
"Selective replay plan could hide assertion evidence."
|
|
610
|
+
);
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
enter(params) {
|
|
615
|
+
const identity = {
|
|
616
|
+
traceFunctionKey: params.traceFunctionKey,
|
|
617
|
+
spanName: params.spanName
|
|
618
|
+
};
|
|
619
|
+
const key = replayNodeKey(identity);
|
|
620
|
+
const fingerprint = jsonFingerprint(params.inputs);
|
|
621
|
+
let node;
|
|
622
|
+
let matchReason = "unmatched-call";
|
|
623
|
+
if (params.parentSpanId === null) {
|
|
624
|
+
node = this.nodes.get(this.plan.rootId);
|
|
625
|
+
} else {
|
|
626
|
+
const recordedParent = this.runtimeParents.get(params.parentSpanId);
|
|
627
|
+
const consumed = this.consumed.get(params.parentSpanId) ?? /* @__PURE__ */ new Set();
|
|
628
|
+
this.consumed.set(params.parentSpanId, consumed);
|
|
629
|
+
const candidates = recordedParent ? (this.children.get(recordedParent) ?? []).filter(
|
|
630
|
+
(candidate) => replayNodeKey(candidate) === key && !consumed.has(candidate.id)
|
|
631
|
+
) : [];
|
|
632
|
+
const exact = fingerprint === void 0 ? [] : candidates.filter(
|
|
633
|
+
(candidate) => this.recordings.get(candidate.id)?.input === fingerprint
|
|
634
|
+
);
|
|
635
|
+
node = exact.length === 1 ? exact[0] : exact.length === 0 && candidates.length === 1 ? candidates[0] : void 0;
|
|
636
|
+
if (node) {
|
|
637
|
+
consumed.add(node.id);
|
|
638
|
+
} else if (candidates.length > 0) {
|
|
639
|
+
matchReason = "ambiguous-call";
|
|
640
|
+
for (const candidate of candidates) {
|
|
641
|
+
consumed.add(candidate.id);
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
this.runtimeParents.set(params.spanId, node?.id ?? null);
|
|
646
|
+
const requiredByManifest = this.required.has(key) || params.parentSpanId !== null && this.requiredScopes.has(params.parentSpanId);
|
|
647
|
+
if (requiredByManifest) {
|
|
648
|
+
this.requiredScopes.add(params.spanId);
|
|
649
|
+
}
|
|
650
|
+
const requiredForAssertion = this.fullTraceForAssertions || this.assertionNames.has(params.spanName) || params.parentSpanId !== null && this.assertionScopes.has(params.parentSpanId);
|
|
651
|
+
if (requiredForAssertion) {
|
|
652
|
+
this.assertionScopes.add(params.spanId);
|
|
653
|
+
}
|
|
654
|
+
const mustRun = params.parentSpanId === null || requiredByManifest || node?.mustRun === true || requiredForAssertion || this.plan.unresolvedMustRun.length > 0;
|
|
655
|
+
const recording = node === void 0 ? void 0 : this.recordings.get(node.id);
|
|
656
|
+
const equalInputs = fingerprint !== void 0 && recording?.input === fingerprint;
|
|
657
|
+
const reusable = recording?.output !== void 0 && equalInputs && params.canReuseOutput;
|
|
658
|
+
const reason = mustRun ? requiredByManifest ? "must-run" : requiredForAssertion ? "assertion" : node?.reason ?? "unresolved-must-run" : !node ? matchReason : !equalInputs ? "input-drift-or-missing-recording" : !params.canReuseOutput ? "unsupported-output-boundary" : !params.safetyMock && !params.reuseAllowed ? "reuse-not-declared" : node.reason;
|
|
659
|
+
const decision = (action, why) => {
|
|
660
|
+
this.decisions.push({
|
|
661
|
+
...identity,
|
|
662
|
+
originalSpanId: node?.id ?? null,
|
|
663
|
+
action,
|
|
664
|
+
reason: why
|
|
665
|
+
});
|
|
666
|
+
};
|
|
667
|
+
if (params.safetyMock && (mustRun || !reusable)) {
|
|
668
|
+
decision("blocked", `safety-conflict:${reason}`);
|
|
669
|
+
this.failure = new BitfabError(
|
|
670
|
+
`Selective replay cannot execute or safely reuse safety-mocked node ${params.spanName} (${reason}).`
|
|
671
|
+
);
|
|
672
|
+
throw this.failure;
|
|
673
|
+
}
|
|
674
|
+
if (!mustRun && reusable && (params.safetyMock || node?.reusable === true && params.reuseAllowed)) {
|
|
675
|
+
decision("mock", params.safetyMock ? "safety-mock" : "unchanged-inputs");
|
|
676
|
+
return {
|
|
677
|
+
mock: true,
|
|
678
|
+
output: JSON.parse(recording.output)
|
|
679
|
+
};
|
|
680
|
+
}
|
|
681
|
+
decision("run", reason);
|
|
682
|
+
this.observedNames.set(
|
|
683
|
+
params.spanName,
|
|
684
|
+
(this.observedNames.get(params.spanName) ?? 0) + 1
|
|
685
|
+
);
|
|
686
|
+
if (this.required.has(key)) {
|
|
687
|
+
this.reached.add(key);
|
|
688
|
+
}
|
|
689
|
+
return { mock: false };
|
|
690
|
+
}
|
|
691
|
+
assertNoSafetyConflict() {
|
|
692
|
+
if (this.failure) {
|
|
693
|
+
throw this.failure;
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
report() {
|
|
697
|
+
return {
|
|
698
|
+
decisions: this.decisions,
|
|
699
|
+
mustRunNotReached: this.options.mustRun.filter(
|
|
700
|
+
(node) => !this.reached.has(replayNodeKey(node))
|
|
701
|
+
),
|
|
702
|
+
unresolvedMustRun: this.plan.unresolvedMustRun,
|
|
703
|
+
assertionIds: this.plan.assertions.map((assertion) => assertion.id),
|
|
704
|
+
unresolvedAssertions: this.unresolvedAssertions,
|
|
705
|
+
assertionTargetsNotReached: this.plan.assertions.flatMap(
|
|
706
|
+
({ id, target }) => target?.kind === "span" && (this.observedNames.get(target.name) ?? 0) <= (typeof target.occurrence === "number" ? target.occurrence : 0) ? [id] : []
|
|
707
|
+
)
|
|
708
|
+
};
|
|
709
|
+
}
|
|
710
|
+
};
|
|
711
|
+
|
|
440
712
|
// src/replay.ts
|
|
441
713
|
var REPLAY_PERSISTENCE_TIMEOUT_MS = 3e4;
|
|
442
714
|
var MAX_REPLAY_ATTEMPTS = 100;
|
|
@@ -446,6 +718,100 @@ function expandReplayWork(serverItems, attempts) {
|
|
|
446
718
|
(_, attempt) => serverItems.map((serverItem) => ({ serverItem, attempt }))
|
|
447
719
|
).flat();
|
|
448
720
|
}
|
|
721
|
+
var RESUME_CONFLICTING_OPTIONS = [
|
|
722
|
+
"traceIds",
|
|
723
|
+
"limit",
|
|
724
|
+
"datasetId",
|
|
725
|
+
"datasetIds",
|
|
726
|
+
"attempts",
|
|
727
|
+
"graderIds",
|
|
728
|
+
"onlyWithAssertions",
|
|
729
|
+
"dryRun",
|
|
730
|
+
"experimentalSelectiveReplay"
|
|
731
|
+
];
|
|
732
|
+
var CARRIED_OVER_ERROR = "This item errored in an earlier run of this experiment, before it was resumed.";
|
|
733
|
+
function validateResume(resume, options) {
|
|
734
|
+
if (resume.trim().length === 0) {
|
|
735
|
+
throw new BitfabError("resume must be the ID of the experiment to resume.");
|
|
736
|
+
}
|
|
737
|
+
const conflicting = RESUME_CONFLICTING_OPTIONS.filter(
|
|
738
|
+
(name) => options[name] !== void 0 && options[name] !== false
|
|
739
|
+
);
|
|
740
|
+
if (conflicting.length > 0) {
|
|
741
|
+
throw new BitfabError(
|
|
742
|
+
`resume continues experiment ${resume} with the traces, attempts, and graders it started with, so it cannot be combined with ${conflicting.join(", ")}. Drop those options, or start a new replay without resume.`
|
|
743
|
+
);
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
async function resumeReplaySession(httpClient, experimentId, traceFunctionKey, codeChangeFiles, force, requestedDbBranch) {
|
|
747
|
+
const response = await httpClient.resumeReplay(
|
|
748
|
+
experimentId,
|
|
749
|
+
traceFunctionKey,
|
|
750
|
+
codeChangeFiles,
|
|
751
|
+
force
|
|
752
|
+
);
|
|
753
|
+
const work = response.work.map(({ itemIndex, attempt }) => {
|
|
754
|
+
const serverItem = response.items[itemIndex];
|
|
755
|
+
if (serverItem === void 0) {
|
|
756
|
+
throw new BitfabError(
|
|
757
|
+
`Replay resume for experiment ${response.experimentId} named item ${itemIndex}, but the server returned only ${response.items.length} item(s).`
|
|
758
|
+
);
|
|
759
|
+
}
|
|
760
|
+
return { serverItem, attempt };
|
|
761
|
+
});
|
|
762
|
+
if (response.skippedOriginalTraceIds.length > 0) {
|
|
763
|
+
try {
|
|
764
|
+
console.warn(
|
|
765
|
+
`Bitfab: ${response.skippedOriginalTraceIds.length} trace(s) selected when experiment ${response.experimentId} started no longer exist, so resume skips them: ${response.skippedOriginalTraceIds.join(", ")}`
|
|
766
|
+
);
|
|
767
|
+
} catch {
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
if (requestedDbBranch !== void 0 && dbBranchEnabled(requestedDbBranch) !== response.includeDbBranchLease) {
|
|
771
|
+
try {
|
|
772
|
+
console.warn(
|
|
773
|
+
`Bitfab: experiment ${response.experimentId} started with database branching ${response.includeDbBranchLease ? "on" : "off"}, so resume keeps it ${response.includeDbBranchLease ? "on" : "off"} and ignores the dbBranch option passed here.`
|
|
774
|
+
);
|
|
775
|
+
} catch {
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
return {
|
|
779
|
+
experimentId: response.experimentId,
|
|
780
|
+
experimentUrl: response.experimentUrl,
|
|
781
|
+
attempts: response.attempts,
|
|
782
|
+
work,
|
|
783
|
+
finished: response.finished,
|
|
784
|
+
includeDbBranchLease: response.includeDbBranchLease,
|
|
785
|
+
dbBranchSettings: response.dbBranchSettings ?? void 0
|
|
786
|
+
};
|
|
787
|
+
}
|
|
788
|
+
function carriedOverReplayItem(finished) {
|
|
789
|
+
return {
|
|
790
|
+
traceId: finished.traceId,
|
|
791
|
+
originalTraceId: finished.originalTraceId,
|
|
792
|
+
originalSpanId: "",
|
|
793
|
+
sourceTraceId: finished.originalTraceId,
|
|
794
|
+
sourceSpanId: "",
|
|
795
|
+
attempt: finished.attempt,
|
|
796
|
+
input: [],
|
|
797
|
+
result: void 0,
|
|
798
|
+
originalOutput: void 0,
|
|
799
|
+
error: finished.hasError ? CARRIED_OVER_ERROR : null,
|
|
800
|
+
traceError: null,
|
|
801
|
+
replayError: null,
|
|
802
|
+
durationMs: null,
|
|
803
|
+
originalDurationMs: null,
|
|
804
|
+
originalTokens: null,
|
|
805
|
+
originalModel: null,
|
|
806
|
+
tokens: null,
|
|
807
|
+
model: null,
|
|
808
|
+
dbSnapshotRef: null,
|
|
809
|
+
dbBranchTimings: null,
|
|
810
|
+
traceOutline: null,
|
|
811
|
+
originalTraceOutline: null,
|
|
812
|
+
carriedOver: true
|
|
813
|
+
};
|
|
814
|
+
}
|
|
449
815
|
function resolveAttempts(attempts) {
|
|
450
816
|
if (attempts === void 0) {
|
|
451
817
|
return 1;
|
|
@@ -622,7 +988,7 @@ function buildMockTree(rootNode) {
|
|
|
622
988
|
}
|
|
623
989
|
return { spans };
|
|
624
990
|
}
|
|
625
|
-
async function processItem(httpClient, serverItem, fn, experimentId, mockStrategy, resolvedOverrides, replayedTraceId, attempt, includeDbBranchLease, dbBranchSettings, adaptInputs, dryRun) {
|
|
991
|
+
async function processItem(httpClient, serverItem, fn, experimentId, mockStrategy, resolvedOverrides, replayedTraceId, attempt, includeDbBranchLease, dbBranchSettings, adaptInputs, dryRun, selectiveOptions) {
|
|
626
992
|
let lease = includeDbBranchLease ? serverItem.dbBranchLease : void 0;
|
|
627
993
|
let leaseError = includeDbBranchLease ? serverItem.dbBranchLeaseError : void 0;
|
|
628
994
|
let dbSnapshotRef = serverItem.dbSnapshotRef;
|
|
@@ -635,6 +1001,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
|
|
|
635
1001
|
let replayError = null;
|
|
636
1002
|
let replayDurationMs = null;
|
|
637
1003
|
let replayStarted = null;
|
|
1004
|
+
let selectiveReplay;
|
|
638
1005
|
const originalTraceId = serverItem.originalTraceId ?? serverItem.sourceTraceId;
|
|
639
1006
|
const originalSpanId = serverItem.originalSpanId ?? serverItem.sourceSpanId;
|
|
640
1007
|
try {
|
|
@@ -667,6 +1034,17 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
|
|
|
667
1034
|
originalTraceId
|
|
668
1035
|
);
|
|
669
1036
|
}
|
|
1037
|
+
if (selectiveOptions !== void 0) {
|
|
1038
|
+
if (serverItem.selectiveReplayPlan === void 0) {
|
|
1039
|
+
throw new BitfabError(
|
|
1040
|
+
"Server did not return a selective replay plan. Update the server or disable experimentalSelectiveReplay."
|
|
1041
|
+
);
|
|
1042
|
+
}
|
|
1043
|
+
selectiveReplay = new SelectiveReplayRuntime(
|
|
1044
|
+
serverItem.selectiveReplayPlan,
|
|
1045
|
+
selectiveOptions
|
|
1046
|
+
);
|
|
1047
|
+
}
|
|
670
1048
|
const span = await httpClient.getExternalSpan(originalSpanId, {
|
|
671
1049
|
view: "replay"
|
|
672
1050
|
});
|
|
@@ -686,7 +1064,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
|
|
|
686
1064
|
});
|
|
687
1065
|
}
|
|
688
1066
|
const hasOverrides = resolvedOverrides.length > 0;
|
|
689
|
-
const needTree = mockStrategy === "all" || mockStrategy === "marked" || hasOverrides;
|
|
1067
|
+
const needTree = selectiveReplay === void 0 && (mockStrategy === "all" || mockStrategy === "marked" || hasOverrides);
|
|
690
1068
|
const includeOutputs = mockStrategy === "all";
|
|
691
1069
|
let mockTree;
|
|
692
1070
|
if (needTree) {
|
|
@@ -761,6 +1139,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
|
|
|
761
1139
|
sourceBitfabTraceId: originalTraceId,
|
|
762
1140
|
replayAttempt: attempt,
|
|
763
1141
|
mockTree,
|
|
1142
|
+
selectiveReplay,
|
|
764
1143
|
callCounters: mockTree ? /* @__PURE__ */ new Map() : void 0,
|
|
765
1144
|
mockStrategy,
|
|
766
1145
|
mockOverrides: hasOverrides ? resolvedOverrides : void 0,
|
|
@@ -772,6 +1151,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
|
|
|
772
1151
|
() => fn(...inputs)
|
|
773
1152
|
);
|
|
774
1153
|
result = maybePromise instanceof Promise ? await maybePromise : maybePromise;
|
|
1154
|
+
selectiveReplay?.assertNoSafetyConflict();
|
|
775
1155
|
replayDurationMs = Math.round(performance.now() - replayStarted);
|
|
776
1156
|
} catch (e) {
|
|
777
1157
|
if (replayStarted !== null) {
|
|
@@ -823,6 +1203,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
|
|
|
823
1203
|
traceError,
|
|
824
1204
|
replayError,
|
|
825
1205
|
durationMs: replayDurationMs,
|
|
1206
|
+
...selectiveReplay === void 0 ? {} : { selectiveReplay: selectiveReplay.report() },
|
|
826
1207
|
originalDurationMs,
|
|
827
1208
|
originalTokens: serverItem.originalTokens ?? serverItem.tokens ?? null,
|
|
828
1209
|
originalModel,
|
|
@@ -920,6 +1301,21 @@ async function mapWithConcurrency(tasks, maxConcurrency, onSettled, onStarted) {
|
|
|
920
1301
|
return results;
|
|
921
1302
|
}
|
|
922
1303
|
async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, registeredOverrides = []) {
|
|
1304
|
+
if (options?.resume !== void 0) {
|
|
1305
|
+
validateResume(options.resume, options);
|
|
1306
|
+
} else if (options?.force === true) {
|
|
1307
|
+
throw new BitfabError(
|
|
1308
|
+
"force only applies when resuming an experiment. Pass it together with resume."
|
|
1309
|
+
);
|
|
1310
|
+
}
|
|
1311
|
+
if (options?.experimentalSelectiveReplay !== void 0) {
|
|
1312
|
+
validateSelectiveReplayOptions(options.experimentalSelectiveReplay);
|
|
1313
|
+
if ((options.mock ?? "marked") !== "marked" || normalizeMockOverrides(options.mockOverride).length > 0 || registeredOverrides.length > 0) {
|
|
1314
|
+
throw new BitfabError(
|
|
1315
|
+
"experimentalSelectiveReplay requires mock: 'marked' and cannot be combined with mock overrides."
|
|
1316
|
+
);
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
923
1319
|
if (options?.traceIds !== void 0) {
|
|
924
1320
|
if (options.traceIds.length === 0) {
|
|
925
1321
|
throw new BitfabError("traceIds must contain at least one trace ID.");
|
|
@@ -939,7 +1335,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
939
1335
|
} catch {
|
|
940
1336
|
}
|
|
941
1337
|
}
|
|
942
|
-
const
|
|
1338
|
+
const requestedAttempts = resolveAttempts(options?.attempts);
|
|
943
1339
|
const maxConcurrency = options?.maxConcurrency ?? 10;
|
|
944
1340
|
if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {
|
|
945
1341
|
throw new BitfabError(
|
|
@@ -958,40 +1354,60 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
958
1354
|
}
|
|
959
1355
|
}
|
|
960
1356
|
}
|
|
961
|
-
const
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
1357
|
+
const includeDbBranchLease = dbBranchEnabled(options?.dbBranch) && options?.dryRun !== true;
|
|
1358
|
+
let session;
|
|
1359
|
+
if (options?.resume === void 0) {
|
|
1360
|
+
const started2 = await httpClient.startReplay(
|
|
1361
|
+
traceFunctionKey,
|
|
1362
|
+
// limit is meaningless with explicit traceIds (the ID list determines
|
|
1363
|
+
// the count), so it's omitted from the request entirely.
|
|
1364
|
+
options?.traceIds ? void 0 : options?.limit ?? 5,
|
|
1365
|
+
options?.traceIds,
|
|
1366
|
+
options?.name,
|
|
1367
|
+
codeChangeDescription,
|
|
1368
|
+
codeChangeFiles,
|
|
1369
|
+
// A dry run executes nothing, so a branch would be provisioned (and billed)
|
|
1370
|
+
// for code that never runs, and a seeded source's refusal would fail the item
|
|
1371
|
+
// before it could report its resolved inputs.
|
|
1372
|
+
includeDbBranchLease,
|
|
1373
|
+
options?.experimentGroupId,
|
|
1374
|
+
datasetIds,
|
|
1375
|
+
options?.graderIds,
|
|
1376
|
+
resolveDbBranchSettings(options?.dbBranch),
|
|
1377
|
+
requestedAttempts,
|
|
1378
|
+
options?.adaptInputs !== void 0,
|
|
1379
|
+
options?.onlyWithAssertions,
|
|
1380
|
+
options?.judgeAssertions,
|
|
1381
|
+
options?.experimentalSelectiveReplay
|
|
1382
|
+
);
|
|
1383
|
+
if (started2.items.length === 0) {
|
|
1384
|
+
try {
|
|
1385
|
+
console.warn(
|
|
1386
|
+
`Bitfab: no traces matched "${traceFunctionKey}", so this replay ran nothing. Capture a trace, or seed one with seedTrace, before replaying.`
|
|
1387
|
+
);
|
|
1388
|
+
} catch {
|
|
1389
|
+
}
|
|
993
1390
|
}
|
|
1391
|
+
session = {
|
|
1392
|
+
experimentId: started2.experimentId,
|
|
1393
|
+
experimentUrl: started2.experimentUrl,
|
|
1394
|
+
attempts: requestedAttempts,
|
|
1395
|
+
work: expandReplayWork(started2.items, requestedAttempts),
|
|
1396
|
+
finished: [],
|
|
1397
|
+
includeDbBranchLease,
|
|
1398
|
+
dbBranchSettings: resolveDbBranchSettings(options?.dbBranch)
|
|
1399
|
+
};
|
|
1400
|
+
} else {
|
|
1401
|
+
session = await resumeReplaySession(
|
|
1402
|
+
httpClient,
|
|
1403
|
+
options.resume,
|
|
1404
|
+
traceFunctionKey,
|
|
1405
|
+
codeChangeFiles,
|
|
1406
|
+
options.force,
|
|
1407
|
+
options.dbBranch
|
|
1408
|
+
);
|
|
994
1409
|
}
|
|
1410
|
+
const { experimentId, experimentUrl, attempts } = session;
|
|
995
1411
|
const mockStrategy = options?.mock ?? "marked";
|
|
996
1412
|
const fullExperimentUrl = `${serviceUrl}${experimentUrl}`;
|
|
997
1413
|
try {
|
|
@@ -1005,7 +1421,10 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1005
1421
|
...normalizeMockOverrides(options?.mockOverride),
|
|
1006
1422
|
...registeredOverrides
|
|
1007
1423
|
];
|
|
1008
|
-
const workItems =
|
|
1424
|
+
const workItems = session.work;
|
|
1425
|
+
const carriedOverItems = session.finished.map(
|
|
1426
|
+
(finished) => carriedOverReplayItem(finished)
|
|
1427
|
+
);
|
|
1009
1428
|
const replayedTraceIds = workItems.map(() => randomUuid());
|
|
1010
1429
|
httpClient.trackTraceDeliveries(replayedTraceIds);
|
|
1011
1430
|
const tasks = workItems.map(
|
|
@@ -1018,18 +1437,19 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1018
1437
|
resolvedOverrides,
|
|
1019
1438
|
replayedTraceIds[index],
|
|
1020
1439
|
attempt,
|
|
1021
|
-
|
|
1022
|
-
|
|
1440
|
+
session.includeDbBranchLease,
|
|
1441
|
+
session.dbBranchSettings,
|
|
1023
1442
|
options?.adaptInputs,
|
|
1024
|
-
options?.dryRun === true
|
|
1443
|
+
options?.dryRun === true,
|
|
1444
|
+
options?.experimentalSelectiveReplay
|
|
1025
1445
|
)
|
|
1026
1446
|
);
|
|
1027
|
-
const total = tasks.length;
|
|
1447
|
+
const total = tasks.length + carriedOverItems.length;
|
|
1028
1448
|
const onItemFinish = options?.onItemFinish ?? options?.onProgress;
|
|
1029
|
-
let completed =
|
|
1030
|
-
let started =
|
|
1031
|
-
let
|
|
1032
|
-
let
|
|
1449
|
+
let completed = carriedOverItems.length;
|
|
1450
|
+
let started = carriedOverItems.length;
|
|
1451
|
+
let errored = carriedOverItems.filter((item) => item.error !== null).length;
|
|
1452
|
+
let succeeded = completed - errored;
|
|
1033
1453
|
let flushInFlight = null;
|
|
1034
1454
|
let flushPending = false;
|
|
1035
1455
|
const flushFinishedItemTrace = () => {
|
|
@@ -1049,7 +1469,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1049
1469
|
}
|
|
1050
1470
|
return flushInFlight;
|
|
1051
1471
|
};
|
|
1052
|
-
const
|
|
1472
|
+
const ranItems = await mapWithConcurrency(
|
|
1053
1473
|
tasks,
|
|
1054
1474
|
maxConcurrency,
|
|
1055
1475
|
async (item, index) => {
|
|
@@ -1094,6 +1514,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1094
1514
|
traceError: item.traceError,
|
|
1095
1515
|
replayError: item.replayError,
|
|
1096
1516
|
durationMs: item.durationMs,
|
|
1517
|
+
...item.selectiveReplay === void 0 ? {} : { selectiveReplay: item.selectiveReplay },
|
|
1097
1518
|
originalDurationMs: item.originalDurationMs,
|
|
1098
1519
|
originalTokens: item.originalTokens,
|
|
1099
1520
|
originalModel: item.originalModel,
|
|
@@ -1136,6 +1557,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1136
1557
|
}
|
|
1137
1558
|
} : void 0
|
|
1138
1559
|
);
|
|
1560
|
+
const resultItems = [...carriedOverItems, ...ranItems];
|
|
1139
1561
|
if (options?.dryRun === true) {
|
|
1140
1562
|
await httpClient.completeReplay(experimentId).catch(() => void 0);
|
|
1141
1563
|
const dryResult = {
|
|
@@ -1155,11 +1577,11 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1155
1577
|
experimentId,
|
|
1156
1578
|
fullExperimentUrl
|
|
1157
1579
|
);
|
|
1158
|
-
for (let index = 0; index <
|
|
1580
|
+
for (let index = 0; index < ranItems.length; index += 1) {
|
|
1159
1581
|
const localId = replayedTraceIds[index];
|
|
1160
1582
|
const readBack = localId ? deliveredTraceIds[localId] : void 0;
|
|
1161
1583
|
if (readBack !== void 0) {
|
|
1162
|
-
|
|
1584
|
+
ranItems[index].traceId = readBack;
|
|
1163
1585
|
}
|
|
1164
1586
|
}
|
|
1165
1587
|
const completeResult = await preserveReplayFailure(
|
|
@@ -1173,8 +1595,8 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
|
|
|
1173
1595
|
if (serverTraceIds !== void 0) {
|
|
1174
1596
|
const missing = [];
|
|
1175
1597
|
let completedCount = 0;
|
|
1176
|
-
for (let index = 0; index <
|
|
1177
|
-
const item =
|
|
1598
|
+
for (let index = 0; index < ranItems.length; index += 1) {
|
|
1599
|
+
const item = ranItems[index];
|
|
1178
1600
|
const localId = replayedTraceIds[index];
|
|
1179
1601
|
const mapped = localId ? serverTraceIds[localId] : void 0;
|
|
1180
1602
|
item.traceId = item.traceId ?? mapped ?? null;
|
|
@@ -1280,4 +1702,4 @@ export {
|
|
|
1280
1702
|
sleepForReplayPersistence,
|
|
1281
1703
|
replay
|
|
1282
1704
|
};
|
|
1283
|
-
//# sourceMappingURL=chunk-
|
|
1705
|
+
//# sourceMappingURL=chunk-IZUZ5NBB.js.map
|