bitfab 0.61.2 → 0.61.3

package/dist/{chunk-A4UVO6BQ.js → chunk-IZUZ5NBB.js}

@@ -5,7 +5,7 @@ import {
   gitRunner,
   replayContextReady,
   runWithReplayContext
-} from "./chunk-INA6W7OB.js";
+} from "./chunk-6IMUONDN.js";
 import {
   runWithCaptureSuppressed,
   warnOnce
@@ -437,6 +437,278 @@ function fallbackUuidV4() {
   });
 }
 
+// src/selectiveReplay.ts
+function replayNodeKey(node) {
+  return JSON.stringify([node.traceFunctionKey, node.spanName]);
+}
+function jsonFingerprint(value) {
+  const seen = /* @__PURE__ */ new Set();
+  function encode(input) {
+    if (input === null || typeof input === "boolean") {
+      return JSON.stringify(input);
+    }
+    if (typeof input === "string" && !input.startsWith("<unserializable")) {
+      return JSON.stringify(input);
+    }
+    if (typeof input === "number" && Number.isFinite(input) && !Object.is(input, -0)) {
+      return JSON.stringify(input);
+    }
+    if (typeof input !== "object" || input === null || seen.has(input)) {
+      throw new Error("Not a lossless JSON value");
+    }
+    seen.add(input);
+    const prototype = Object.getPrototypeOf(input);
+    if (prototype !== (Array.isArray(input) ? Array.prototype : Object.prototype)) {
+      throw new Error("Not a plain JSON object");
+    }
+    if (Object.getOwnPropertySymbols(input).length > 0) {
+      throw new Error("Symbol properties are not recorded");
+    }
+    const keys = Object.keys(input);
+    if (Object.getOwnPropertyNames(input).length !== keys.length + (Array.isArray(input) ? 1 : 0)) {
+      throw new Error("Non-enumerable properties are not recorded");
+    }
+    if (Array.isArray(input) && (keys.length !== input.length || keys.some((key, index) => key !== String(index)))) {
+      throw new Error("Not a dense JSON array");
+    }
+    const properties = keys.map((key) => {
+      const descriptor = Object.getOwnPropertyDescriptor(input, key);
+      if (!descriptor || !("value" in descriptor)) {
+        throw new Error("Accessor properties are not replay inputs");
+      }
+      return `${Array.isArray(input) ? "" : `${JSON.stringify(key)}:`}${encode(descriptor.value)}`;
+    });
+    return Array.isArray(input) ? `[${properties.join(",")}]` : `{${properties.join(",")}}`;
+  }
+  try {
+    return encode(value);
+  } catch {
+    return void 0;
+  }
+}
+
+// src/selectiveReplayRuntime.ts
+function validateSelectiveReplayOptions(options) {
+  const validNodes = (nodes) => Array.isArray(nodes) && nodes.length <= 100 && nodes.every(
+    (node) => typeof node?.traceFunctionKey === "string" && node.traceFunctionKey.length > 0 && node.traceFunctionKey.length <= 500 && typeof node.spanName === "string" && node.spanName.length > 0 && node.spanName.length <= 500
+  );
+  if (!options || !validNodes(options.mustRun) || options.mustRun.length === 0 || Object.keys(options).some((key) => key !== "mustRun")) {
+    throw new BitfabError(
+      "experimentalSelectiveReplay accepts only 1\u2013100 mustRun with traceFunctionKey and spanName. Declare replayReusable on eligible spans, not reusableNodes in the replay request."
+    );
+  }
+}
+function policyKey(options) {
+  return JSON.stringify(options.mustRun.map(replayNodeKey).sort());
+}
+var SelectiveReplayRuntime = class {
+  constructor(plan, options) {
+    this.plan = plan;
+    this.options = options;
+    this.nodes = /* @__PURE__ */ new Map();
+    this.children = /* @__PURE__ */ new Map();
+    this.runtimeParents = /* @__PURE__ */ new Map();
+    this.consumed = /* @__PURE__ */ new Map();
+    this.requiredScopes = /* @__PURE__ */ new Set();
+    this.recordings = /* @__PURE__ */ new Map();
+    this.reached = /* @__PURE__ */ new Set();
+    this.decisions = [];
+    this.assertionNames = /* @__PURE__ */ new Set();
+    this.assertionScopes = /* @__PURE__ */ new Set();
+    this.observedNames = /* @__PURE__ */ new Map();
+    this.unresolvedAssertions = [];
+    validateSelectiveReplayOptions(options);
+    if (plan?.version !== 4 || !Array.isArray(plan.nodes) || plan.nodes.length === 0 || plan.nodes.length > 500 || !Array.isArray(plan.unresolvedMustRun) || !Array.isArray(plan.assertions) || !plan.options) {
+      throw new BitfabError(
+        "Server did not return a compatible selective replay plan. Update the server or disable experimentalSelectiveReplay."
+      );
+    }
+    validateSelectiveReplayOptions(plan.options);
+    if (policyKey(plan.options) !== policyKey(options)) {
+      throw new BitfabError(
+        "Selective replay plan does not match the requested mustRun nodes."
+      );
+    }
+    this.required = new Set(options.mustRun.map(replayNodeKey));
+    for (const node of plan.nodes) {
+      if (typeof node.id !== "string" || this.nodes.has(node.id) || typeof node.traceFunctionKey !== "string" || typeof node.spanName !== "string" || typeof node.mustRun !== "boolean" || typeof node.reusable !== "boolean") {
+        throw new BitfabError("Invalid selective replay node.");
+      }
+      this.nodes.set(node.id, node);
+      if (node.recording !== void 0) {
+        this.recordings.set(node.id, {
+          input: node.recording.kwargs === void 0 ? jsonFingerprint(node.recording.input) : void 0,
+          output: jsonFingerprint(node.recording.output)
+        });
+      }
+      if (node.parentId !== null) {
+        const siblings = this.children.get(node.parentId) ?? [];
+        siblings.push(node);
+        this.children.set(node.parentId, siblings);
+      }
+    }
+    const root = this.nodes.get(plan.rootId);
+    if (!root || root.parentId !== null || !root.mustRun) {
+      throw new BitfabError("Selective replay requires a live recorded root.");
+    }
+    const assertionIds = /* @__PURE__ */ new Set();
+    let wholeTrace = false;
+    for (const assertion of plan.assertions) {
+      const target = assertion?.target;
+      if (typeof assertion?.id !== "string" || assertion.id.length === 0 || assertionIds.has(assertion.id) || target !== null && (typeof target !== "object" || target.kind !== "output" && target.kind !== "span")) {
+        throw new BitfabError(
+          "Invalid selective replay assertion requirements."
+        );
+      }
+      assertionIds.add(assertion.id);
+      if (target === null) {
+        wholeTrace = true;
+      } else if (target.kind === "span") {
+        if (typeof target.name !== "string" || target.name.length === 0 || target.occurrence !== void 0 && target.occurrence !== "first" && target.occurrence !== "last" && !(Number.isInteger(target.occurrence) && typeof target.occurrence === "number" && target.occurrence >= 0)) {
+          throw new BitfabError("Invalid selective replay assertion target.");
+        }
+        this.assertionNames.add(target.name);
+        const count = plan.nodes.filter(
+          (node) => node.spanName === target.name
+        ).length;
+        if (count <= (typeof target.occurrence === "number" ? target.occurrence : 0)) {
+          this.unresolvedAssertions.push(assertion.id);
+        }
+      }
+    }
+    this.fullTraceForAssertions = wholeTrace || this.unresolvedAssertions.length > 0;
+    const recordedKeys = new Set(plan.nodes.map(replayNodeKey));
+    const unresolved = options.mustRun.filter(
+      (node) => !recordedKeys.has(replayNodeKey(node))
+    );
+    if (JSON.stringify(unresolved.map(replayNodeKey).sort()) !== JSON.stringify(plan.unresolvedMustRun.map(replayNodeKey).sort())) {
+      throw new BitfabError(
+        "Selective replay plan has incomplete mustRun coverage."
+      );
+    }
+    for (const node of plan.nodes) {
+      const ancestors = /* @__PURE__ */ new Set([node.id]);
+      let requiredByManifest = this.required.has(replayNodeKey(node));
+      let requiredForAssertion = this.fullTraceForAssertions || this.assertionNames.has(node.spanName);
+      let parentId = node.parentId;
+      while (parentId !== null) {
+        const parent = this.nodes.get(parentId);
+        if (!parent || ancestors.has(parentId) || node.mustRun && !parent.mustRun) {
+          throw new BitfabError("Invalid selective replay ancestry.");
+        }
+        ancestors.add(parentId);
+        requiredByManifest || (requiredByManifest = this.required.has(replayNodeKey(parent)));
+        requiredForAssertion || (requiredForAssertion = this.assertionNames.has(parent.spanName));
+        parentId = parent.parentId;
+      }
+      if (!ancestors.has(plan.rootId) || (requiredByManifest || unresolved.length > 0) && !node.mustRun) {
+        throw new BitfabError("Selective replay plan could hide required code.");
+      }
+      if (requiredForAssertion && !node.mustRun) {
+        throw new BitfabError(
+          "Selective replay plan could hide assertion evidence."
+        );
+      }
+    }
+  }
+  enter(params) {
+    const identity = {
+      traceFunctionKey: params.traceFunctionKey,
+      spanName: params.spanName
+    };
+    const key = replayNodeKey(identity);
+    const fingerprint = jsonFingerprint(params.inputs);
+    let node;
+    let matchReason = "unmatched-call";
+    if (params.parentSpanId === null) {
+      node = this.nodes.get(this.plan.rootId);
+    } else {
+      const recordedParent = this.runtimeParents.get(params.parentSpanId);
+      const consumed = this.consumed.get(params.parentSpanId) ?? /* @__PURE__ */ new Set();
+      this.consumed.set(params.parentSpanId, consumed);
+      const candidates = recordedParent ? (this.children.get(recordedParent) ?? []).filter(
+        (candidate) => replayNodeKey(candidate) === key && !consumed.has(candidate.id)
+      ) : [];
+      const exact = fingerprint === void 0 ? [] : candidates.filter(
+        (candidate) => this.recordings.get(candidate.id)?.input === fingerprint
+      );
+      node = exact.length === 1 ? exact[0] : exact.length === 0 && candidates.length === 1 ? candidates[0] : void 0;
+      if (node) {
+        consumed.add(node.id);
+      } else if (candidates.length > 0) {
+        matchReason = "ambiguous-call";
+        for (const candidate of candidates) {
+          consumed.add(candidate.id);
+        }
+      }
+    }
+    this.runtimeParents.set(params.spanId, node?.id ?? null);
+    const requiredByManifest = this.required.has(key) || params.parentSpanId !== null && this.requiredScopes.has(params.parentSpanId);
+    if (requiredByManifest) {
+      this.requiredScopes.add(params.spanId);
+    }
+    const requiredForAssertion = this.fullTraceForAssertions || this.assertionNames.has(params.spanName) || params.parentSpanId !== null && this.assertionScopes.has(params.parentSpanId);
+    if (requiredForAssertion) {
+      this.assertionScopes.add(params.spanId);
+    }
+    const mustRun = params.parentSpanId === null || requiredByManifest || node?.mustRun === true || requiredForAssertion || this.plan.unresolvedMustRun.length > 0;
+    const recording = node === void 0 ? void 0 : this.recordings.get(node.id);
+    const equalInputs = fingerprint !== void 0 && recording?.input === fingerprint;
+    const reusable = recording?.output !== void 0 && equalInputs && params.canReuseOutput;
+    const reason = mustRun ? requiredByManifest ? "must-run" : requiredForAssertion ? "assertion" : node?.reason ?? "unresolved-must-run" : !node ? matchReason : !equalInputs ? "input-drift-or-missing-recording" : !params.canReuseOutput ? "unsupported-output-boundary" : !params.safetyMock && !params.reuseAllowed ? "reuse-not-declared" : node.reason;
+    const decision = (action, why) => {
+      this.decisions.push({
+        ...identity,
+        originalSpanId: node?.id ?? null,
+        action,
+        reason: why
+      });
+    };
+    if (params.safetyMock && (mustRun || !reusable)) {
+      decision("blocked", `safety-conflict:${reason}`);
+      this.failure = new BitfabError(
+        `Selective replay cannot execute or safely reuse safety-mocked node ${params.spanName} (${reason}).`
+      );
+      throw this.failure;
+    }
+    if (!mustRun && reusable && (params.safetyMock || node?.reusable === true && params.reuseAllowed)) {
+      decision("mock", params.safetyMock ? "safety-mock" : "unchanged-inputs");
+      return {
+        mock: true,
+        output: JSON.parse(recording.output)
+      };
+    }
+    decision("run", reason);
+    this.observedNames.set(
+      params.spanName,
+      (this.observedNames.get(params.spanName) ?? 0) + 1
+    );
+    if (this.required.has(key)) {
+      this.reached.add(key);
+    }
+    return { mock: false };
+  }
+  assertNoSafetyConflict() {
+    if (this.failure) {
+      throw this.failure;
+    }
+  }
+  report() {
+    return {
+      decisions: this.decisions,
+      mustRunNotReached: this.options.mustRun.filter(
+        (node) => !this.reached.has(replayNodeKey(node))
+      ),
+      unresolvedMustRun: this.plan.unresolvedMustRun,
+      assertionIds: this.plan.assertions.map((assertion) => assertion.id),
+      unresolvedAssertions: this.unresolvedAssertions,
+      assertionTargetsNotReached: this.plan.assertions.flatMap(
+        ({ id, target }) => target?.kind === "span" && (this.observedNames.get(target.name) ?? 0) <= (typeof target.occurrence === "number" ? target.occurrence : 0) ? [id] : []
+      )
+    };
+  }
+};
+
 // src/replay.ts
 var REPLAY_PERSISTENCE_TIMEOUT_MS = 3e4;
 var MAX_REPLAY_ATTEMPTS = 100;
@@ -454,7 +726,8 @@ var RESUME_CONFLICTING_OPTIONS = [
   "attempts",
   "graderIds",
   "onlyWithAssertions",
-  "dryRun"
+  "dryRun",
+  "experimentalSelectiveReplay"
 ];
 var CARRIED_OVER_ERROR = "This item errored in an earlier run of this experiment, before it was resumed.";
 function validateResume(resume, options) {
@@ -715,7 +988,7 @@ function buildMockTree(rootNode) {
   }
   return { spans };
 }
-async function processItem(httpClient, serverItem, fn, experimentId, mockStrategy, resolvedOverrides, replayedTraceId, attempt, includeDbBranchLease, dbBranchSettings, adaptInputs, dryRun) {
+async function processItem(httpClient, serverItem, fn, experimentId, mockStrategy, resolvedOverrides, replayedTraceId, attempt, includeDbBranchLease, dbBranchSettings, adaptInputs, dryRun, selectiveOptions) {
   let lease = includeDbBranchLease ? serverItem.dbBranchLease : void 0;
   let leaseError = includeDbBranchLease ? serverItem.dbBranchLeaseError : void 0;
   let dbSnapshotRef = serverItem.dbSnapshotRef;
@@ -728,6 +1001,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
   let replayError = null;
   let replayDurationMs = null;
   let replayStarted = null;
+  let selectiveReplay;
   const originalTraceId = serverItem.originalTraceId ?? serverItem.sourceTraceId;
   const originalSpanId = serverItem.originalSpanId ?? serverItem.sourceSpanId;
   try {
@@ -760,6 +1034,17 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
         originalTraceId
       );
     }
+    if (selectiveOptions !== void 0) {
+      if (serverItem.selectiveReplayPlan === void 0) {
+        throw new BitfabError(
+          "Server did not return a selective replay plan. Update the server or disable experimentalSelectiveReplay."
+        );
+      }
+      selectiveReplay = new SelectiveReplayRuntime(
+        serverItem.selectiveReplayPlan,
+        selectiveOptions
+      );
+    }
     const span = await httpClient.getExternalSpan(originalSpanId, {
       view: "replay"
     });
@@ -779,7 +1064,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
       });
     }
     const hasOverrides = resolvedOverrides.length > 0;
-    const needTree = mockStrategy === "all" || mockStrategy === "marked" || hasOverrides;
+    const needTree = selectiveReplay === void 0 && (mockStrategy === "all" || mockStrategy === "marked" || hasOverrides);
     const includeOutputs = mockStrategy === "all";
     let mockTree;
     if (needTree) {
@@ -854,6 +1139,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
           sourceBitfabTraceId: originalTraceId,
           replayAttempt: attempt,
           mockTree,
+          selectiveReplay,
           callCounters: mockTree ? /* @__PURE__ */ new Map() : void 0,
           mockStrategy,
           mockOverrides: hasOverrides ? resolvedOverrides : void 0,
@@ -865,6 +1151,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
         () => fn(...inputs)
       );
       result = maybePromise instanceof Promise ? await maybePromise : maybePromise;
+      selectiveReplay?.assertNoSafetyConflict();
       replayDurationMs = Math.round(performance.now() - replayStarted);
     } catch (e) {
       if (replayStarted !== null) {
@@ -916,6 +1203,7 @@ async function processItem(httpClient, serverItem, fn, experimentId, mockStrateg
     traceError,
     replayError,
     durationMs: replayDurationMs,
+    ...selectiveReplay === void 0 ? {} : { selectiveReplay: selectiveReplay.report() },
     originalDurationMs,
     originalTokens: serverItem.originalTokens ?? serverItem.tokens ?? null,
     originalModel,
@@ -1020,6 +1308,14 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       "force only applies when resuming an experiment. Pass it together with resume."
     );
   }
+  if (options?.experimentalSelectiveReplay !== void 0) {
+    validateSelectiveReplayOptions(options.experimentalSelectiveReplay);
+    if ((options.mock ?? "marked") !== "marked" || normalizeMockOverrides(options.mockOverride).length > 0 || registeredOverrides.length > 0) {
+      throw new BitfabError(
+        "experimentalSelectiveReplay requires mock: 'marked' and cannot be combined with mock overrides."
+      );
+    }
+  }
   if (options?.traceIds !== void 0) {
     if (options.traceIds.length === 0) {
       throw new BitfabError("traceIds must contain at least one trace ID.");
@@ -1081,7 +1377,8 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       requestedAttempts,
       options?.adaptInputs !== void 0,
       options?.onlyWithAssertions,
-      options?.judgeAssertions
+      options?.judgeAssertions,
+      options?.experimentalSelectiveReplay
     );
     if (started2.items.length === 0) {
       try {
@@ -1143,7 +1440,8 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       session.includeDbBranchLease,
       session.dbBranchSettings,
       options?.adaptInputs,
-      options?.dryRun === true
+      options?.dryRun === true,
+      options?.experimentalSelectiveReplay
     )
   );
   const total = tasks.length + carriedOverItems.length;
@@ -1216,6 +1514,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
             traceError: item.traceError,
             replayError: item.replayError,
             durationMs: item.durationMs,
+            ...item.selectiveReplay === void 0 ? {} : { selectiveReplay: item.selectiveReplay },
             originalDurationMs: item.originalDurationMs,
             originalTokens: item.originalTokens,
             originalModel: item.originalModel,
@@ -1403,4 +1702,4 @@ export {
   sleepForReplayPersistence,
   replay
 };
-//# sourceMappingURL=chunk-A4UVO6BQ.js.map
+//# sourceMappingURL=chunk-IZUZ5NBB.js.map