bitfab 0.38.10 → 0.40.0

package/dist/index.cjs

@@ -51,7 +51,7 @@ var __version__, __packageName__;
 var init_version_generated = __esm({
   "src/version.generated.ts"() {
     "use strict";
-    __version__ = "0.38.10";
+    __version__ = "0.40.0";
     __packageName__ = "bitfab";
   }
 });
@@ -1207,6 +1207,16 @@ var init_transport = __esm({
 });
 
 // src/http.ts
+var http_exports = {};
+__export(http_exports, {
+  BitfabError: () => BitfabError,
+  HttpClient: () => HttpClient,
+  awaitOnExit: () => awaitOnExit,
+  awaitPendingRequests: () => awaitPendingRequests,
+  flushTraces: () => flushTraces,
+  parseRetryAfterMs: () => parseRetryAfterMs,
+  serializePayloadBody: () => serializePayloadBody
+});
 function awaitOnExit(promise) {
   pendingTracePromises.add(promise);
   void promise.finally(() => {
@@ -1731,6 +1741,10 @@ var init_http = __esm({
         const response = await this.get(endpoint);
         return response.span;
       }
+      /**
+       * GET a JSON endpoint on the service with the client's API key. Throws a
+       * `BitfabError` carrying the status text for any non-2xx response.
+       */
       async get(endpoint) {
         const url = `${this.serviceUrl}${endpoint}`;
         const controller = new AbortController();
@@ -1744,7 +1758,10 @@ var init_http = __esm({
           if (!response.ok) {
             const errorText = await response.text();
             throw new BitfabError(
-              `HTTP ${response.status}: ${errorText.slice(0, 500)}`
+              `HTTP ${response.status}: ${errorText.slice(0, 500)}`,
+              void 0,
+              response.status,
+              parseRetryAfterMs(readHeader(response, "retry-after"))
             );
           }
           return await response.json();
@@ -2551,7 +2568,7 @@ function buildMockTree(rootNode) {
   }
   return { spans };
 }
-async function processItem(httpClient, serverItem, fn, testRunId, mockStrategy, resolvedOverrides, replayedTraceId, includeDbBranchLease, dbBranchSettings, adaptInputs) {
+async function processItem(httpClient, serverItem, fn, testRunId, mockStrategy, resolvedOverrides, replayedTraceId, includeDbBranchLease, dbBranchSettings, adaptInputs, dryRun) {
   let lease = includeDbBranchLease ? serverItem.dbBranchLease : void 0;
   let leaseError = includeDbBranchLease ? serverItem.dbBranchLeaseError : void 0;
   let dbSnapshotRef = serverItem.dbSnapshotRef;
@@ -2646,6 +2663,32 @@ async function processItem(httpClient, serverItem, fn, testRunId, mockStrategy,
       }
       return pending;
     } : void 0;
+    if (dryRun) {
+      return {
+        traceId: null,
+        originalTraceId,
+        originalSpanId,
+        sourceTraceId: originalTraceId,
+        sourceSpanId: originalSpanId,
+        input: inputs,
+        result: void 0,
+        originalOutput,
+        ...serverItem.ingestionType && {
+          ingestionType: serverItem.ingestionType
+        },
+        error: null,
+        traceError: null,
+        replayError: null,
+        durationMs: null,
+        originalDurationMs: serverItem.originalDurationMs ?? serverItem.durationMs ?? null,
+        originalTokens: serverItem.originalTokens ?? serverItem.tokens ?? null,
+        originalModel: serverItem.originalModel ?? serverItem.model ?? null,
+        tokens: null,
+        model: serverItem.originalModel ?? serverItem.model ?? null,
+        dbSnapshotRef: dbSnapshotRef ?? null,
+        dbBranchTimings: dbBranchTimings ?? null
+      };
+    }
     try {
       replayStarted = performance.now();
       const maybePromise = runWithReplayContext(
@@ -2709,6 +2752,9 @@ async function processItem(httpClient, serverItem, fn, testRunId, mockStrategy,
     input: inputs,
     result,
     originalOutput,
+    ...serverItem.ingestionType && {
+      ingestionType: serverItem.ingestionType
+    },
     error,
     traceError,
     replayError,
@@ -2856,13 +2902,23 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
     options?.name,
     codeChangeDescription,
     codeChangeFiles,
-    dbBranchEnabled(options?.dbBranch),
-    // includeDbBranchLease
+    // A dry run executes nothing, so a branch would be provisioned (and billed)
+    // for code that never runs, and a seeded source's refusal would fail the item
+    // before it could report its resolved inputs.
+    dbBranchEnabled(options?.dbBranch) && options?.dryRun !== true,
     options?.experimentGroupId,
     options?.datasetId,
     options?.graderIds,
     resolveDbBranchSettings(options?.dbBranch)
   );
+  if (serverItems.length === 0) {
+    try {
+      console.warn(
+        `Bitfab: no traces matched "${traceFunctionKey}", so this replay ran nothing. Capture a trace, or seed one with seedTrace, before replaying.`
+      );
+    } catch {
+    }
+  }
   const mockStrategy = options?.mock ?? "marked";
   const maxConcurrency = options?.maxConcurrency ?? 10;
   const fullTestRunUrl = `${serviceUrl}${testRunUrl}`;
@@ -2881,9 +2937,10 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       mockStrategy,
       resolvedOverrides,
       replayedTraceIds[index],
-      dbBranchEnabled(options?.dbBranch),
+      dbBranchEnabled(options?.dbBranch) && options?.dryRun !== true,
       resolveDbBranchSettings(options?.dbBranch),
-      options?.adaptInputs
+      options?.adaptInputs,
+      options?.dryRun === true
     )
   );
   const total = tasks.length;
@@ -2991,6 +3048,16 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       }
     } : void 0
   );
+  if (options?.dryRun === true) {
+    await httpClient.completeReplay(testRunId).catch(() => void 0);
+    const dryResult = {
+      items: resultItems,
+      testRunId,
+      testRunUrl: fullTestRunUrl
+    };
+    await writeReplayResultFile(dryResult);
+    return dryResult;
+  }
   const deliveredTraceIds = await preserveReplayFailure(
     () => waitForReplayPersistence(httpClient, testRunId, replayedTraceIds),
     resultItems,
@@ -3146,6 +3213,7 @@ __export(index_exports, {
   BitfabOpenAITracingProcessor: () => BitfabOpenAITracingProcessor,
   BitfabVercelAiHandler: () => BitfabVercelAiHandler,
   DEFAULT_SERVICE_URL: () => DEFAULT_SERVICE_URL,
+  DatasetsClient: () => DatasetsClient,
   DbBranchReplayError: () => DbBranchReplayError,
   HttpClient: () => HttpClient,
   NO_MOCK_OVERRIDE: () => NO_MOCK_OVERRIDE,
@@ -3159,6 +3227,7 @@ __export(index_exports, {
   getCurrentSpan: () => getCurrentSpan,
   getCurrentTrace: () => getCurrentTrace,
   reportReplayProgress: () => reportReplayProgress,
+  seedFromRegistry: () => seedFromRegistry,
   serializeReplayResult: () => serializeReplayResult
 });
 module.exports = __toCommonJS(index_exports);
@@ -4224,6 +4293,131 @@ async function runFunctionWithBaml(bamlSource, inputs, providers, envVars) {
 // src/client.ts
 init_constants();
 
+// src/datasets.ts
+var DEFAULT_RERUN_TIMEOUT_MS = 9e4;
+var DEFAULT_RERUN_POLL_INTERVAL_MS = 1e3;
+var TERMINAL_RERUN_STATUSES = /* @__PURE__ */ new Set([
+  "completed",
+  "errored"
+]);
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function datasetPath(datasetId, suffix = "") {
+  return `/api/sdk/datasets/${encodeURIComponent(datasetId)}${suffix}`;
+}
+var DatasetsClient = class {
+  constructor(httpClient) {
+    this.httpClient = httpClient;
+  }
+  /**
+   * Create a dataset, or update the one already named this way under the same
+   * trace function. `created` reports which happened. An omitted description
+   * leaves an existing one untouched.
+   */
+  async save(params) {
+    return this.httpClient.request("/api/sdk/datasets", {
+      traceFunctionKey: params.traceFunctionKey,
+      name: params.name,
+      ...params.description === void 0 ? {} : { description: params.description }
+    });
+  }
+  /**
+   * List datasets, scoped to one trace function when `traceFunctionKey` is
+   * given and organization-wide otherwise.
+   */
+  async list(params = {}) {
+    const query = params.traceFunctionKey === void 0 ? "" : `?traceFunctionKey=${encodeURIComponent(params.traceFunctionKey)}`;
+    const response = await this.httpClient.get(
+      `/api/sdk/datasets${query}`
+    );
+    return response.datasets;
+  }
+  /** Fetch one dataset by id. Rejects with a 404 `BitfabError` when it is not in this organization. */
+  async get(datasetId) {
+    const response = await this.httpClient.get(
+      datasetPath(datasetId)
+    );
+    return response.dataset;
+  }
+  /** The ids of every trace in the dataset, the same membership a replay with `datasetId` selects. */
+  async listTraces(datasetId) {
+    return this.httpClient.get(
+      datasetPath(datasetId, "/traces")
+    );
+  }
+  /**
+   * Add traces to the dataset (1 to 100 ids per call). Traces outside the
+   * organization or under another trace function are reported in
+   * `skippedTraceIds` rather than failing the call.
+   */
+  async addTraces(datasetId, traceIds) {
+    return this.httpClient.request(
+      datasetPath(datasetId, "/traces"),
+      { traceIds }
+    );
+  }
+  /** Remove traces from the dataset. The traces themselves are never deleted. */
+  async removeTraces(datasetId, traceIds) {
+    return this.httpClient.request(
+      datasetPath(datasetId, "/removeTraces"),
+      { traceIds }
+    );
+  }
+  /**
+   * Assign graders to the dataset (1 to 100 ids per call). Graders outside the
+   * organization or under another trace function are reported in
+   * `skippedGraderIds` rather than failing the call.
+   */
+  async addGraders(datasetId, graderIds) {
+    return this.httpClient.request(
+      datasetPath(datasetId, "/graders"),
+      { graderIds }
+    );
+  }
+  /** Unassign graders from the dataset. */
+  async removeGraders(datasetId, graderIds) {
+    return this.httpClient.request(
+      datasetPath(datasetId, "/removeGraders"),
+      { graderIds }
+    );
+  }
+  /**
+   * Re-run graders over every trace in the dataset. Defaults to every assigned
+   * grader; an unassigned id is rejected. Waits for the run to finish (up to
+   * `timeoutMs`, default 90s) unless `wait` is `false`, and returns the last
+   * run state seen either way. A request matching an in-flight run joins it.
+   */
+  async rerunGraders(datasetId, options = {}) {
+    const started = await this.httpClient.request(
+      datasetPath(datasetId, "/rerunGraders"),
+      options.graderIds === void 0 ? {} : { graderIds: options.graderIds }
+    );
+    if (options.wait === false) {
+      return started;
+    }
+    const deadline = Date.now() + (options.timeoutMs ?? DEFAULT_RERUN_TIMEOUT_MS);
+    const interval = options.pollIntervalMs ?? DEFAULT_RERUN_POLL_INTERVAL_MS;
+    let run = started.run;
+    while (!TERMINAL_RERUN_STATUSES.has(run.status) && Date.now() < deadline) {
+      await sleep(interval);
+      run = await this.getGraderRerun(datasetId, run.id) ?? run;
+    }
+    return { run, joinedExisting: started.joinedExisting };
+  }
+  /**
+   * The dataset's active grader re-run, or the run named by `runId`. Returns
+   * `null` when nothing is active or the run does not belong to this dataset.
+   */
+  async getGraderRerun(datasetId, runId) {
+    const query = runId === void 0 ? "" : `?runId=${encodeURIComponent(runId)}`;
+    const response = await this.httpClient.get(
+      datasetPath(datasetId, `/rerunGraders${query}`)
+    );
+    return response.run;
+  }
+};
+
 // src/dbSnapshot.ts
 init_errors();
 var SUPPORTED_PROVIDERS = ["neon"];
@@ -5959,6 +6153,7 @@ var Bitfab = class {
       serviceUrl: this.serviceUrl,
       timeout: this.timeout
     });
+    this.datasets = new DatasetsClient(this.httpClient);
   }
   /**
    * Decorate a class method as an automatically expanded trace root.
@@ -6869,6 +7064,7 @@ var Bitfab = class {
                 inputSourceTraceId: traceState?.inputSourceTraceId,
                 dbSnapshotRef: traceState?.dbSnapshotRef,
                 dropped: traceState?.dropped,
+                ingestionType: traceState?.ingestionType,
                 // Built AFTER the wrapped fn finished, so `accessed` reflects
                 // whether customer code obtained the branch URL during this
                 // item. Omitted entirely when no lease was attached, so the
@@ -7092,6 +7288,7 @@ var Bitfab = class {
     Object.defineProperty(wrappedFn, "_bitfabTraceFunctionKey", {
       value: traceFunctionKey
     });
+    Object.defineProperty(wrappedFn, "_bitfabWrappedFn", { value: fn });
     return wrappedFn;
   }
   /**
@@ -7254,6 +7451,9 @@ var Bitfab = class {
     if (params.dbSnapshotRef) {
       rawTrace.db_snapshot_ref = params.dbSnapshotRef;
     }
+    if (params.ingestionType) {
+      rawTrace.ingestion_type = params.ingestionType;
+    }
     if (params.dbSnapshotUsage) {
       rawTrace.db_snapshot_usage = {
         neon_branch_id: params.dbSnapshotUsage.neonBranchId,
@@ -7400,6 +7600,68 @@ var Bitfab = class {
   clearMockOverrides() {
     this.mockOverrides.length = 0;
   }
+  /**
+   * Write a replayable trace from a case, without running anything.
+   *
+   * Use this to turn a corpus you already hold (a Braintrust dataset, a
+   * spreadsheet, hand-written cases) into traces that {@link replay} can
+   * select. The recorded root span carries `input` as its input and `expected`
+   * as its output, so replay reports each item against the value you expected
+   * rather than against a previous run.
+   *
+   * A seeded trace has no child spans and no database pin, so replay mocking
+   * has nothing recorded to substitute and `dbBranch` refuses it. Pass
+   * `mockOverride` at replay time for calls that must not run.
+   *
+   * @returns The trace ID, usable with `replay({ traceIds: [...] })`.
+   */
+  seedTrace(traceFunctionKey, options) {
+    const { input } = options;
+    const fn = options.fn?._bitfabWrappedFn ?? options.fn;
+    if (fn && input.length < fn.length) {
+      throw new BitfabError(
+        `Seeded case supplies ${input.length} argument(s) but ${fn.name === "" ? "the function" : fn.name} requires ${fn.length}. Fix the case, or omit fn to seed it anyway.`
+      );
+    }
+    const traceId = randomUuid();
+    const startedAt = nowIsoTimestamp();
+    activeTraceStates.set(traceId, {
+      traceId,
+      startedAt,
+      contexts: [],
+      ingestionType: "seeded",
+      ...options.sessionId !== void 0 && { sessionId: options.sessionId },
+      ...options.metadata !== void 0 && { metadata: options.metadata }
+    });
+    try {
+      this.sendWrapperSpan({
+        traceFunctionKey,
+        spanName: options.spanName ?? traceFunctionKey,
+        traceId,
+        spanId: randomUuid(),
+        parentSpanId: null,
+        inputs: input,
+        result: options.expected,
+        startedAt,
+        endedAt: startedAt,
+        spanType: options.spanType ?? "agent",
+        captureContent: true
+      });
+      this.sendTraceCompletion({
+        traceFunctionKey,
+        traceId,
+        startedAt,
+        endedAt: startedAt,
+        sessionId: options.sessionId,
+        metadata: options.metadata,
+        contexts: [],
+        ingestionType: "seeded"
+      });
+    } finally {
+      activeTraceStates.delete(traceId);
+    }
+    return traceId;
+  }
   async replay(traceFunctionKey, fn, options) {
     const wrappedKey = fn._bitfabTraceFunctionKey;
     let replayFn = fn;
@@ -7660,6 +7922,37 @@ init_replay();
 function defineReplayRegistry(registry) {
   return registry;
 }
+async function seedFromRegistry(registry, pipeline, cases) {
+  const registration = registry[pipeline];
+  if (registration === void 0) {
+    throw new BitfabError(
+      `Unknown pipeline '${pipeline}'. Registered: ${Object.keys(registry).join(", ")}`
+    );
+  }
+  const traceFunctionKey = resolveTraceFunctionKey(registration);
+  const traceIds = cases.map(
+    (seedCase) => registration.client.seedTrace(traceFunctionKey, {
+      input: seedCase.input,
+      expected: seedCase.expected,
+      fn: registration.fn,
+      metadata: seedCase.metadata,
+      sessionId: seedCase.sessionId
+    })
+  );
+  const { flushTraces: flushTraces2 } = await Promise.resolve().then(() => (init_http(), http_exports));
+  await flushTraces2(3e4);
+  return { pipeline, traceFunctionKey, traceIds };
+}
+function resolveTraceFunctionKey(registration) {
+  const wrappedKey = registration.fn._bitfabTraceFunctionKey;
+  const key = registration.traceFunctionKey ?? wrappedKey;
+  if (key === void 0) {
+    throw new BitfabError(
+      "Replay registry entry uses a plain function. Set traceFunctionKey to the key its production handler records."
+    );
+  }
+  return key;
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   BITFAB_PROGRESS_PREFIX,
@@ -7674,6 +7967,7 @@ function defineReplayRegistry(registry) {
   BitfabOpenAITracingProcessor,
   BitfabVercelAiHandler,
   DEFAULT_SERVICE_URL,
+  DatasetsClient,
   DbBranchReplayError,
   HttpClient,
   NO_MOCK_OVERRIDE,
@@ -7687,6 +7981,7 @@ function defineReplayRegistry(registry) {
   getCurrentSpan,
   getCurrentTrace,
   reportReplayProgress,
+  seedFromRegistry,
   serializeReplayResult
 });
 //# sourceMappingURL=index.cjs.map