bitfab 0.59.0 → 0.60.0

package/dist/index.d.cts

@@ -196,8 +196,8 @@ type MockOverrideInput = MockOverride | MockOverrideResolver;
 /**
  * Replay context propagation via AsyncLocalStorage.
  *
- * When set, the withSpan wrapper injects testRunId into the span payload
- * so that new spans created during replay are linked to the test run.
+ * When set, the withSpan wrapper injects experimentId into the span payload
+ * so that new spans created during replay are linked to the experiment.
  * Optionally carries a mock tree so child spans can return historical
  * outputs instead of executing.
  */
@@ -299,7 +299,7 @@ interface MockTree {
     spans: Map<string, MockSpan>;
 }
 interface ReplayContext {
-    testRunId: string;
+    experimentId: string;
     traceId?: string;
     inputSourceSpanId?: string;
     /**
@@ -600,7 +600,7 @@ declare class HttpClient {
     }): Promise<void>;
     /**
      * Start a replay session by fetching historical traces.
-     * Blocking call - creates a test run and returns lightweight item references.
+     * Blocking call - creates an experiment and returns lightweight item references.
      */
     startReplay(traceFunctionKey: string, limit: number | undefined, traceIds?: string[], name?: string, codeChangeDescription?: string | null, codeChangeFiles?: CodeChangeFile[] | null, includeDbBranchLease?: boolean, experimentGroupId?: string, datasetIds?: string[], graderIds?: string[], dbBranchSettings?: DbBranchSettings, attempts?: number, includeOriginalMetadata?: boolean, onlyWithAssertions?: boolean, judgeAssertions?: boolean): Promise<StartReplayResponse>;
     /**
@@ -631,12 +631,12 @@ declare class HttpClient {
      * has a final status AND at least that many persisted spans, which is what
      * makes this a real barrier rather than a "the row exists" check.
      */
-    getReplayStatus(testRunId: string, expectedSpanCounts: Record<string, number>): Promise<ReplayStatusResponse>;
+    getReplayStatus(experimentId: string, expectedSpanCounts: Record<string, number>): Promise<ReplayStatusResponse>;
     /**
-     * Mark a replay test run as completed.
+     * Mark a replay experiment as completed.
      * Blocking call.
      */
-    completeReplay(testRunId: string): Promise<CompleteReplayResponse>;
+    completeReplay(experimentId: string): Promise<CompleteReplayResponse>;
     getReseedSource(traceId: string): Promise<ReseedSourceResponse>;
     reseedTrace(traceId: string, runTraceId: string): Promise<ReseedTraceResponse>;
     /**
@@ -645,7 +645,7 @@ declare class HttpClient {
      * snapshot + preview branch and polls operations to readiness, which
      * can take seconds.
      */
-    resolveDbBranchLease(testRunId: string, traceId: string, dbBranchSettings?: DbBranchSettings, attempt?: number): Promise<{
+    resolveDbBranchLease(experimentId: string, traceId: string, dbBranchSettings?: DbBranchSettings, attempt?: number): Promise<{
         dbSnapshotRef: DbSnapshotRef | null;
         lease: DbBranchLease | null;
         leaseError: {
@@ -702,8 +702,8 @@ interface CodeChangeFile {
     after: string;
 }
 interface StartReplayResponse {
-    testRunId: string;
-    testRunUrl: string;
+    experimentId: string;
+    experimentUrl: string;
     attempts?: number;
     items: Array<{
         /** Bitfab trace ID of the original (historical) trace being replayed. */
@@ -791,7 +791,7 @@ interface CompleteReplayResponse {
     traceOutlines?: Record<string, TraceOutline>;
     originalTraceOutlines?: Record<string, TraceOutline>;
     /**
-     * Number of traces the server has persisted for this test run at
+     * Number of traces the server has persisted for this experiment at
      * completion time. Lets the SDK distinguish "uploads failed" from
      * "server never saw them" when the trace-ID mapping is incomplete.
      */
@@ -1379,6 +1379,7 @@ type SaveAssertion = {
     category_assertion_id?: string | null;
     id?: string;
     assertion: string;
+    humanNote?: string | null;
     passCriteria?: string | null;
     failCriteria?: string | null;
     targetOnEvaluatedTrace?: TraceTarget | null;
@@ -1523,12 +1524,12 @@ type TraceLabels = {
 declare class LabelsClient {
     private readonly httpClient;
     constructor(httpClient: HttpClient);
-    save(update: LabelUpdate, testRunId?: string): Promise<LabelOutcome>;
-    saveAll(updates: LabelUpdate[], testRunId?: string): Promise<LabelOutcome[]>;
+    save(update: LabelUpdate, experimentId?: string): Promise<LabelOutcome>;
+    saveAll(updates: LabelUpdate[], experimentId?: string): Promise<LabelOutcome[]>;
     /** Withhold a verdict for a trace or replay attempt whose check did not run. */
-    skip(target: LabelTarget, testRunId?: string): Promise<LabelOutcome>;
+    skip(target: LabelTarget, experimentId?: string): Promise<LabelOutcome>;
     /** Clear a previous verdict for a trace, replay attempt, or assertion. */
-    archive(target: LabelTarget, testRunId?: string): Promise<LabelOutcome>;
+    archive(target: LabelTarget, experimentId?: string): Promise<LabelOutcome>;
     saveHuman(update: HumanLabelUpdate): Promise<HumanLabelOutcome>;
     saveHumanAll(updates: HumanLabelUpdate[]): Promise<HumanLabelOutcome[]>;
     get(traceId: string): Promise<TraceLabels | null>;
@@ -1829,12 +1830,12 @@ declare class OrganizationMembersClient {
 }
 
 /**
- * Replay historical traces through a function and create a test run.
+ * Replay historical traces through a function and create an experiment.
  *
  * The replay flow has three phases:
- * 1. Start: fetches historical traces from the server and creates a test run
+ * 1. Start: fetches historical traces from the server and creates an experiment
  * 2. Execute: re-runs each trace's inputs through the provided function locally
- * 3. Complete: marks the test run as completed on the server
+ * 3. Complete: marks the experiment as completed on the server
  */
 
 type MockStrategy = "none" | "all" | "marked";
@@ -1886,7 +1887,7 @@ interface ReplayOptions {
      * it selects by ID with no dataset attribution.
      */
     traceIds?: string[];
-    /** Optional display name for the resulting experiment/test run. */
+    /** Optional display name for the resulting experiment. */
     name?: string;
     /** Positive integer limiting parallel items. Set to 1 for sequential. Default 10. */
     maxConcurrency?: number;
@@ -1944,14 +1945,14 @@ interface ReplayOptions {
      * Dataset this replay runs against. Mutually exclusive with `datasetIds`.
      * Alone it replays the dataset's full membership, and with `traceIds` it
      * replays only those members. When set, the resulting experiment is
-     * durably attributed to the dataset (stored on the test run), so it appears
+     * durably attributed to the dataset (stored on the experiment), so it appears
      * under the dataset's experiments even if the trace lineage can't be
      * reconstructed. Validated server-side against the org.
      */
     datasetId?: string;
     datasetIds?: string[];
     /**
-     * Graders to attach directly to this experiment (test run), independent of any
+     * Graders to attach directly to this experiment, independent of any
      * graders already on the dataset. The resulting experiment is graded by the
      * union of these and the dataset's runnable graders at completion, so use this
      * to grade a single run with a check you don't want to add to the dataset
@@ -2035,11 +2036,27 @@ interface ReplayOptions {
      * are still in flight. A throwing callback never crashes the run.
      */
     onItemStart?: (progress: ReplayItemStartProgress) => void;
+    /**
+     * Called once, right after the server creates the experiment and before any
+     * item runs, with the experiment ID and its dashboard URL. Use it to show
+     * where results will appear while the replay is still running. A throwing
+     * callback never crashes the run.
+     */
+    onExperimentStart?: (experiment: ReplayExperiment) => void;
+}
+/** The experiment a replay records into, passed to {@link ReplayOptions.onExperimentStart}. */
+interface ReplayExperiment {
+    /** Experiment ID created for this replay. */
+    experimentId: string;
+    /** Full URL to view the experiment in the dashboard. */
+    experimentUrl: string;
 }
 /** Emitted through {@link ReplayOptions.onItemStart} when a worker starts an item. */
 interface ReplayItemStartProgress {
     type: "started";
-    /** Test run ID created for this replay. */
+    /** Experiment ID created for this replay. */
+    experimentId: string;
+    /** @deprecated Use {@link experimentId}. Same value. */
     testRunId: string;
     /** Items whose processing has started so far. */
     started: number;
@@ -2066,7 +2083,9 @@ interface ReplayItemStartProgress {
 interface ReplayItemFinishProgress {
     /** Event kind, omitted for backward-compatible per-item events. */
     type?: "item";
-    /** Test run ID created for this replay. */
+    /** Experiment ID created for this replay. */
+    experimentId: string;
+    /** @deprecated Use {@link experimentId}. Same value. */
     testRunId: string;
     /** Items that have finished so far, whether they succeeded or errored. */
     completed: number;
@@ -2136,6 +2155,8 @@ interface ReplayProgressItem {
 interface ReplayProgress {
     type?: "item" | "complete";
     result?: ReplayResult<unknown>;
+    experimentId?: string;
+    /** @deprecated Use {@link experimentId}. Same value. */
     testRunId?: string;
     completed: number;
     total: number;
@@ -2202,7 +2223,7 @@ interface ReplayItem<T> {
      * trace row; the client-side id used to correlate spans during the run is
      * never surfaced here. Null until completion, on older servers that omit the
      * mapping, or if the item produced no trace. Not the verdict-persistence key:
-     * that is the original-trace lineage (`originalTraceId` + `testRunId`).
+     * that is the original-trace lineage (`originalTraceId` + `experimentId`).
      */
     traceId: string | null;
     /** Bitfab trace ID of the original (historical) trace being replayed. */
@@ -2285,9 +2306,13 @@ interface ReplayItem<T> {
 interface ReplayResult<T> {
     /** Individual replay items with inputs, results, and comparison data. */
     items: ReplayItem<T>[];
-    /** The test run ID created on the server. */
+    /** The experiment ID created on the server. */
+    experimentId: string;
+    /** Full URL to view the experiment in the dashboard. */
+    experimentUrl: string;
+    /** @deprecated Use {@link experimentId}. Same value. */
     testRunId: string;
-    /** Full URL to view the test run in the dashboard. */
+    /** @deprecated Use {@link experimentUrl}. Same value. */
     testRunUrl: string;
     attempts: number;
 }
@@ -2297,10 +2322,14 @@ interface ReplayResult<T> {
  */
 declare class ReplayError<T = unknown> extends BitfabError {
     readonly items: ReplayItem<T>[];
+    readonly experimentId: string;
+    readonly experimentUrl: string;
+    readonly cause: unknown;
+    /** @deprecated Use {@link experimentId}. Same value. */
     readonly testRunId: string;
+    /** @deprecated Use {@link experimentUrl}. Same value. */
     readonly testRunUrl: string;
-    readonly cause: unknown;
-    constructor(message: string, items: ReplayItem<T>[], testRunId: string, testRunUrl: string, cause: unknown);
+    constructor(message: string, items: ReplayItem<T>[], experimentId: string, experimentUrl: string, cause: unknown);
 }
 /**
  * A database branch requested for one replay item could not be resolved.
@@ -2668,7 +2697,8 @@ interface CurrentSpan {
  * Unlike `getCurrentTrace()`, this handle is not tied to AsyncLocalStorage -
  * each method sends to the server immediately. Useful for adding context
  * to a trace from a different process, request, or thread (e.g. a forked
- * agent that wants to annotate the original conversation's trace).
+ * agent that wants to annotate the original conversation's trace). Its methods
+ * write whether or not capture is on.
  */
 interface DetachedTrace {
     /** The canonical Bitfab trace ID this handle resolves. */
@@ -2875,7 +2905,12 @@ interface SpanOptions {
      * real child.
      */
     mockOnReplay?: boolean;
-    /** Optional test run ID included on the span and on the trace it starts. */
+    /** Optional experiment ID included on the span and on the trace it starts. */
+    experimentId?: string;
+    /**
+     * @deprecated Use {@link experimentId}. Same meaning. Passing both with
+     * different values throws.
+     */
     testRunId?: string;
     /**
      * Record a serializable view of a non-serializable result (e.g. a live
@@ -3404,14 +3439,14 @@ declare class Bitfab {
      */
     private sendWrapperSpan;
     /**
-     * Replay historical traces through a function and create a test run.
+     * Replay historical traces through a function and create an experiment.
      *
      * Fetches the last N traces for the given trace function key, re-runs each
      * through the provided function, and returns comparison data.
      *
      * Accepts either a `withSpan`-wrapped function (under the same key) or any
      * plain callable: plain callables are wrapped internally so each replayed
-     * invocation records a trace tied to the test run. The plain-callable form
+     * invocation records a trace tied to the experiment. The plain-callable form
      * is how handler-instrumented workflows (LangGraph/LangChain, Claude Agent
      * SDK) replay - those record traces under a key with no `withSpan`-wrapped
      * root in the app.
@@ -3421,7 +3456,7 @@ declare class Bitfab {
      * @param options - Optional replay options. When `traceIds` is passed,
      *   `limit` is ignored (with a warning): an explicit ID list already
      *   determines how many traces replay.
-     * @returns ReplayResult with items, testRunId, and testRunUrl
+     * @returns ReplayResult with items, experimentId, and experimentUrl
      */
     /**
      * Register a mock override applied to every subsequent `replay` on this
@@ -3681,7 +3716,7 @@ interface CommitRef {
 /**
  * SDK version from package.json (injected at build time)
  */
-declare const __version__ = "0.59.0";
+declare const __version__ = "0.60.0";
 
 /**
  * Constants for the Bitfab SDK.
@@ -3750,7 +3785,7 @@ declare const finalizers: {
 };
 
 type ReplayFunction = (...args: any[]) => unknown | Promise<unknown>;
-type ReplayRegistryOptions = Omit<ReplayOptions, "onItemStart" | "onItemFinish" | "onProgress">;
+type ReplayRegistryOptions = Omit<ReplayOptions, "onItemStart" | "onItemFinish" | "onProgress" | "onExperimentStart">;
 interface ReplayRegistryContext {
     /** Values supplied through `--params` and repeated `--param name=value`. */
     params: Readonly<Record<string, unknown>>;
@@ -3820,4 +3855,4 @@ declare function seedFromRegistry(registry: ReplayRegistry, pipeline: string, ca
     run?: boolean;
 }): Promise<SeedResult>;
 
-export { type ActiveSpanContext, type AdaptContext, type AdaptInputsFn, type AddDatasetGradersResult, type AddDatasetTracesResult, type AllowedEnvVars, type ApprovalFields, type ApprovalState, type Approver, type ArchiveAssertionsParams, AssertionCategoriesClient, type AssertionCategory, type AssertionCategorySummary, type AssertionEvidenceParameter, type AssertionGeneration, AssertionGenerationError, type AssertionGenerationStatus, AssertionGenerationTimeoutError, type AssertionLabelEvidence, type AssertionVerdict, type Assignee, BITFAB_PROGRESS_PREFIX, type BamlExecutionResult, Bitfab, BitfabClaudeAgentHandler, type BitfabConfig, BitfabError, BitfabFunction, BitfabLangGraphCallbackHandler as BitfabLangChainCallbackHandler, BitfabLangGraphCallbackHandler, BitfabLangGraphIntegration, type BitfabLanguageModelMiddleware, BitfabOpenAIAgentHandler, BitfabOpenAITracingProcessor, BitfabRivetIntegration, BitfabVercelAiHandler, type CaptureSurface, type CaptureWhen, type CapturedSpan, type CodeChangeFile, type CommitRef, type CurrentSpan, type CurrentTrace, DEFAULT_SERVICE_URL, type Dataset, type DatasetGraderRef, type DatasetTraceIds, DatasetsClient, type DbBranchOptions, DbBranchReplayError, type DbBranchTimings, type DbSnapshotConfig, type DbSnapshotProvider, type DbSnapshotRef, type DetachedTrace, type GenerateAssertionsOptions, type GetGraderLabelsParams, type GraderLabel, type GraderLabelSource, type GraderRerun, GraderRerunError, type GraderRerunProgress, type GraderRerunResult, type GraderRerunStatus, GraderRerunTimeoutError, GradersClient, HttpClient, type HumanLabelOutcome, type HumanLabelUpdate, type Justification, type JustificationSpan, type LabelAction, type LabelConfidence, type LabelOutcome, type LabelSource, type LabelStatus, type LabelTarget, type LabelUpdate, LabelsClient, type LangGraphIntegrationOptions, type ListDatasetsParams, MixedTracingError, type MockOverride, type MockOverrideCtx, type MockOverrideInput, type MockOverrideResolver, type MockStrategy, type MockValue, NO_MOCK_OVERRIDE, type NodeMatcher, type NodeMethodDecorator, type NodeOptions, type OrganizationMember, OrganizationMembersClient, type ProviderDefinition, type RemoveDatasetGradersResult, type RemoveDatasetTracesResult, ReplayBranch, ReplayError, type ReplayItem, type ReplayItemFinishProgress, type ReplayItemStartProgress, type ReplayOptions, type ReplayOptionsFactory, type ReplayProgress, type ReplayProgressItem, type ReplayRegistration, type ReplayRegistry, type ReplayRegistryContext, type ReplayRegistryOptions, type ReplayResult, type RerunGradersOptions, type RerunGradersResult, type ReseedOptions, type ReseedRegistryResult, type ReseedResult, type RivetContextLike, type RivetEffectType, type RivetIntegrationOptions, SUPPORTED_PROVIDERS, type SaveAssertion, type SaveAssertionCategoryParams, type SaveAssertionsAllParams, type SaveAssertionsParams, type SaveDatasetParams, type SaveDatasetResult, type SeedCase, type SeedCaseOptions, type SeedResult, type SeedRunOptions, type SpanLookup, type SpanMethodDecorator, type SpanMethodDecoratorContext, type SpanNodeMeta, type SpanOccurrence, type SpanOptions, type SpanType, type TokenUsage, type TraceAssertion, type TraceAssertionSource, type TraceAssertionsResult, type TraceAssertionsUpdate, type TraceIngestionType, type TraceLabels, type TraceOutline, type TraceOutlineSpan, type TraceOutlineSpanError, type TraceResponse, type TraceSearchFilters, type TraceSearchParams, type TraceSearchResult, type TraceTarget, type TraceTargetOccurrence, TracesClient, type TracingProcessor, type VercelCallParams, type VercelGenerateResult, type VercelStreamResult, type WrapBAMLOptions, type WrappedBamlFn, __version__, defineReplayRegistry, finalizers, flushTraces, getCurrentReplayBranch, getCurrentSpan, getCurrentTrace, reportReplayProgress, reseedFromRegistry, seedFromRegistry, serializeReplayResult };
+export { type ActiveSpanContext, type AdaptContext, type AdaptInputsFn, type AddDatasetGradersResult, type AddDatasetTracesResult, type AllowedEnvVars, type ApprovalFields, type ApprovalState, type Approver, type ArchiveAssertionsParams, AssertionCategoriesClient, type AssertionCategory, type AssertionCategorySummary, type AssertionEvidenceParameter, type AssertionGeneration, AssertionGenerationError, type AssertionGenerationStatus, AssertionGenerationTimeoutError, type AssertionLabelEvidence, type AssertionVerdict, type Assignee, BITFAB_PROGRESS_PREFIX, type BamlExecutionResult, Bitfab, BitfabClaudeAgentHandler, type BitfabConfig, BitfabError, BitfabFunction, BitfabLangGraphCallbackHandler as BitfabLangChainCallbackHandler, BitfabLangGraphCallbackHandler, BitfabLangGraphIntegration, type BitfabLanguageModelMiddleware, BitfabOpenAIAgentHandler, BitfabOpenAITracingProcessor, BitfabRivetIntegration, BitfabVercelAiHandler, type CaptureSurface, type CaptureWhen, type CapturedSpan, type CodeChangeFile, type CommitRef, type CurrentSpan, type CurrentTrace, DEFAULT_SERVICE_URL, type Dataset, type DatasetGraderRef, type DatasetTraceIds, DatasetsClient, type DbBranchOptions, DbBranchReplayError, type DbBranchTimings, type DbSnapshotConfig, type DbSnapshotProvider, type DbSnapshotRef, type DetachedTrace, type GenerateAssertionsOptions, type GetGraderLabelsParams, type GraderLabel, type GraderLabelSource, type GraderRerun, GraderRerunError, type GraderRerunProgress, type GraderRerunResult, type GraderRerunStatus, GraderRerunTimeoutError, GradersClient, HttpClient, type HumanLabelOutcome, type HumanLabelUpdate, type Justification, type JustificationSpan, type LabelAction, type LabelConfidence, type LabelOutcome, type LabelSource, type LabelStatus, type LabelTarget, type LabelUpdate, LabelsClient, type LangGraphIntegrationOptions, type ListDatasetsParams, MixedTracingError, type MockOverride, type MockOverrideCtx, type MockOverrideInput, type MockOverrideResolver, type MockStrategy, type MockValue, NO_MOCK_OVERRIDE, type NodeMatcher, type NodeMethodDecorator, type NodeOptions, type OrganizationMember, OrganizationMembersClient, type ProviderDefinition, type RemoveDatasetGradersResult, type RemoveDatasetTracesResult, ReplayBranch, ReplayError, type ReplayExperiment, type ReplayItem, type ReplayItemFinishProgress, type ReplayItemStartProgress, type ReplayOptions, type ReplayOptionsFactory, type ReplayProgress, type ReplayProgressItem, type ReplayRegistration, type ReplayRegistry, type ReplayRegistryContext, type ReplayRegistryOptions, type ReplayResult, type RerunGradersOptions, type RerunGradersResult, type ReseedOptions, type ReseedRegistryResult, type ReseedResult, type RivetContextLike, type RivetEffectType, type RivetIntegrationOptions, SUPPORTED_PROVIDERS, type SaveAssertion, type SaveAssertionCategoryParams, type SaveAssertionsAllParams, type SaveAssertionsParams, type SaveDatasetParams, type SaveDatasetResult, type SeedCase, type SeedCaseOptions, type SeedResult, type SeedRunOptions, type SpanLookup, type SpanMethodDecorator, type SpanMethodDecoratorContext, type SpanNodeMeta, type SpanOccurrence, type SpanOptions, type SpanType, type TokenUsage, type TraceAssertion, type TraceAssertionSource, type TraceAssertionsResult, type TraceAssertionsUpdate, type TraceIngestionType, type TraceLabels, type TraceOutline, type TraceOutlineSpan, type TraceOutlineSpanError, type TraceResponse, type TraceSearchFilters, type TraceSearchParams, type TraceSearchResult, type TraceTarget, type TraceTargetOccurrence, TracesClient, type TracingProcessor, type VercelCallParams, type VercelGenerateResult, type VercelStreamResult, type WrapBAMLOptions, type WrappedBamlFn, __version__, defineReplayRegistry, finalizers, flushTraces, getCurrentReplayBranch, getCurrentSpan, getCurrentTrace, reportReplayProgress, reseedFromRegistry, seedFromRegistry, serializeReplayResult };

package/dist/index.d.ts

@@ -196,8 +196,8 @@ type MockOverrideInput = MockOverride | MockOverrideResolver;
 /**
  * Replay context propagation via AsyncLocalStorage.
  *
- * When set, the withSpan wrapper injects testRunId into the span payload
- * so that new spans created during replay are linked to the test run.
+ * When set, the withSpan wrapper injects experimentId into the span payload
+ * so that new spans created during replay are linked to the experiment.
  * Optionally carries a mock tree so child spans can return historical
  * outputs instead of executing.
  */
@@ -299,7 +299,7 @@ interface MockTree {
     spans: Map<string, MockSpan>;
 }
 interface ReplayContext {
-    testRunId: string;
+    experimentId: string;
     traceId?: string;
     inputSourceSpanId?: string;
     /**
@@ -600,7 +600,7 @@ declare class HttpClient {
     }): Promise<void>;
     /**
      * Start a replay session by fetching historical traces.
-     * Blocking call - creates a test run and returns lightweight item references.
+     * Blocking call - creates an experiment and returns lightweight item references.
      */
     startReplay(traceFunctionKey: string, limit: number | undefined, traceIds?: string[], name?: string, codeChangeDescription?: string | null, codeChangeFiles?: CodeChangeFile[] | null, includeDbBranchLease?: boolean, experimentGroupId?: string, datasetIds?: string[], graderIds?: string[], dbBranchSettings?: DbBranchSettings, attempts?: number, includeOriginalMetadata?: boolean, onlyWithAssertions?: boolean, judgeAssertions?: boolean): Promise<StartReplayResponse>;
     /**
@@ -631,12 +631,12 @@ declare class HttpClient {
      * has a final status AND at least that many persisted spans, which is what
      * makes this a real barrier rather than a "the row exists" check.
      */
-    getReplayStatus(testRunId: string, expectedSpanCounts: Record<string, number>): Promise<ReplayStatusResponse>;
+    getReplayStatus(experimentId: string, expectedSpanCounts: Record<string, number>): Promise<ReplayStatusResponse>;
     /**
-     * Mark a replay test run as completed.
+     * Mark a replay experiment as completed.
      * Blocking call.
      */
-    completeReplay(testRunId: string): Promise<CompleteReplayResponse>;
+    completeReplay(experimentId: string): Promise<CompleteReplayResponse>;
     getReseedSource(traceId: string): Promise<ReseedSourceResponse>;
     reseedTrace(traceId: string, runTraceId: string): Promise<ReseedTraceResponse>;
     /**
@@ -645,7 +645,7 @@ declare class HttpClient {
      * snapshot + preview branch and polls operations to readiness, which
      * can take seconds.
      */
-    resolveDbBranchLease(testRunId: string, traceId: string, dbBranchSettings?: DbBranchSettings, attempt?: number): Promise<{
+    resolveDbBranchLease(experimentId: string, traceId: string, dbBranchSettings?: DbBranchSettings, attempt?: number): Promise<{
         dbSnapshotRef: DbSnapshotRef | null;
         lease: DbBranchLease | null;
         leaseError: {
@@ -702,8 +702,8 @@ interface CodeChangeFile {
     after: string;
 }
 interface StartReplayResponse {
-    testRunId: string;
-    testRunUrl: string;
+    experimentId: string;
+    experimentUrl: string;
     attempts?: number;
     items: Array<{
         /** Bitfab trace ID of the original (historical) trace being replayed. */
@@ -791,7 +791,7 @@ interface CompleteReplayResponse {
     traceOutlines?: Record<string, TraceOutline>;
     originalTraceOutlines?: Record<string, TraceOutline>;
     /**
-     * Number of traces the server has persisted for this test run at
+     * Number of traces the server has persisted for this experiment at
      * completion time. Lets the SDK distinguish "uploads failed" from
      * "server never saw them" when the trace-ID mapping is incomplete.
      */
@@ -1379,6 +1379,7 @@ type SaveAssertion = {
     category_assertion_id?: string | null;
     id?: string;
     assertion: string;
+    humanNote?: string | null;
     passCriteria?: string | null;
     failCriteria?: string | null;
     targetOnEvaluatedTrace?: TraceTarget | null;
@@ -1523,12 +1524,12 @@ type TraceLabels = {
 declare class LabelsClient {
     private readonly httpClient;
     constructor(httpClient: HttpClient);
-    save(update: LabelUpdate, testRunId?: string): Promise<LabelOutcome>;
-    saveAll(updates: LabelUpdate[], testRunId?: string): Promise<LabelOutcome[]>;
+    save(update: LabelUpdate, experimentId?: string): Promise<LabelOutcome>;
+    saveAll(updates: LabelUpdate[], experimentId?: string): Promise<LabelOutcome[]>;
     /** Withhold a verdict for a trace or replay attempt whose check did not run. */
-    skip(target: LabelTarget, testRunId?: string): Promise<LabelOutcome>;
+    skip(target: LabelTarget, experimentId?: string): Promise<LabelOutcome>;
     /** Clear a previous verdict for a trace, replay attempt, or assertion. */
-    archive(target: LabelTarget, testRunId?: string): Promise<LabelOutcome>;
+    archive(target: LabelTarget, experimentId?: string): Promise<LabelOutcome>;
     saveHuman(update: HumanLabelUpdate): Promise<HumanLabelOutcome>;
     saveHumanAll(updates: HumanLabelUpdate[]): Promise<HumanLabelOutcome[]>;
     get(traceId: string): Promise<TraceLabels | null>;
@@ -1829,12 +1830,12 @@ declare class OrganizationMembersClient {
 }
 
 /**
- * Replay historical traces through a function and create a test run.
+ * Replay historical traces through a function and create an experiment.
  *
  * The replay flow has three phases:
- * 1. Start: fetches historical traces from the server and creates a test run
+ * 1. Start: fetches historical traces from the server and creates an experiment
  * 2. Execute: re-runs each trace's inputs through the provided function locally
- * 3. Complete: marks the test run as completed on the server
+ * 3. Complete: marks the experiment as completed on the server
  */
 
 type MockStrategy = "none" | "all" | "marked";
@@ -1886,7 +1887,7 @@ interface ReplayOptions {
      * it selects by ID with no dataset attribution.
      */
     traceIds?: string[];
-    /** Optional display name for the resulting experiment/test run. */
+    /** Optional display name for the resulting experiment. */
     name?: string;
     /** Positive integer limiting parallel items. Set to 1 for sequential. Default 10. */
     maxConcurrency?: number;
@@ -1944,14 +1945,14 @@ interface ReplayOptions {
      * Dataset this replay runs against. Mutually exclusive with `datasetIds`.
      * Alone it replays the dataset's full membership, and with `traceIds` it
      * replays only those members. When set, the resulting experiment is
-     * durably attributed to the dataset (stored on the test run), so it appears
+     * durably attributed to the dataset (stored on the experiment), so it appears
      * under the dataset's experiments even if the trace lineage can't be
      * reconstructed. Validated server-side against the org.
      */
     datasetId?: string;
     datasetIds?: string[];
     /**
-     * Graders to attach directly to this experiment (test run), independent of any
+     * Graders to attach directly to this experiment, independent of any
      * graders already on the dataset. The resulting experiment is graded by the
      * union of these and the dataset's runnable graders at completion, so use this
      * to grade a single run with a check you don't want to add to the dataset
@@ -2035,11 +2036,27 @@ interface ReplayOptions {
      * are still in flight. A throwing callback never crashes the run.
      */
     onItemStart?: (progress: ReplayItemStartProgress) => void;
+    /**
+     * Called once, right after the server creates the experiment and before any
+     * item runs, with the experiment ID and its dashboard URL. Use it to show
+     * where results will appear while the replay is still running. A throwing
+     * callback never crashes the run.
+     */
+    onExperimentStart?: (experiment: ReplayExperiment) => void;
+}
+/** The experiment a replay records into, passed to {@link ReplayOptions.onExperimentStart}. */
+interface ReplayExperiment {
+    /** Experiment ID created for this replay. */
+    experimentId: string;
+    /** Full URL to view the experiment in the dashboard. */
+    experimentUrl: string;
 }
 /** Emitted through {@link ReplayOptions.onItemStart} when a worker starts an item. */
 interface ReplayItemStartProgress {
     type: "started";
-    /** Test run ID created for this replay. */
+    /** Experiment ID created for this replay. */
+    experimentId: string;
+    /** @deprecated Use {@link experimentId}. Same value. */
     testRunId: string;
     /** Items whose processing has started so far. */
     started: number;
@@ -2066,7 +2083,9 @@ interface ReplayItemStartProgress {
 interface ReplayItemFinishProgress {
     /** Event kind, omitted for backward-compatible per-item events. */
     type?: "item";
-    /** Test run ID created for this replay. */
+    /** Experiment ID created for this replay. */
+    experimentId: string;
+    /** @deprecated Use {@link experimentId}. Same value. */
     testRunId: string;
     /** Items that have finished so far, whether they succeeded or errored. */
     completed: number;
@@ -2136,6 +2155,8 @@ interface ReplayProgressItem {
 interface ReplayProgress {
     type?: "item" | "complete";
     result?: ReplayResult<unknown>;
+    experimentId?: string;
+    /** @deprecated Use {@link experimentId}. Same value. */
     testRunId?: string;
     completed: number;
     total: number;
@@ -2202,7 +2223,7 @@ interface ReplayItem<T> {
      * trace row; the client-side id used to correlate spans during the run is
      * never surfaced here. Null until completion, on older servers that omit the
      * mapping, or if the item produced no trace. Not the verdict-persistence key:
-     * that is the original-trace lineage (`originalTraceId` + `testRunId`).
+     * that is the original-trace lineage (`originalTraceId` + `experimentId`).
      */
     traceId: string | null;
     /** Bitfab trace ID of the original (historical) trace being replayed. */
@@ -2285,9 +2306,13 @@ interface ReplayItem<T> {
 interface ReplayResult<T> {
     /** Individual replay items with inputs, results, and comparison data. */
     items: ReplayItem<T>[];
-    /** The test run ID created on the server. */
+    /** The experiment ID created on the server. */
+    experimentId: string;
+    /** Full URL to view the experiment in the dashboard. */
+    experimentUrl: string;
+    /** @deprecated Use {@link experimentId}. Same value. */
     testRunId: string;
-    /** Full URL to view the test run in the dashboard. */
+    /** @deprecated Use {@link experimentUrl}. Same value. */
     testRunUrl: string;
     attempts: number;
 }
@@ -2297,10 +2322,14 @@ interface ReplayResult<T> {
  */
 declare class ReplayError<T = unknown> extends BitfabError {
     readonly items: ReplayItem<T>[];
+    readonly experimentId: string;
+    readonly experimentUrl: string;
+    readonly cause: unknown;
+    /** @deprecated Use {@link experimentId}. Same value. */
     readonly testRunId: string;
+    /** @deprecated Use {@link experimentUrl}. Same value. */
     readonly testRunUrl: string;
-    readonly cause: unknown;
-    constructor(message: string, items: ReplayItem<T>[], testRunId: string, testRunUrl: string, cause: unknown);
+    constructor(message: string, items: ReplayItem<T>[], experimentId: string, experimentUrl: string, cause: unknown);
 }
 /**
  * A database branch requested for one replay item could not be resolved.
@@ -2668,7 +2697,8 @@ interface CurrentSpan {
  * Unlike `getCurrentTrace()`, this handle is not tied to AsyncLocalStorage -
  * each method sends to the server immediately. Useful for adding context
  * to a trace from a different process, request, or thread (e.g. a forked
- * agent that wants to annotate the original conversation's trace).
+ * agent that wants to annotate the original conversation's trace). Its methods
+ * write whether or not capture is on.
  */
 interface DetachedTrace {
     /** The canonical Bitfab trace ID this handle resolves. */
@@ -2875,7 +2905,12 @@ interface SpanOptions {
      * real child.
      */
     mockOnReplay?: boolean;
-    /** Optional test run ID included on the span and on the trace it starts. */
+    /** Optional experiment ID included on the span and on the trace it starts. */
+    experimentId?: string;
+    /**
+     * @deprecated Use {@link experimentId}. Same meaning. Passing both with
+     * different values throws.
+     */
     testRunId?: string;
     /**
      * Record a serializable view of a non-serializable result (e.g. a live
@@ -3404,14 +3439,14 @@ declare class Bitfab {
      */
     private sendWrapperSpan;
     /**
-     * Replay historical traces through a function and create a test run.
+     * Replay historical traces through a function and create an experiment.
      *
      * Fetches the last N traces for the given trace function key, re-runs each
      * through the provided function, and returns comparison data.
      *
      * Accepts either a `withSpan`-wrapped function (under the same key) or any
      * plain callable: plain callables are wrapped internally so each replayed
-     * invocation records a trace tied to the test run. The plain-callable form
+     * invocation records a trace tied to the experiment. The plain-callable form
      * is how handler-instrumented workflows (LangGraph/LangChain, Claude Agent
      * SDK) replay - those record traces under a key with no `withSpan`-wrapped
      * root in the app.
@@ -3421,7 +3456,7 @@ declare class Bitfab {
      * @param options - Optional replay options. When `traceIds` is passed,
      *   `limit` is ignored (with a warning): an explicit ID list already
      *   determines how many traces replay.
-     * @returns ReplayResult with items, testRunId, and testRunUrl
+     * @returns ReplayResult with items, experimentId, and experimentUrl
      */
     /**
      * Register a mock override applied to every subsequent `replay` on this
@@ -3681,7 +3716,7 @@ interface CommitRef {
 /**
  * SDK version from package.json (injected at build time)
  */
-declare const __version__ = "0.59.0";
+declare const __version__ = "0.60.0";
 
 /**
  * Constants for the Bitfab SDK.
@@ -3750,7 +3785,7 @@ declare const finalizers: {
 };
 
 type ReplayFunction = (...args: any[]) => unknown | Promise<unknown>;
-type ReplayRegistryOptions = Omit<ReplayOptions, "onItemStart" | "onItemFinish" | "onProgress">;
+type ReplayRegistryOptions = Omit<ReplayOptions, "onItemStart" | "onItemFinish" | "onProgress" | "onExperimentStart">;
 interface ReplayRegistryContext {
     /** Values supplied through `--params` and repeated `--param name=value`. */
     params: Readonly<Record<string, unknown>>;
@@ -3820,4 +3855,4 @@ declare function seedFromRegistry(registry: ReplayRegistry, pipeline: string, ca
     run?: boolean;
 }): Promise<SeedResult>;
 
-export { type ActiveSpanContext, type AdaptContext, type AdaptInputsFn, type AddDatasetGradersResult, type AddDatasetTracesResult, type AllowedEnvVars, type ApprovalFields, type ApprovalState, type Approver, type ArchiveAssertionsParams, AssertionCategoriesClient, type AssertionCategory, type AssertionCategorySummary, type AssertionEvidenceParameter, type AssertionGeneration, AssertionGenerationError, type AssertionGenerationStatus, AssertionGenerationTimeoutError, type AssertionLabelEvidence, type AssertionVerdict, type Assignee, BITFAB_PROGRESS_PREFIX, type BamlExecutionResult, Bitfab, BitfabClaudeAgentHandler, type BitfabConfig, BitfabError, BitfabFunction, BitfabLangGraphCallbackHandler as BitfabLangChainCallbackHandler, BitfabLangGraphCallbackHandler, BitfabLangGraphIntegration, type BitfabLanguageModelMiddleware, BitfabOpenAIAgentHandler, BitfabOpenAITracingProcessor, BitfabRivetIntegration, BitfabVercelAiHandler, type CaptureSurface, type CaptureWhen, type CapturedSpan, type CodeChangeFile, type CommitRef, type CurrentSpan, type CurrentTrace, DEFAULT_SERVICE_URL, type Dataset, type DatasetGraderRef, type DatasetTraceIds, DatasetsClient, type DbBranchOptions, DbBranchReplayError, type DbBranchTimings, type DbSnapshotConfig, type DbSnapshotProvider, type DbSnapshotRef, type DetachedTrace, type GenerateAssertionsOptions, type GetGraderLabelsParams, type GraderLabel, type GraderLabelSource, type GraderRerun, GraderRerunError, type GraderRerunProgress, type GraderRerunResult, type GraderRerunStatus, GraderRerunTimeoutError, GradersClient, HttpClient, type HumanLabelOutcome, type HumanLabelUpdate, type Justification, type JustificationSpan, type LabelAction, type LabelConfidence, type LabelOutcome, type LabelSource, type LabelStatus, type LabelTarget, type LabelUpdate, LabelsClient, type LangGraphIntegrationOptions, type ListDatasetsParams, MixedTracingError, type MockOverride, type MockOverrideCtx, type MockOverrideInput, type MockOverrideResolver, type MockStrategy, type MockValue, NO_MOCK_OVERRIDE, type NodeMatcher, type NodeMethodDecorator, type NodeOptions, type OrganizationMember, OrganizationMembersClient, type ProviderDefinition, type RemoveDatasetGradersResult, type RemoveDatasetTracesResult, ReplayBranch, ReplayError, type ReplayItem, type ReplayItemFinishProgress, type ReplayItemStartProgress, type ReplayOptions, type ReplayOptionsFactory, type ReplayProgress, type ReplayProgressItem, type ReplayRegistration, type ReplayRegistry, type ReplayRegistryContext, type ReplayRegistryOptions, type ReplayResult, type RerunGradersOptions, type RerunGradersResult, type ReseedOptions, type ReseedRegistryResult, type ReseedResult, type RivetContextLike, type RivetEffectType, type RivetIntegrationOptions, SUPPORTED_PROVIDERS, type SaveAssertion, type SaveAssertionCategoryParams, type SaveAssertionsAllParams, type SaveAssertionsParams, type SaveDatasetParams, type SaveDatasetResult, type SeedCase, type SeedCaseOptions, type SeedResult, type SeedRunOptions, type SpanLookup, type SpanMethodDecorator, type SpanMethodDecoratorContext, type SpanNodeMeta, type SpanOccurrence, type SpanOptions, type SpanType, type TokenUsage, type TraceAssertion, type TraceAssertionSource, type TraceAssertionsResult, type TraceAssertionsUpdate, type TraceIngestionType, type TraceLabels, type TraceOutline, type TraceOutlineSpan, type TraceOutlineSpanError, type TraceResponse, type TraceSearchFilters, type TraceSearchParams, type TraceSearchResult, type TraceTarget, type TraceTargetOccurrence, TracesClient, type TracingProcessor, type VercelCallParams, type VercelGenerateResult, type VercelStreamResult, type WrapBAMLOptions, type WrappedBamlFn, __version__, defineReplayRegistry, finalizers, flushTraces, getCurrentReplayBranch, getCurrentSpan, getCurrentTrace, reportReplayProgress, reseedFromRegistry, seedFromRegistry, serializeReplayResult };
+export { type ActiveSpanContext, type AdaptContext, type AdaptInputsFn, type AddDatasetGradersResult, type AddDatasetTracesResult, type AllowedEnvVars, type ApprovalFields, type ApprovalState, type Approver, type ArchiveAssertionsParams, AssertionCategoriesClient, type AssertionCategory, type AssertionCategorySummary, type AssertionEvidenceParameter, type AssertionGeneration, AssertionGenerationError, type AssertionGenerationStatus, AssertionGenerationTimeoutError, type AssertionLabelEvidence, type AssertionVerdict, type Assignee, BITFAB_PROGRESS_PREFIX, type BamlExecutionResult, Bitfab, BitfabClaudeAgentHandler, type BitfabConfig, BitfabError, BitfabFunction, BitfabLangGraphCallbackHandler as BitfabLangChainCallbackHandler, BitfabLangGraphCallbackHandler, BitfabLangGraphIntegration, type BitfabLanguageModelMiddleware, BitfabOpenAIAgentHandler, BitfabOpenAITracingProcessor, BitfabRivetIntegration, BitfabVercelAiHandler, type CaptureSurface, type CaptureWhen, type CapturedSpan, type CodeChangeFile, type CommitRef, type CurrentSpan, type CurrentTrace, DEFAULT_SERVICE_URL, type Dataset, type DatasetGraderRef, type DatasetTraceIds, DatasetsClient, type DbBranchOptions, DbBranchReplayError, type DbBranchTimings, type DbSnapshotConfig, type DbSnapshotProvider, type DbSnapshotRef, type DetachedTrace, type GenerateAssertionsOptions, type GetGraderLabelsParams, type GraderLabel, type GraderLabelSource, type GraderRerun, GraderRerunError, type GraderRerunProgress, type GraderRerunResult, type GraderRerunStatus, GraderRerunTimeoutError, GradersClient, HttpClient, type HumanLabelOutcome, type HumanLabelUpdate, type Justification, type JustificationSpan, type LabelAction, type LabelConfidence, type LabelOutcome, type LabelSource, type LabelStatus, type LabelTarget, type LabelUpdate, LabelsClient, type LangGraphIntegrationOptions, type ListDatasetsParams, MixedTracingError, type MockOverride, type MockOverrideCtx, type MockOverrideInput, type MockOverrideResolver, type MockStrategy, type MockValue, NO_MOCK_OVERRIDE, type NodeMatcher, type NodeMethodDecorator, type NodeOptions, type OrganizationMember, OrganizationMembersClient, type ProviderDefinition, type RemoveDatasetGradersResult, type RemoveDatasetTracesResult, ReplayBranch, ReplayError, type ReplayExperiment, type ReplayItem, type ReplayItemFinishProgress, type ReplayItemStartProgress, type ReplayOptions, type ReplayOptionsFactory, type ReplayProgress, type ReplayProgressItem, type ReplayRegistration, type ReplayRegistry, type ReplayRegistryContext, type ReplayRegistryOptions, type ReplayResult, type RerunGradersOptions, type RerunGradersResult, type ReseedOptions, type ReseedRegistryResult, type ReseedResult, type RivetContextLike, type RivetEffectType, type RivetIntegrationOptions, SUPPORTED_PROVIDERS, type SaveAssertion, type SaveAssertionCategoryParams, type SaveAssertionsAllParams, type SaveAssertionsParams, type SaveDatasetParams, type SaveDatasetResult, type SeedCase, type SeedCaseOptions, type SeedResult, type SeedRunOptions, type SpanLookup, type SpanMethodDecorator, type SpanMethodDecoratorContext, type SpanNodeMeta, type SpanOccurrence, type SpanOptions, type SpanType, type TokenUsage, type TraceAssertion, type TraceAssertionSource, type TraceAssertionsResult, type TraceAssertionsUpdate, type TraceIngestionType, type TraceLabels, type TraceOutline, type TraceOutlineSpan, type TraceOutlineSpanError, type TraceResponse, type TraceSearchFilters, type TraceSearchParams, type TraceSearchResult, type TraceTarget, type TraceTargetOccurrence, TracesClient, type TracingProcessor, type VercelCallParams, type VercelGenerateResult, type VercelStreamResult, type WrapBAMLOptions, type WrappedBamlFn, __version__, defineReplayRegistry, finalizers, flushTraces, getCurrentReplayBranch, getCurrentSpan, getCurrentTrace, reportReplayProgress, reseedFromRegistry, seedFromRegistry, serializeReplayResult };

package/dist/index.js

@@ -35,8 +35,8 @@ import {
   getCurrentTrace,
   reseedFromRegistry,
   seedFromRegistry
-} from "./chunk-RYSCYPHQ.js";
-import "./chunk-ETZUSSF6.js";
+} from "./chunk-RXKKTRSN.js";
+import "./chunk-5DNB7EN7.js";
 import {
   BITFAB_PROGRESS_PREFIX,
   DbBranchReplayError,
@@ -44,7 +44,7 @@ import {
   ReplayError,
   reportReplayProgress,
   serializeReplayResult
-} from "./chunk-FHEEAPG2.js";
+} from "./chunk-7GIZS4E2.js";
 import {
   BitfabError,
   DEFAULT_SERVICE_URL,
@@ -52,7 +52,7 @@ import {
   MixedTracingError,
   __version__,
   flushTraces
-} from "./chunk-KFZ5OIN5.js";
+} from "./chunk-HKQGI5WP.js";
 import "./chunk-BAOIRE5T.js";
 export {
   AssertionCategoriesClient,