bitfab 0.64.27 → 0.64.29

package/dist/index.d.cts

@@ -733,7 +733,7 @@ declare class HttpClient {
      * Start a replay session by fetching historical traces.
      * Blocking call - creates an experiment and returns lightweight item references.
      */
-    startReplay(traceFunctionKey: string, limit: number | undefined, traceIds?: string[], name?: string, codeChangeDescription?: string | null, codeChangeFiles?: CodeChangeFile[] | null, includeDbBranchLease?: boolean, experimentGroupId?: string, datasetIds?: string[], graderIds?: string[], dbBranchSettings?: DbBranchSettings, attempts?: number, includeOriginalMetadata?: boolean, onlyWithAssertions?: boolean, skipAssertionJudging?: boolean, experimentalSelectiveReplay?: SelectiveReplayOptions, notes?: string, metadata?: Record<string, string>, dryRun?: boolean, invocation?: ReplayInvocation): Promise<StartReplayResponse>;
+    startReplay(traceFunctionKey: string, limit: number | undefined, traceIds?: string[], name?: string, codeChangeDescription?: string | null, codeChangeFiles?: CodeChangeFile[] | null, includeDbBranchLease?: boolean, experimentGroupId?: string, datasetIds?: string[], graderIds?: string[], dbBranchSettings?: DbBranchSettings, attempts?: number, includeOriginalMetadata?: boolean, onlyWithAssertions?: boolean, skipAssertionJudging?: boolean, experimentalSelectiveReplay?: SelectiveReplayOptions, notes?: string, metadata?: Record<string, string>, dryRun?: boolean, experimentGroupName?: string, invocation?: ReplayInvocation): Promise<StartReplayResponse>;
     resumeReplay(experimentId: string, traceFunctionKey: string, codeChangeFiles?: CodeChangeFile[] | null, force?: boolean): Promise<ResumeReplayResponse>;
     /**
      * Fetch an external span by ID.
@@ -1511,6 +1511,8 @@ type ExperimentTally = {
     classified: number;
     passing: number;
     failing: number;
+    /** Failing verdicts where the judge abstained on every attempt because it could not find the evidence. */
+    abstained: number;
 };
 type ExperimentTotals = {
     total: number;
@@ -1860,9 +1862,11 @@ declare class TracesClient {
 }
 
 type LabelConfidence = "VeryLow" | "Low" | "Medium" | "High" | "VeryHigh";
-type LabelAction = "set" | "archived" | "no-active-label" | "skipped";
+type LabelAction = "set" | "archived" | "no-active-label" | "skipped" | "abstained";
 type LabelStatus = "labeled" | "skipped" | "unlabeled";
 type LabelSource = "human" | "agent";
+/** An assertion verdict. `abstained` means the judge could not find the evidence the assertion needs; it counts as a fail. */
+type Verdict = "pass" | "fail" | "abstained";
 type LabelAddress = {
     traceId: string;
     originalTraceId?: never;
@@ -1886,6 +1890,9 @@ type LabelVerdict = {
 } | {
     skip: true;
     annotation?: string;
+} | {
+    abstain: true;
+    annotation?: string;
 };
 type LabelUpdate = (LabelTarget & LabelVerdict) | (ArchiveLabelTarget & {
     archive: true;
@@ -1914,6 +1921,8 @@ type AssertionVerdict = {
     assertion: string | null;
     labelStatus: LabelStatus;
     label: boolean | null;
+    /** The verdict, or null when there is none. `label` carries the same verdict as a boolean, false for both fail and abstained. */
+    verdict: Verdict | null;
     annotation: string | null;
     evidence: Justification | null;
     confidence: LabelConfidence | null;
@@ -1929,6 +1938,8 @@ type TraceLabels = {
     approved: boolean;
     passed: number;
     failed: number;
+    /** How many of the failed assertion verdicts are abstentions, where the judge could not find the evidence. */
+    abstained: number;
     assertions: AssertionVerdict[];
 };
 declare class LabelsClient {
@@ -1939,6 +1950,15 @@ declare class LabelsClient {
     skip(target: LabelTarget & {
         annotation?: string;
     }, experimentId?: string): Promise<LabelOutcome>;
+    /**
+     * Record that the evidence does not show the part of the run this assertion
+     * checks, so there was nothing to judge. Counted as a fail in every pass
+     * rate, but stored and read back as an abstention, apart from a fail. Pass
+     * `annotation` to say what the evidence would have had to show.
+     */
+    abstain(target: LabelTarget & {
+        annotation?: string;
+    }, experimentId?: string): Promise<LabelOutcome>;
     archive(target: ArchiveLabelTarget, experimentId?: string): Promise<LabelOutcome>;
     saveHuman(update: HumanLabelUpdate): Promise<HumanLabelOutcome>;
     saveHumanAll(updates: HumanLabelUpdate[]): Promise<HumanLabelOutcome[]>;
@@ -2393,8 +2413,17 @@ interface ReplayOptions {
      * not use the historical data you asked for.
      */
     dbBranch?: DbBranchOptions | boolean;
-    /** Group ID to associate this replay with an experiment group for live streaming on the experiments page. */
+    /**
+     * Experiment group this replay joins, shown together on the experiments
+     * page. An ID with no group yet creates the group.
+     */
     experimentGroupId?: string;
+    /**
+     * Name for the experiment group in `experimentGroupId`. Applied when this
+     * replay creates the group, or when the group exists without a name. A group
+     * that already has a name keeps it. Requires `experimentGroupId`.
+     */
+    experimentGroupName?: string;
     /**
      * Dataset this replay runs against. Mutually exclusive with `datasetIds`.
      * Alone it replays the dataset's full membership, and with `traceIds` it
@@ -4222,7 +4251,7 @@ interface CommitRef {
 /**
  * SDK version from package.json (injected at build time)
  */
-declare const __version__ = "0.64.27";
+declare const __version__ = "0.64.29";
 
 /**
  * Constants for the Bitfab SDK.
@@ -4361,4 +4390,4 @@ declare function seedFromRegistry(registry: ReplayRegistry, pipeline: string, ca
     run?: boolean;
 }): Promise<SeedResult>;
 
-export { type ActiveSpanContext, type AdaptContext, type AdaptInputsFn, type AddDatasetGradersResult, type AddDatasetTracesResult, type AllowedEnvVars, type ApprovalFields, type ApprovalState, type Approver, type ArchiveAssertionsParams, type ArchiveLabelTarget, AssertionCategoriesClient, type AssertionCategory, type AssertionCategorySummary, type AssertionEvidenceParameter, type AssertionGeneration, AssertionGenerationError, type AssertionGenerationStatus, AssertionGenerationTimeoutError, type AssertionLabelEvidence, type AssertionVerdict, type Assignee, BITFAB_PROGRESS_PREFIX, type BamlExecutionResult, Bitfab, BitfabClaudeAgentHandler, type BitfabConfig, BitfabError, BitfabFunction, BitfabLangGraphCallbackHandler as BitfabLangChainCallbackHandler, BitfabLangGraphCallbackHandler, BitfabLangGraphIntegration, type BitfabLanguageModelMiddleware, BitfabOpenAIAgentHandler, BitfabOpenAITracingProcessor, BitfabRivetIntegration, BitfabVercelAiHandler, type CaptureSurface, type CaptureWhen, type CapturedSpan, type CodeChangeFile, type CommitRef, type CurrentSpan, type CurrentTrace, DEFAULT_SERVICE_URL, type Dataset, type DatasetGraderRef, type DatasetTraceIds, DatasetsClient, type DbBranchOptions, DbBranchReplayError, type DbBranchTimings, type DbSnapshotConfig, type DbSnapshotProvider, type DbSnapshotRef, type DetachedTrace, type Experiment, type ExperimentCategoryTally, type ExperimentCodeChangeTotals, type ExperimentJitter, type ExperimentPage, type ExperimentRollup, type ExperimentRollupResult, type ExperimentRunBy, type ExperimentStatus, type ExperimentTally, type ExperimentTotals, ExperimentsClient, type GenerateAssertionsOptions, type GetGraderLabelsParams, type GraderLabel, type GraderLabelOutcome, type GraderLabelSource, type GraderLabelUpdate, type GraderRerun, GraderRerunError, type GraderRerunProgress, type GraderRerunResult, type GraderRerunStatus, GraderRerunTimeoutError, GradersClient, HttpClient, type HumanLabelOutcome, type HumanLabelUpdate, type Justification, type JustificationSpan, type LabelAction, type LabelConfidence, type LabelOutcome, type LabelSource, type LabelStatus, type LabelTarget, type LabelUpdate, LabelsClient, type LangGraphIntegrationOptions, type ListDatasetsParams, type ListExperimentsParams, type ListedSpan, MixedTracingError, type MockOverride, type MockOverrideCtx, type MockOverrideInput, type MockOverrideResolver, type MockStrategy, type MockValue, NO_MOCK_OVERRIDE, type NodeMatcher, type NodeMethodDecorator, type NodeOptions, type OrganizationMember, OrganizationMembersClient, type ProviderDefinition, type RemoveDatasetGradersResult, type RemoveDatasetTracesResult, ReplayBranch, ReplayError, type ReplayExperiment, type ReplayInterrupt, type ReplayInterruptSignal, type ReplayItem, type ReplayItemFinishProgress, type ReplayItemStartProgress, type ReplayNodeIdentity, type ReplayOptions, type ReplayOptionsFactory, type ReplayProgress, type ReplayProgressItem, type ReplayRegistration, type ReplayRegistry, type ReplayRegistryContext, type ReplayRegistryOptions, type ReplayResult, type RerunGradersOptions, type RerunGradersResult, type ReseedOptions, type ReseedRegistryResult, type ReseedResult, type RivetContextLike, type RivetEffectType, type RivetIntegrationOptions, SUPPORTED_PROVIDERS, type SaveAssertion, type SaveAssertionCategoryParams, type SaveAssertionsAllParams, type SaveAssertionsParams, type SaveDatasetParams, type SaveDatasetResult, type SaveScenarioActualOutcome, type SaveScenarioData, type SaveScenarioHistoryEntry, type SaveScenarioParams, type Scenario, type ScenarioAction, type ScenarioActualOutcome, type ScenarioData, type ScenarioDataAppend, type ScenarioDataOrigin, type ScenarioHistoryEntry, type ScenarioSource, type ScenarioTrigger, type SeedCase, type SeedCaseOptions, type SeedResult, type SeedRunOptions, type SelectiveReplayDecision, type SelectiveReplayOptions, type SelectiveReplayReport, type SpanListField, type SpanListParams, type SpanListResult, type SpanLookup, type SpanMethodDecorator, type SpanMethodDecoratorContext, type SpanNodeMeta, type SpanOccurrence, type SpanOptions, type SpanType, SpansClient, type TokenUsage, type TraceAssertion, type TraceAssertionSource, type TraceAssertionsResult, type TraceAssertionsUpdate, type TraceIngestionType, type TraceLabels, type TraceResponse, type TraceSearchFilters, type TraceSearchParams, type TraceSearchResult, type TraceTarget, type TraceTargetOccurrence, TracesClient, type TracingProcessor, type UpdateDatasetParams, type VercelCallParams, type VercelGenerateResult, type VercelStreamResult, type WrapBAMLOptions, type WrappedBamlFn, __version__, defineReplayRegistry, finalizers, flushTraces, getCurrentReplayBranch, getCurrentSpan, getCurrentTrace, reportReplayProgress, reseedFromRegistry, seedFromRegistry, serializeReplayResult };
+export { type ActiveSpanContext, type AdaptContext, type AdaptInputsFn, type AddDatasetGradersResult, type AddDatasetTracesResult, type AllowedEnvVars, type ApprovalFields, type ApprovalState, type Approver, type ArchiveAssertionsParams, type ArchiveLabelTarget, AssertionCategoriesClient, type AssertionCategory, type AssertionCategorySummary, type AssertionEvidenceParameter, type AssertionGeneration, AssertionGenerationError, type AssertionGenerationStatus, AssertionGenerationTimeoutError, type AssertionLabelEvidence, type AssertionVerdict, type Assignee, BITFAB_PROGRESS_PREFIX, type BamlExecutionResult, Bitfab, BitfabClaudeAgentHandler, type BitfabConfig, BitfabError, BitfabFunction, BitfabLangGraphCallbackHandler as BitfabLangChainCallbackHandler, BitfabLangGraphCallbackHandler, BitfabLangGraphIntegration, type BitfabLanguageModelMiddleware, BitfabOpenAIAgentHandler, BitfabOpenAITracingProcessor, BitfabRivetIntegration, BitfabVercelAiHandler, type CaptureSurface, type CaptureWhen, type CapturedSpan, type CodeChangeFile, type CommitRef, type CurrentSpan, type CurrentTrace, DEFAULT_SERVICE_URL, type Dataset, type DatasetGraderRef, type DatasetTraceIds, DatasetsClient, type DbBranchOptions, DbBranchReplayError, type DbBranchTimings, type DbSnapshotConfig, type DbSnapshotProvider, type DbSnapshotRef, type DetachedTrace, type Experiment, type ExperimentCategoryTally, type ExperimentCodeChangeTotals, type ExperimentJitter, type ExperimentPage, type ExperimentRollup, type ExperimentRollupResult, type ExperimentRunBy, type ExperimentStatus, type ExperimentTally, type ExperimentTotals, ExperimentsClient, type GenerateAssertionsOptions, type GetGraderLabelsParams, type GraderLabel, type GraderLabelOutcome, type GraderLabelSource, type GraderLabelUpdate, type GraderRerun, GraderRerunError, type GraderRerunProgress, type GraderRerunResult, type GraderRerunStatus, GraderRerunTimeoutError, GradersClient, HttpClient, type HumanLabelOutcome, type HumanLabelUpdate, type Justification, type JustificationSpan, type LabelAction, type LabelConfidence, type LabelOutcome, type LabelSource, type LabelStatus, type LabelTarget, type LabelUpdate, LabelsClient, type LangGraphIntegrationOptions, type ListDatasetsParams, type ListExperimentsParams, type ListedSpan, MixedTracingError, type MockOverride, type MockOverrideCtx, type MockOverrideInput, type MockOverrideResolver, type MockStrategy, type MockValue, NO_MOCK_OVERRIDE, type NodeMatcher, type NodeMethodDecorator, type NodeOptions, type OrganizationMember, OrganizationMembersClient, type ProviderDefinition, type RemoveDatasetGradersResult, type RemoveDatasetTracesResult, ReplayBranch, ReplayError, type ReplayExperiment, type ReplayInterrupt, type ReplayInterruptSignal, type ReplayItem, type ReplayItemFinishProgress, type ReplayItemStartProgress, type ReplayNodeIdentity, type ReplayOptions, type ReplayOptionsFactory, type ReplayProgress, type ReplayProgressItem, type ReplayRegistration, type ReplayRegistry, type ReplayRegistryContext, type ReplayRegistryOptions, type ReplayResult, type RerunGradersOptions, type RerunGradersResult, type ReseedOptions, type ReseedRegistryResult, type ReseedResult, type RivetContextLike, type RivetEffectType, type RivetIntegrationOptions, SUPPORTED_PROVIDERS, type SaveAssertion, type SaveAssertionCategoryParams, type SaveAssertionsAllParams, type SaveAssertionsParams, type SaveDatasetParams, type SaveDatasetResult, type SaveScenarioActualOutcome, type SaveScenarioData, type SaveScenarioHistoryEntry, type SaveScenarioParams, type Scenario, type ScenarioAction, type ScenarioActualOutcome, type ScenarioData, type ScenarioDataAppend, type ScenarioDataOrigin, type ScenarioHistoryEntry, type ScenarioSource, type ScenarioTrigger, type SeedCase, type SeedCaseOptions, type SeedResult, type SeedRunOptions, type SelectiveReplayDecision, type SelectiveReplayOptions, type SelectiveReplayReport, type SpanListField, type SpanListParams, type SpanListResult, type SpanLookup, type SpanMethodDecorator, type SpanMethodDecoratorContext, type SpanNodeMeta, type SpanOccurrence, type SpanOptions, type SpanType, SpansClient, type TokenUsage, type TraceAssertion, type TraceAssertionSource, type TraceAssertionsResult, type TraceAssertionsUpdate, type TraceIngestionType, type TraceLabels, type TraceResponse, type TraceSearchFilters, type TraceSearchParams, type TraceSearchResult, type TraceTarget, type TraceTargetOccurrence, TracesClient, type TracingProcessor, type UpdateDatasetParams, type VercelCallParams, type VercelGenerateResult, type VercelStreamResult, type Verdict, type WrapBAMLOptions, type WrappedBamlFn, __version__, defineReplayRegistry, finalizers, flushTraces, getCurrentReplayBranch, getCurrentSpan, getCurrentTrace, reportReplayProgress, reseedFromRegistry, seedFromRegistry, serializeReplayResult };

package/dist/index.d.ts

@@ -733,7 +733,7 @@ declare class HttpClient {
      * Start a replay session by fetching historical traces.
      * Blocking call - creates an experiment and returns lightweight item references.
      */
-    startReplay(traceFunctionKey: string, limit: number | undefined, traceIds?: string[], name?: string, codeChangeDescription?: string | null, codeChangeFiles?: CodeChangeFile[] | null, includeDbBranchLease?: boolean, experimentGroupId?: string, datasetIds?: string[], graderIds?: string[], dbBranchSettings?: DbBranchSettings, attempts?: number, includeOriginalMetadata?: boolean, onlyWithAssertions?: boolean, skipAssertionJudging?: boolean, experimentalSelectiveReplay?: SelectiveReplayOptions, notes?: string, metadata?: Record<string, string>, dryRun?: boolean, invocation?: ReplayInvocation): Promise<StartReplayResponse>;
+    startReplay(traceFunctionKey: string, limit: number | undefined, traceIds?: string[], name?: string, codeChangeDescription?: string | null, codeChangeFiles?: CodeChangeFile[] | null, includeDbBranchLease?: boolean, experimentGroupId?: string, datasetIds?: string[], graderIds?: string[], dbBranchSettings?: DbBranchSettings, attempts?: number, includeOriginalMetadata?: boolean, onlyWithAssertions?: boolean, skipAssertionJudging?: boolean, experimentalSelectiveReplay?: SelectiveReplayOptions, notes?: string, metadata?: Record<string, string>, dryRun?: boolean, experimentGroupName?: string, invocation?: ReplayInvocation): Promise<StartReplayResponse>;
     resumeReplay(experimentId: string, traceFunctionKey: string, codeChangeFiles?: CodeChangeFile[] | null, force?: boolean): Promise<ResumeReplayResponse>;
     /**
      * Fetch an external span by ID.
@@ -1511,6 +1511,8 @@ type ExperimentTally = {
     classified: number;
     passing: number;
     failing: number;
+    /** Failing verdicts where the judge abstained on every attempt because it could not find the evidence. */
+    abstained: number;
 };
 type ExperimentTotals = {
     total: number;
@@ -1860,9 +1862,11 @@ declare class TracesClient {
 }
 
 type LabelConfidence = "VeryLow" | "Low" | "Medium" | "High" | "VeryHigh";
-type LabelAction = "set" | "archived" | "no-active-label" | "skipped";
+type LabelAction = "set" | "archived" | "no-active-label" | "skipped" | "abstained";
 type LabelStatus = "labeled" | "skipped" | "unlabeled";
 type LabelSource = "human" | "agent";
+/** An assertion verdict. `abstained` means the judge could not find the evidence the assertion needs; it counts as a fail. */
+type Verdict = "pass" | "fail" | "abstained";
 type LabelAddress = {
     traceId: string;
     originalTraceId?: never;
@@ -1886,6 +1890,9 @@ type LabelVerdict = {
 } | {
     skip: true;
     annotation?: string;
+} | {
+    abstain: true;
+    annotation?: string;
 };
 type LabelUpdate = (LabelTarget & LabelVerdict) | (ArchiveLabelTarget & {
     archive: true;
@@ -1914,6 +1921,8 @@ type AssertionVerdict = {
     assertion: string | null;
     labelStatus: LabelStatus;
     label: boolean | null;
+    /** The verdict, or null when there is none. `label` carries the same verdict as a boolean, false for both fail and abstained. */
+    verdict: Verdict | null;
     annotation: string | null;
     evidence: Justification | null;
     confidence: LabelConfidence | null;
@@ -1929,6 +1938,8 @@ type TraceLabels = {
     approved: boolean;
     passed: number;
     failed: number;
+    /** How many of the failed assertion verdicts are abstentions, where the judge could not find the evidence. */
+    abstained: number;
     assertions: AssertionVerdict[];
 };
 declare class LabelsClient {
@@ -1939,6 +1950,15 @@ declare class LabelsClient {
     skip(target: LabelTarget & {
         annotation?: string;
     }, experimentId?: string): Promise<LabelOutcome>;
+    /**
+     * Record that the evidence does not show the part of the run this assertion
+     * checks, so there was nothing to judge. Counted as a fail in every pass
+     * rate, but stored and read back as an abstention, apart from a fail. Pass
+     * `annotation` to say what the evidence would have had to show.
+     */
+    abstain(target: LabelTarget & {
+        annotation?: string;
+    }, experimentId?: string): Promise<LabelOutcome>;
     archive(target: ArchiveLabelTarget, experimentId?: string): Promise<LabelOutcome>;
     saveHuman(update: HumanLabelUpdate): Promise<HumanLabelOutcome>;
     saveHumanAll(updates: HumanLabelUpdate[]): Promise<HumanLabelOutcome[]>;
@@ -2393,8 +2413,17 @@ interface ReplayOptions {
      * not use the historical data you asked for.
      */
     dbBranch?: DbBranchOptions | boolean;
-    /** Group ID to associate this replay with an experiment group for live streaming on the experiments page. */
+    /**
+     * Experiment group this replay joins, shown together on the experiments
+     * page. An ID with no group yet creates the group.
+     */
     experimentGroupId?: string;
+    /**
+     * Name for the experiment group in `experimentGroupId`. Applied when this
+     * replay creates the group, or when the group exists without a name. A group
+     * that already has a name keeps it. Requires `experimentGroupId`.
+     */
+    experimentGroupName?: string;
     /**
      * Dataset this replay runs against. Mutually exclusive with `datasetIds`.
      * Alone it replays the dataset's full membership, and with `traceIds` it
@@ -4222,7 +4251,7 @@ interface CommitRef {
 /**
  * SDK version from package.json (injected at build time)
  */
-declare const __version__ = "0.64.27";
+declare const __version__ = "0.64.29";
 
 /**
  * Constants for the Bitfab SDK.
@@ -4361,4 +4390,4 @@ declare function seedFromRegistry(registry: ReplayRegistry, pipeline: string, ca
     run?: boolean;
 }): Promise<SeedResult>;
 
-export { type ActiveSpanContext, type AdaptContext, type AdaptInputsFn, type AddDatasetGradersResult, type AddDatasetTracesResult, type AllowedEnvVars, type ApprovalFields, type ApprovalState, type Approver, type ArchiveAssertionsParams, type ArchiveLabelTarget, AssertionCategoriesClient, type AssertionCategory, type AssertionCategorySummary, type AssertionEvidenceParameter, type AssertionGeneration, AssertionGenerationError, type AssertionGenerationStatus, AssertionGenerationTimeoutError, type AssertionLabelEvidence, type AssertionVerdict, type Assignee, BITFAB_PROGRESS_PREFIX, type BamlExecutionResult, Bitfab, BitfabClaudeAgentHandler, type BitfabConfig, BitfabError, BitfabFunction, BitfabLangGraphCallbackHandler as BitfabLangChainCallbackHandler, BitfabLangGraphCallbackHandler, BitfabLangGraphIntegration, type BitfabLanguageModelMiddleware, BitfabOpenAIAgentHandler, BitfabOpenAITracingProcessor, BitfabRivetIntegration, BitfabVercelAiHandler, type CaptureSurface, type CaptureWhen, type CapturedSpan, type CodeChangeFile, type CommitRef, type CurrentSpan, type CurrentTrace, DEFAULT_SERVICE_URL, type Dataset, type DatasetGraderRef, type DatasetTraceIds, DatasetsClient, type DbBranchOptions, DbBranchReplayError, type DbBranchTimings, type DbSnapshotConfig, type DbSnapshotProvider, type DbSnapshotRef, type DetachedTrace, type Experiment, type ExperimentCategoryTally, type ExperimentCodeChangeTotals, type ExperimentJitter, type ExperimentPage, type ExperimentRollup, type ExperimentRollupResult, type ExperimentRunBy, type ExperimentStatus, type ExperimentTally, type ExperimentTotals, ExperimentsClient, type GenerateAssertionsOptions, type GetGraderLabelsParams, type GraderLabel, type GraderLabelOutcome, type GraderLabelSource, type GraderLabelUpdate, type GraderRerun, GraderRerunError, type GraderRerunProgress, type GraderRerunResult, type GraderRerunStatus, GraderRerunTimeoutError, GradersClient, HttpClient, type HumanLabelOutcome, type HumanLabelUpdate, type Justification, type JustificationSpan, type LabelAction, type LabelConfidence, type LabelOutcome, type LabelSource, type LabelStatus, type LabelTarget, type LabelUpdate, LabelsClient, type LangGraphIntegrationOptions, type ListDatasetsParams, type ListExperimentsParams, type ListedSpan, MixedTracingError, type MockOverride, type MockOverrideCtx, type MockOverrideInput, type MockOverrideResolver, type MockStrategy, type MockValue, NO_MOCK_OVERRIDE, type NodeMatcher, type NodeMethodDecorator, type NodeOptions, type OrganizationMember, OrganizationMembersClient, type ProviderDefinition, type RemoveDatasetGradersResult, type RemoveDatasetTracesResult, ReplayBranch, ReplayError, type ReplayExperiment, type ReplayInterrupt, type ReplayInterruptSignal, type ReplayItem, type ReplayItemFinishProgress, type ReplayItemStartProgress, type ReplayNodeIdentity, type ReplayOptions, type ReplayOptionsFactory, type ReplayProgress, type ReplayProgressItem, type ReplayRegistration, type ReplayRegistry, type ReplayRegistryContext, type ReplayRegistryOptions, type ReplayResult, type RerunGradersOptions, type RerunGradersResult, type ReseedOptions, type ReseedRegistryResult, type ReseedResult, type RivetContextLike, type RivetEffectType, type RivetIntegrationOptions, SUPPORTED_PROVIDERS, type SaveAssertion, type SaveAssertionCategoryParams, type SaveAssertionsAllParams, type SaveAssertionsParams, type SaveDatasetParams, type SaveDatasetResult, type SaveScenarioActualOutcome, type SaveScenarioData, type SaveScenarioHistoryEntry, type SaveScenarioParams, type Scenario, type ScenarioAction, type ScenarioActualOutcome, type ScenarioData, type ScenarioDataAppend, type ScenarioDataOrigin, type ScenarioHistoryEntry, type ScenarioSource, type ScenarioTrigger, type SeedCase, type SeedCaseOptions, type SeedResult, type SeedRunOptions, type SelectiveReplayDecision, type SelectiveReplayOptions, type SelectiveReplayReport, type SpanListField, type SpanListParams, type SpanListResult, type SpanLookup, type SpanMethodDecorator, type SpanMethodDecoratorContext, type SpanNodeMeta, type SpanOccurrence, type SpanOptions, type SpanType, SpansClient, type TokenUsage, type TraceAssertion, type TraceAssertionSource, type TraceAssertionsResult, type TraceAssertionsUpdate, type TraceIngestionType, type TraceLabels, type TraceResponse, type TraceSearchFilters, type TraceSearchParams, type TraceSearchResult, type TraceTarget, type TraceTargetOccurrence, TracesClient, type TracingProcessor, type UpdateDatasetParams, type VercelCallParams, type VercelGenerateResult, type VercelStreamResult, type WrapBAMLOptions, type WrappedBamlFn, __version__, defineReplayRegistry, finalizers, flushTraces, getCurrentReplayBranch, getCurrentSpan, getCurrentTrace, reportReplayProgress, reseedFromRegistry, seedFromRegistry, serializeReplayResult };
+export { type ActiveSpanContext, type AdaptContext, type AdaptInputsFn, type AddDatasetGradersResult, type AddDatasetTracesResult, type AllowedEnvVars, type ApprovalFields, type ApprovalState, type Approver, type ArchiveAssertionsParams, type ArchiveLabelTarget, AssertionCategoriesClient, type AssertionCategory, type AssertionCategorySummary, type AssertionEvidenceParameter, type AssertionGeneration, AssertionGenerationError, type AssertionGenerationStatus, AssertionGenerationTimeoutError, type AssertionLabelEvidence, type AssertionVerdict, type Assignee, BITFAB_PROGRESS_PREFIX, type BamlExecutionResult, Bitfab, BitfabClaudeAgentHandler, type BitfabConfig, BitfabError, BitfabFunction, BitfabLangGraphCallbackHandler as BitfabLangChainCallbackHandler, BitfabLangGraphCallbackHandler, BitfabLangGraphIntegration, type BitfabLanguageModelMiddleware, BitfabOpenAIAgentHandler, BitfabOpenAITracingProcessor, BitfabRivetIntegration, BitfabVercelAiHandler, type CaptureSurface, type CaptureWhen, type CapturedSpan, type CodeChangeFile, type CommitRef, type CurrentSpan, type CurrentTrace, DEFAULT_SERVICE_URL, type Dataset, type DatasetGraderRef, type DatasetTraceIds, DatasetsClient, type DbBranchOptions, DbBranchReplayError, type DbBranchTimings, type DbSnapshotConfig, type DbSnapshotProvider, type DbSnapshotRef, type DetachedTrace, type Experiment, type ExperimentCategoryTally, type ExperimentCodeChangeTotals, type ExperimentJitter, type ExperimentPage, type ExperimentRollup, type ExperimentRollupResult, type ExperimentRunBy, type ExperimentStatus, type ExperimentTally, type ExperimentTotals, ExperimentsClient, type GenerateAssertionsOptions, type GetGraderLabelsParams, type GraderLabel, type GraderLabelOutcome, type GraderLabelSource, type GraderLabelUpdate, type GraderRerun, GraderRerunError, type GraderRerunProgress, type GraderRerunResult, type GraderRerunStatus, GraderRerunTimeoutError, GradersClient, HttpClient, type HumanLabelOutcome, type HumanLabelUpdate, type Justification, type JustificationSpan, type LabelAction, type LabelConfidence, type LabelOutcome, type LabelSource, type LabelStatus, type LabelTarget, type LabelUpdate, LabelsClient, type LangGraphIntegrationOptions, type ListDatasetsParams, type ListExperimentsParams, type ListedSpan, MixedTracingError, type MockOverride, type MockOverrideCtx, type MockOverrideInput, type MockOverrideResolver, type MockStrategy, type MockValue, NO_MOCK_OVERRIDE, type NodeMatcher, type NodeMethodDecorator, type NodeOptions, type OrganizationMember, OrganizationMembersClient, type ProviderDefinition, type RemoveDatasetGradersResult, type RemoveDatasetTracesResult, ReplayBranch, ReplayError, type ReplayExperiment, type ReplayInterrupt, type ReplayInterruptSignal, type ReplayItem, type ReplayItemFinishProgress, type ReplayItemStartProgress, type ReplayNodeIdentity, type ReplayOptions, type ReplayOptionsFactory, type ReplayProgress, type ReplayProgressItem, type ReplayRegistration, type ReplayRegistry, type ReplayRegistryContext, type ReplayRegistryOptions, type ReplayResult, type RerunGradersOptions, type RerunGradersResult, type ReseedOptions, type ReseedRegistryResult, type ReseedResult, type RivetContextLike, type RivetEffectType, type RivetIntegrationOptions, SUPPORTED_PROVIDERS, type SaveAssertion, type SaveAssertionCategoryParams, type SaveAssertionsAllParams, type SaveAssertionsParams, type SaveDatasetParams, type SaveDatasetResult, type SaveScenarioActualOutcome, type SaveScenarioData, type SaveScenarioHistoryEntry, type SaveScenarioParams, type Scenario, type ScenarioAction, type ScenarioActualOutcome, type ScenarioData, type ScenarioDataAppend, type ScenarioDataOrigin, type ScenarioHistoryEntry, type ScenarioSource, type ScenarioTrigger, type SeedCase, type SeedCaseOptions, type SeedResult, type SeedRunOptions, type SelectiveReplayDecision, type SelectiveReplayOptions, type SelectiveReplayReport, type SpanListField, type SpanListParams, type SpanListResult, type SpanLookup, type SpanMethodDecorator, type SpanMethodDecoratorContext, type SpanNodeMeta, type SpanOccurrence, type SpanOptions, type SpanType, SpansClient, type TokenUsage, type TraceAssertion, type TraceAssertionSource, type TraceAssertionsResult, type TraceAssertionsUpdate, type TraceIngestionType, type TraceLabels, type TraceResponse, type TraceSearchFilters, type TraceSearchParams, type TraceSearchResult, type TraceTarget, type TraceTargetOccurrence, TracesClient, type TracingProcessor, type UpdateDatasetParams, type VercelCallParams, type VercelGenerateResult, type VercelStreamResult, type Verdict, type WrapBAMLOptions, type WrappedBamlFn, __version__, defineReplayRegistry, finalizers, flushTraces, getCurrentReplayBranch, getCurrentSpan, getCurrentTrace, reportReplayProgress, reseedFromRegistry, seedFromRegistry, serializeReplayResult };

package/dist/index.js

@@ -37,7 +37,7 @@ import {
   getCurrentTrace,
   reseedFromRegistry,
   seedFromRegistry
-} from "./chunk-ZHLRO2UG.js";
+} from "./chunk-W2AQ56UP.js";
 import "./chunk-YWXQM6J3.js";
 import {
   BITFAB_PROGRESS_PREFIX,
@@ -46,7 +46,7 @@ import {
   ReplayError,
   reportReplayProgress,
   serializeReplayResult
-} from "./chunk-CJRTXQ2W.js";
+} from "./chunk-E2N56K23.js";
 import {
   BitfabError,
   DEFAULT_SERVICE_URL,
@@ -54,7 +54,7 @@ import {
   MixedTracingError,
   __version__,
   flushTraces
-} from "./chunk-A3BOXSQY.js";
+} from "./chunk-24JBL2FV.js";
 import "./chunk-MT36GJMW.js";
 export {
   AssertionCategoriesClient,

package/dist/node.cjs

@@ -97,7 +97,7 @@ var __version__, __packageName__;
 var init_version_generated = __esm({
   "src/version.generated.ts"() {
     "use strict";
-    __version__ = "0.64.27";
+    __version__ = "0.64.29";
     __packageName__ = "bitfab";
   }
 });
@@ -2935,7 +2935,7 @@ var init_http = __esm({
        * Start a replay session by fetching historical traces.
        * Blocking call - creates an experiment and returns lightweight item references.
        */
-      async startReplay(traceFunctionKey, limit, traceIds, name, codeChangeDescription, codeChangeFiles, includeDbBranchLease, experimentGroupId, datasetIds, graderIds, dbBranchSettings, attempts, includeOriginalMetadata, onlyWithAssertions, skipAssertionJudging, experimentalSelectiveReplay, notes, metadata, dryRun, invocation) {
+      async startReplay(traceFunctionKey, limit, traceIds, name, codeChangeDescription, codeChangeFiles, includeDbBranchLease, experimentGroupId, datasetIds, graderIds, dbBranchSettings, attempts, includeOriginalMetadata, onlyWithAssertions, skipAssertionJudging, experimentalSelectiveReplay, notes, metadata, dryRun, experimentGroupName, invocation) {
         const payload = { traceFunctionKey };
         if (experimentalSelectiveReplay !== void 0) {
           payload.experimentalSelectiveReplay = experimentalSelectiveReplay;
@@ -2968,6 +2968,9 @@ var init_http = __esm({
         if (experimentGroupId !== void 0) {
           payload.experimentGroupId = experimentGroupId;
         }
+        if (experimentGroupName !== void 0) {
+          payload.experimentGroupName = experimentGroupName;
+        }
         if (datasetIds !== void 0) {
           if (datasetIds.length === 1) {
             payload.datasetId = datasetIds[0];
@@ -4787,6 +4790,11 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       "force only applies when resuming an experiment. Pass it together with resume."
     );
   }
+  if (options?.experimentGroupName !== void 0 && options.experimentGroupId === void 0) {
+    throw new BitfabError(
+      "experimentGroupName only applies with experimentGroupId. Pass it together with experimentGroupId."
+    );
+  }
   if (options?.experimentalSelectiveReplay !== void 0) {
     validateSelectiveReplayOptions(options.experimentalSelectiveReplay);
     if ((options.mock ?? "marked") !== "marked" || normalizeMockOverrides(options.mockOverride).length > 0 || registeredOverrides.length > 0) {
@@ -4861,6 +4869,7 @@ async function replay(httpClient, serviceUrl, traceFunctionKey, fn, options, reg
       options?.notes,
       options?.metadata,
       options?.dryRun === true,
+      options?.experimentGroupName,
       safeReplayInvocation(
         () => buildReplayInvocation(
           Object.entries(cliFlags(options) ?? explicitReplayOptions(options))
@@ -5313,6 +5322,7 @@ var init_replay = __esm({
       "mockOverride",
       "dbBranch",
       "experimentGroupId",
+      "experimentGroupName",
       "datasetId",
       "datasetIds",
       "graderIds",
@@ -7162,7 +7172,7 @@ function assertEveryVerdictNamesAnAssertion(updates) {
       (update) => update.traceId ?? update.originalTraceId
     );
     throw new BitfabError(
-      `Agent verdicts must name an assertion: ${targets.join(", ")}. Pass the assertionId of the assertion each verdict scores to labels.save, saveAll, and skip. Only archive may omit it, to clear an old whole-trace verdict.`
+      `Agent verdicts must name an assertion: ${targets.join(", ")}. Pass the assertionId of the assertion each verdict scores to labels.save, saveAll, skip, and abstain. Only archive may omit it, to clear an old whole-trace verdict.`
     );
   }
 }
@@ -7202,6 +7212,15 @@ var LabelsClient = class {
   async skip(target, experimentId) {
     return this.save({ ...target, skip: true }, experimentId);
   }
+  /**
+   * Record that the evidence does not show the part of the run this assertion
+   * checks, so there was nothing to judge. Counted as a fail in every pass
+   * rate, but stored and read back as an abstention, apart from a fail. Pass
+   * `annotation` to say what the evidence would have had to show.
+   */
+  async abstain(target, experimentId) {
+    return this.save({ ...target, abstain: true }, experimentId);
+  }
   async archive(target, experimentId) {
     return this.save({ ...target, archive: true }, experimentId);
   }