@bitfab/sdk 0.65.4 → 0.65.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/autoTrace-CZu0BXhG.js +270 -0
- package/dist/autoTrace-CZu0BXhG.js.map +1 -0
- package/dist/autoTrace-KM9pYqk9.cjs +395 -0
- package/dist/autoTrace-KM9pYqk9.cjs.map +1 -0
- package/dist/autoTrace.cjs +15 -353
- package/dist/autoTrace.d.cts +44 -44
- package/dist/autoTrace.d.ts +44 -44
- package/dist/autoTrace.js +1 -28
- package/dist/babelPlugin-C70OxrqA.js +209 -0
- package/dist/babelPlugin-C70OxrqA.js.map +1 -0
- package/dist/babelPlugin-Nta4-VcL.cjs +232 -0
- package/dist/babelPlugin-Nta4-VcL.cjs.map +1 -0
- package/dist/http-CG1Irqxx.js +2154 -0
- package/dist/http-CG1Irqxx.js.map +1 -0
- package/dist/index-BFkAwvuc.d.cts +4410 -0
- package/dist/index-BFkAwvuc.d.ts +4410 -0
- package/dist/index.cjs +48 -12504
- package/dist/index.d.cts +2 -4443
- package/dist/index.d.ts +2 -4443
- package/dist/index.js +3 -96
- package/dist/loaderRuntime-0huddOOP.d.ts +12 -0
- package/dist/loaderRuntime-CslfDFO7.d.cts +12 -0
- package/dist/node.cjs +56 -12520
- package/dist/node.cjs.map +1 -1
- package/dist/node.d.cts +2 -1
- package/dist/node.d.ts +2 -1
- package/dist/node.js +10 -105
- package/dist/node.js.map +1 -1
- package/dist/registryCommand-Di8x0YoJ.js +1111 -0
- package/dist/registryCommand-Di8x0YoJ.js.map +1 -0
- package/dist/replayCli.js +49 -87
- package/dist/replayCli.js.map +1 -1
- package/dist/rolldown-runtime-D7D4PA-g.js +13 -0
- package/dist/seedCli.js +10 -24
- package/dist/seedCli.js.map +1 -1
- package/dist/src-BRKm0P_S.cjs +8434 -0
- package/dist/src-BRKm0P_S.cjs.map +1 -0
- package/dist/src-CfFkki5l.js +6011 -0
- package/dist/src-CfFkki5l.js.map +1 -0
- package/dist/transform/babel.cjs +5 -422
- package/dist/transform/babel.d.cts +16 -16
- package/dist/transform/babel.d.ts +16 -16
- package/dist/transform/babel.js +1 -7
- package/dist/transform/bun.cjs +9 -478
- package/dist/transform/bun.cjs.map +1 -1
- package/dist/transform/bun.d.cts +5 -7
- package/dist/transform/bun.d.ts +5 -7
- package/dist/transform/bun.js +6 -11
- package/dist/transform/bun.js.map +1 -1
- package/dist/transform/esbuild.cjs +9 -478
- package/dist/transform/esbuild.cjs.map +1 -1
- package/dist/transform/esbuild.d.cts +5 -7
- package/dist/transform/esbuild.d.ts +5 -7
- package/dist/transform/esbuild.js +6 -11
- package/dist/transform/esbuild.js.map +1 -1
- package/dist/transform/loaderRuntime.cjs +85 -532
- package/dist/transform/loaderRuntime.cjs.map +1 -1
- package/dist/transform/loaderRuntime.d.cts +2 -12
- package/dist/transform/loaderRuntime.d.ts +2 -12
- package/dist/transform/loaderRuntime.js +56 -98
- package/dist/transform/loaderRuntime.js.map +1 -1
- package/dist/transform/nest.cjs +12 -481
- package/dist/transform/nest.cjs.map +1 -1
- package/dist/transform/nest.d.cts +8 -9
- package/dist/transform/nest.d.ts +8 -9
- package/dist/transform/nest.js +9 -14
- package/dist/transform/nest.js.map +1 -1
- package/dist/transform/next.cjs +56 -528
- package/dist/transform/next.cjs.map +1 -1
- package/dist/transform/next.d.cts +25 -25
- package/dist/transform/next.d.ts +24 -24
- package/dist/transform/next.js +52 -60
- package/dist/transform/next.js.map +1 -1
- package/dist/transform/nodeLoader.js +260 -506
- package/dist/transform/nodeLoader.js.map +1 -1
- package/dist/transform/register.js +4 -1
- package/dist/transform/register.js.map +1 -1
- package/dist/transform/rolldown.cjs +9 -478
- package/dist/transform/rolldown.cjs.map +1 -1
- package/dist/transform/rolldown.d.cts +5 -7
- package/dist/transform/rolldown.d.ts +5 -7
- package/dist/transform/rolldown.js +6 -11
- package/dist/transform/rolldown.js.map +1 -1
- package/dist/transform/rollup.cjs +9 -478
- package/dist/transform/rollup.cjs.map +1 -1
- package/dist/transform/rollup.d.cts +5 -7
- package/dist/transform/rollup.d.ts +5 -7
- package/dist/transform/rollup.js +6 -11
- package/dist/transform/rollup.js.map +1 -1
- package/dist/transform/rsbuild.cjs +9 -478
- package/dist/transform/rsbuild.cjs.map +1 -1
- package/dist/transform/rsbuild.d.cts +5 -7
- package/dist/transform/rsbuild.d.ts +5 -7
- package/dist/transform/rsbuild.js +6 -11
- package/dist/transform/rsbuild.js.map +1 -1
- package/dist/transform/rspack.cjs +9 -478
- package/dist/transform/rspack.cjs.map +1 -1
- package/dist/transform/rspack.d.cts +5 -7
- package/dist/transform/rspack.d.ts +5 -7
- package/dist/transform/rspack.js +6 -11
- package/dist/transform/rspack.js.map +1 -1
- package/dist/transform/swc.cjs +18 -486
- package/dist/transform/swc.cjs.map +1 -1
- package/dist/transform/swc.d.cts +15 -14
- package/dist/transform/swc.d.ts +15 -14
- package/dist/transform/swc.js +14 -32
- package/dist/transform/swc.js.map +1 -1
- package/dist/transform/unplugin.cjs +20 -476
- package/dist/transform/unplugin.cjs.map +1 -1
- package/dist/transform/unplugin.d.cts +4 -6
- package/dist/transform/unplugin.d.ts +4 -6
- package/dist/transform/unplugin.js +19 -8
- package/dist/transform/unplugin.js.map +1 -1
- package/dist/transform/vite.cjs +9 -478
- package/dist/transform/vite.cjs.map +1 -1
- package/dist/transform/vite.d.cts +5 -7
- package/dist/transform/vite.d.ts +5 -7
- package/dist/transform/vite.js +6 -11
- package/dist/transform/vite.js.map +1 -1
- package/dist/transform/webpack.cjs +9 -478
- package/dist/transform/webpack.cjs.map +1 -1
- package/dist/transform/webpack.d.cts +5 -7
- package/dist/transform/webpack.d.ts +5 -7
- package/dist/transform/webpack.js +6 -11
- package/dist/transform/webpack.js.map +1 -1
- package/dist/transform-BcyUocwm.js +36 -0
- package/dist/transform-BcyUocwm.js.map +1 -0
- package/dist/transform-BjxHGGKc.cjs +47 -0
- package/dist/transform-BjxHGGKc.cjs.map +1 -0
- package/dist/transformOptions-BsjCELkh.d.cts +11 -0
- package/dist/transformOptions-BsjCELkh.d.ts +11 -0
- package/package.json +5 -5
- package/dist/autoTrace.cjs.map +0 -1
- package/dist/autoTrace.js.map +0 -1
- package/dist/chunk-27B3UBSS.js +0 -2547
- package/dist/chunk-27B3UBSS.js.map +0 -1
- package/dist/chunk-B2M4KELA.js +0 -1912
- package/dist/chunk-B2M4KELA.js.map +0 -1
- package/dist/chunk-I233ERUH.js +0 -2979
- package/dist/chunk-I233ERUH.js.map +0 -1
- package/dist/chunk-ISU6FQKY.js +0 -6877
- package/dist/chunk-ISU6FQKY.js.map +0 -1
- package/dist/chunk-MT36GJMW.js +0 -106
- package/dist/chunk-MT36GJMW.js.map +0 -1
- package/dist/chunk-PTCLRG4E.js +0 -28
- package/dist/chunk-PTCLRG4E.js.map +0 -1
- package/dist/chunk-QRNHRY7F.js +0 -270
- package/dist/chunk-QRNHRY7F.js.map +0 -1
- package/dist/chunk-R6M54SPO.js +0 -1176
- package/dist/chunk-R6M54SPO.js.map +0 -1
- package/dist/chunk-SOP5AFMH.js +0 -2060
- package/dist/chunk-SOP5AFMH.js.map +0 -1
- package/dist/chunk-YVJAOGHE.js +0 -45
- package/dist/chunk-YVJAOGHE.js.map +0 -1
- package/dist/chunk-ZG6SHKEF.js +0 -404
- package/dist/chunk-ZG6SHKEF.js.map +0 -1
- package/dist/http-LL3JD2JE.js +0 -24
- package/dist/http-LL3JD2JE.js.map +0 -1
- package/dist/http-QZDTQC3C.js +0 -23
- package/dist/http-QZDTQC3C.js.map +0 -1
- package/dist/index.cjs.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/replay-636XZIS2.js +0 -29
- package/dist/replay-636XZIS2.js.map +0 -1
- package/dist/replay-S4DNF4VR.js +0 -28
- package/dist/replay-S4DNF4VR.js.map +0 -1
- package/dist/transform/babel.cjs.map +0 -1
- package/dist/transform/babel.js.map +0 -1
- package/dist/transformOptions-Df2_Chtc.d.cts +0 -10
- package/dist/transformOptions-Df2_Chtc.d.ts +0 -10
package/dist/index.d.cts
CHANGED
|
@@ -1,4443 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
type JustificationSpan
|
|
3
|
-
spanId: string;
|
|
4
|
-
text: string;
|
|
5
|
-
};
|
|
6
|
-
/** Evidence for an assertion decision: one entry per span relied on. */
|
|
7
|
-
type Justification = JustificationSpan[];
|
|
8
|
-
/** Whether a person has reviewed an assertion decision. */
|
|
9
|
-
type ApprovalState = "pending" | "approved" | "rejected";
|
|
10
|
-
type Approver = {
|
|
11
|
-
id: string;
|
|
12
|
-
fullName: string | null;
|
|
13
|
-
email: string | null;
|
|
14
|
-
imageUrl: string | null;
|
|
15
|
-
};
|
|
16
|
-
/** The organization member on the hook for reviewing an assertion. */
|
|
17
|
-
type Assignee = Approver;
|
|
18
|
-
/** Read-only on this client: only a person in Bitfab sets these. */
|
|
19
|
-
type ApprovalFields = {
|
|
20
|
-
approvalState: ApprovalState;
|
|
21
|
-
approvedBy: Approver | null;
|
|
22
|
-
approvedAt: string | null;
|
|
23
|
-
};
|
|
24
|
-
|
|
25
|
-
/**
|
|
26
|
-
* BAML execution utilities for the Bitfab TypeScript SDK.
|
|
27
|
-
* This module provides functions to execute BAML prompts dynamically on the client side.
|
|
28
|
-
*/
|
|
29
|
-
/**
|
|
30
|
-
* Provider definition from the server.
|
|
31
|
-
*/
|
|
32
|
-
interface ProviderDefinition {
|
|
33
|
-
provider: string;
|
|
34
|
-
apiKeyEnv: string;
|
|
35
|
-
models: Array<{
|
|
36
|
-
model: string;
|
|
37
|
-
description: string;
|
|
38
|
-
}>;
|
|
39
|
-
}
|
|
40
|
-
/**
|
|
41
|
-
* Result of a BAML function execution with raw collector data.
|
|
42
|
-
*/
|
|
43
|
-
interface BamlExecutionResult {
|
|
44
|
-
/** The parsed result of the function */
|
|
45
|
-
result: unknown;
|
|
46
|
-
/** Raw collector data for the server to parse */
|
|
47
|
-
rawCollector: Record<string, unknown> | null;
|
|
48
|
-
}
|
|
49
|
-
/**
|
|
50
|
-
* Type for allowed environment variables.
|
|
51
|
-
* Only OPENAI_API_KEY is currently supported.
|
|
52
|
-
*/
|
|
53
|
-
type AllowedEnvVars = {
|
|
54
|
-
OPENAI_API_KEY?: string;
|
|
55
|
-
};
|
|
56
|
-
|
|
57
|
-
/** A node is identified by its workflow key and span name, never its display label. */
|
|
58
|
-
interface ReplayNodeIdentity {
|
|
59
|
-
traceFunctionKey: string;
|
|
60
|
-
spanName: string;
|
|
61
|
-
}
|
|
62
|
-
/** Experimental, explicit policy for the TypeScript selective replay prototype. */
|
|
63
|
-
interface SelectiveReplayOptions {
|
|
64
|
-
/** Subtrees that must execute: changed code and unchanged code with required effects. */
|
|
65
|
-
mustRun: ReplayNodeIdentity[];
|
|
66
|
-
}
|
|
67
|
-
interface SelectiveReplayNode extends ReplayNodeIdentity {
|
|
68
|
-
id: string;
|
|
69
|
-
parentId: string | null;
|
|
70
|
-
mustRun: boolean;
|
|
71
|
-
reusable: boolean;
|
|
72
|
-
reason: "root" | "must-run" | "ancestor" | "assertion" | "unresolved-must-run" | "eligible" | "not-reusable" | "missing-recording";
|
|
73
|
-
recording?: {
|
|
74
|
-
input: unknown[];
|
|
75
|
-
kwargs?: Record<string, unknown>;
|
|
76
|
-
output: unknown;
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
interface SelectiveReplayAssertion {
|
|
80
|
-
id: string;
|
|
81
|
-
target: null | {
|
|
82
|
-
kind: "output";
|
|
83
|
-
} | {
|
|
84
|
-
kind: "span";
|
|
85
|
-
name: string;
|
|
86
|
-
occurrence?: "first" | "last" | number;
|
|
87
|
-
};
|
|
88
|
-
}
|
|
89
|
-
interface SelectiveReplayPlan {
|
|
90
|
-
version: 4;
|
|
91
|
-
rootId: string;
|
|
92
|
-
options: SelectiveReplayOptions;
|
|
93
|
-
unresolvedMustRun: ReplayNodeIdentity[];
|
|
94
|
-
nodes: SelectiveReplayNode[];
|
|
95
|
-
assertions: SelectiveReplayAssertion[];
|
|
96
|
-
}
|
|
97
|
-
interface SelectiveReplayDecision extends ReplayNodeIdentity {
|
|
98
|
-
originalSpanId: string | null;
|
|
99
|
-
action: "run" | "mock" | "blocked";
|
|
100
|
-
reason: string;
|
|
101
|
-
}
|
|
102
|
-
/** Per-item execution evidence. A node not reached is not claimed as tested. */
|
|
103
|
-
interface SelectiveReplayReport {
|
|
104
|
-
decisions: SelectiveReplayDecision[];
|
|
105
|
-
mustRunNotReached: ReplayNodeIdentity[];
|
|
106
|
-
unresolvedMustRun: ReplayNodeIdentity[];
|
|
107
|
-
/** Assertion IDs loaded for planning; these are not verdicts. */
|
|
108
|
-
assertionIds: string[];
|
|
109
|
-
/** Assertion IDs whose span target was absent/out of range in the recording. */
|
|
110
|
-
unresolvedAssertions: string[];
|
|
111
|
-
/** Assertion IDs whose requested span occurrence was not reached in this replay. */
|
|
112
|
-
assertionTargetsNotReached: string[];
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
declare class TraceCompletion {
|
|
116
|
-
private readonly traces;
|
|
117
|
-
private readonly closed;
|
|
118
|
-
private state;
|
|
119
|
-
start(traceId: string, spanId: string): void;
|
|
120
|
-
open(traceId: string): void;
|
|
121
|
-
hold(traceId: string, key: string): void;
|
|
122
|
-
record(traceId: string, spanId: string): void;
|
|
123
|
-
end(traceId: string, spanId: string): void;
|
|
124
|
-
abort(traceId: string, spanId: string): void;
|
|
125
|
-
close(traceId: string, complete: (count: number) => void, dropped?: boolean): void;
|
|
126
|
-
private drain;
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
/**
|
|
130
|
-
* Per-trace database snapshot ref capture.
|
|
131
|
-
*
|
|
132
|
-
* Every root span carries a `DbSnapshotRef` that pins the DB state at trace
|
|
133
|
-
* open by wall-clock timestamp. Capturing the timestamp is free (no IO) and
|
|
134
|
-
* harmless, so it happens on every trace regardless of configuration: that
|
|
135
|
-
* lets any trace be replayed against a historical branch later. A `provider`
|
|
136
|
-
* is attached only when the customer configured `dbSnapshot`; when absent it
|
|
137
|
-
* is resolved at replay time. The Bitfab service uses the timestamp to
|
|
138
|
-
* materialize an ephemeral branch from `customer-main`.
|
|
139
|
-
*/
|
|
140
|
-
declare const SUPPORTED_PROVIDERS: readonly ["neon"];
|
|
141
|
-
type DbSnapshotProvider = (typeof SUPPORTED_PROVIDERS)[number];
|
|
142
|
-
interface DbSnapshotConfig {
|
|
143
|
-
/** Discriminator for the server-side resolver. */
|
|
144
|
-
provider: DbSnapshotProvider;
|
|
145
|
-
}
|
|
146
|
-
interface DbSnapshotRef {
|
|
147
|
-
/**
|
|
148
|
-
* The wall-clock ISO timestamp the SDK observed immediately before
|
|
149
|
-
* invoking the wrapped function. The name encodes its provenance:
|
|
150
|
-
* SDK-observed, wall clock (not monotonic), captured before user code
|
|
151
|
-
* began executing. Always present.
|
|
152
|
-
*/
|
|
153
|
-
sdkWallClockBeforeFn: string;
|
|
154
|
-
/**
|
|
155
|
-
* The configured provider for server-side branch resolution. Only set when
|
|
156
|
-
* the customer configured `dbSnapshot`; otherwise the provider is resolved
|
|
157
|
-
* at replay time.
|
|
158
|
-
*/
|
|
159
|
-
provider?: DbSnapshotProvider;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
/**
|
|
163
|
-
* Shared error type for Bitfab SDK runtime errors. Lives in its own
|
|
164
|
-
* module to avoid import cycles between `http.ts` and modules that need
|
|
165
|
-
* to throw structured errors (e.g. `dbSnapshot.ts` validation).
|
|
166
|
-
*/
|
|
167
|
-
declare class BitfabError extends Error {
|
|
168
|
-
readonly url?: string | undefined;
|
|
169
|
-
/**
|
|
170
|
-
* HTTP status the request failed with, when it failed with one. The
|
|
171
|
-
* transport's retry policy needs the code itself (retry 408/425/429/5xx,
|
|
172
|
-
* never a 4xx the server will reject again), which a formatted message
|
|
173
|
-
* cannot supply. Absent for network failures and non-HTTP errors.
|
|
174
|
-
*/
|
|
175
|
-
readonly status?: number | undefined;
|
|
176
|
-
/**
|
|
177
|
-
* `Retry-After` in milliseconds, when the server sent one. A 429 or 503
|
|
178
|
-
* carries the server's own instruction about when to come back; retrying
|
|
179
|
-
* on our own schedule ignores it and keeps the pressure on.
|
|
180
|
-
*/
|
|
181
|
-
readonly retryAfterMs?: number | undefined;
|
|
182
|
-
constructor(message: string, url?: string | undefined,
|
|
183
|
-
/**
|
|
184
|
-
* HTTP status the request failed with, when it failed with one. The
|
|
185
|
-
* transport's retry policy needs the code itself (retry 408/425/429/5xx,
|
|
186
|
-
* never a 4xx the server will reject again), which a formatted message
|
|
187
|
-
* cannot supply. Absent for network failures and non-HTTP errors.
|
|
188
|
-
*/
|
|
189
|
-
status?: number | undefined,
|
|
190
|
-
/**
|
|
191
|
-
* `Retry-After` in milliseconds, when the server sent one. A 429 or 503
|
|
192
|
-
* carries the server's own instruction about when to come back; retrying
|
|
193
|
-
* on our own schedule ignores it and keeps the pressure on.
|
|
194
|
-
*/
|
|
195
|
-
retryAfterMs?: number | undefined);
|
|
196
|
-
}
|
|
197
|
-
declare class MixedTracingError extends Error {
|
|
198
|
-
constructor(message: string);
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
/**
|
|
202
|
-
* Selective mock overrides for replay.
|
|
203
|
-
*
|
|
204
|
-
* A mock override injects a custom value into a specific span (node) during
|
|
205
|
-
* replay: the matched span short-circuits its real execution and returns the
|
|
206
|
-
* value you supply, so downstream real code runs against the substituted
|
|
207
|
-
* output. This is a third mock mode alongside "run real code" and "replay
|
|
208
|
-
* recorded output" (see {@link MockStrategy}).
|
|
209
|
-
*
|
|
210
|
-
* The matcher and value are deliberately separate so matching stays cheap
|
|
211
|
-
* (structural metadata only, no output fetch) and the recorded output is
|
|
212
|
-
* fetched lazily - and only if the value function actually asks for it via
|
|
213
|
-
* {@link MockOverrideCtx.getOriginalOutput}.
|
|
214
|
-
*/
|
|
215
|
-
/**
|
|
216
|
-
* Structural identity of a span during replay, passed to a {@link NodeMatcher}.
|
|
217
|
-
* Carries no output payload - matching must not depend on the recorded output,
|
|
218
|
-
* so the output fetch stays lazy and gated.
|
|
219
|
-
*/
|
|
220
|
-
interface SpanNodeMeta {
|
|
221
|
-
/** The `withSpan` key of the executing span (its code-side identity). */
|
|
222
|
-
traceFunctionKey: string;
|
|
223
|
-
/** Resolved span name: `options.name ?? fn.name ?? traceFunctionKey`. */
|
|
224
|
-
spanName: string;
|
|
225
|
-
/** Span type, e.g. "llm", "agent", "tool", "custom". */
|
|
226
|
-
type: string;
|
|
227
|
-
/**
|
|
228
|
-
* The id of this span in the original (replayed) trace, when it exists in
|
|
229
|
-
* that trace's tree. Undefined when the live span has no recorded
|
|
230
|
-
* counterpart (e.g. a span the changed code newly introduced).
|
|
231
|
-
*/
|
|
232
|
-
originalSpanId?: string;
|
|
233
|
-
}
|
|
234
|
-
/** Context passed to a {@link MockValue} function for a matched span. */
|
|
235
|
-
interface MockOverrideCtx {
|
|
236
|
-
/** The matched span's structural metadata. */
|
|
237
|
-
node: SpanNodeMeta;
|
|
238
|
-
/** The live replay args passed to the wrapped function this run. */
|
|
239
|
-
inputs: unknown[];
|
|
240
|
-
/**
|
|
241
|
-
* Lazily fetch this span's ORIGINAL recorded output (deserialized). The fetch
|
|
242
|
-
* happens only when called and is memoized per replay item, so a purely flat
|
|
243
|
-
* override that never calls it triggers zero output round trips. Rejects if
|
|
244
|
-
* the span has no recorded counterpart. Being async, it is only usable for
|
|
245
|
-
* spans wrapping async functions.
|
|
246
|
-
*/
|
|
247
|
-
getOriginalOutput: () => Promise<unknown>;
|
|
248
|
-
}
|
|
249
|
-
/**
|
|
250
|
-
* Return this from a mock override resolver to decline the override for the
|
|
251
|
-
* current span. Resolution continues with the next override, then the replay's
|
|
252
|
-
* base mock strategy.
|
|
253
|
-
*/
|
|
254
|
-
declare const NO_MOCK_OVERRIDE: unique symbol;
|
|
255
|
-
/** Selects which spans an override applies to. Runs on structural metadata. */
|
|
256
|
-
type NodeMatcher = (node: SpanNodeMeta) => boolean;
|
|
257
|
-
/** The function form of {@link MockValue}, receiving the override context. */
|
|
258
|
-
type MockValueFn = (ctx: MockOverrideCtx) => unknown | Promise<unknown>;
|
|
259
|
-
/**
|
|
260
|
-
* A client-wide, keyed, or per-replay resolver. A global resolver can route on
|
|
261
|
-
* `ctx.node.traceFunctionKey`; a keyed resolver is only invoked for spans with
|
|
262
|
-
* its registered key. Return {@link NO_MOCK_OVERRIDE} for spans the resolver
|
|
263
|
-
* does not want to override.
|
|
264
|
-
*/
|
|
265
|
-
type MockOverrideResolver = (ctx: MockOverrideCtx) => unknown | Promise<unknown>;
|
|
266
|
-
/**
|
|
267
|
-
* The value injected for a matched span: either a flat value used as-is, or a
|
|
268
|
-
* function of the {@link MockOverrideCtx} that returns one (or a Promise of
|
|
269
|
-
* one). Full replacement - the value IS the span's output, no merge with the
|
|
270
|
-
* recorded output.
|
|
271
|
-
*
|
|
272
|
-
* The flat side is spelled out (rather than `unknown`) so an inline function
|
|
273
|
-
* still gets a typed `ctx`: `unknown | Fn` would collapse to `unknown` and drop
|
|
274
|
-
* the contextual type. To inject a value that is itself a function, wrap it:
|
|
275
|
-
* `value: () => theFunction`.
|
|
276
|
-
*/
|
|
277
|
-
type MockValue = MockValueFn | string | number | boolean | bigint | symbol | object | null | undefined;
|
|
278
|
-
/** One (match, value) pair. */
|
|
279
|
-
interface MockOverride {
|
|
280
|
-
match: NodeMatcher;
|
|
281
|
-
value: MockValue;
|
|
282
|
-
}
|
|
283
|
-
/** Accepted shape for one mock override declaration. */
|
|
284
|
-
type MockOverrideInput = MockOverride | MockOverrideResolver;
|
|
285
|
-
|
|
286
|
-
declare class SelectiveReplayRuntime {
|
|
287
|
-
private readonly plan;
|
|
288
|
-
private readonly options;
|
|
289
|
-
private readonly nodes;
|
|
290
|
-
private readonly children;
|
|
291
|
-
private readonly runtimeParents;
|
|
292
|
-
private readonly consumed;
|
|
293
|
-
private readonly required;
|
|
294
|
-
private readonly requiredScopes;
|
|
295
|
-
private readonly recordings;
|
|
296
|
-
private readonly reached;
|
|
297
|
-
private readonly decisions;
|
|
298
|
-
private readonly assertionNames;
|
|
299
|
-
private readonly assertionScopes;
|
|
300
|
-
private readonly observedNames;
|
|
301
|
-
private readonly unresolvedAssertions;
|
|
302
|
-
private readonly fullTraceForAssertions;
|
|
303
|
-
private failure?;
|
|
304
|
-
constructor(plan: SelectiveReplayPlan, options: SelectiveReplayOptions);
|
|
305
|
-
enter(params: ReplayNodeIdentity & {
|
|
306
|
-
spanId: string;
|
|
307
|
-
parentSpanId: string | null;
|
|
308
|
-
inputs: unknown[];
|
|
309
|
-
safetyMock: boolean;
|
|
310
|
-
canReuseOutput: boolean;
|
|
311
|
-
reuseAllowed: boolean;
|
|
312
|
-
}): {
|
|
313
|
-
mock: false;
|
|
314
|
-
} | {
|
|
315
|
-
mock: true;
|
|
316
|
-
output: unknown;
|
|
317
|
-
};
|
|
318
|
-
assertNoSafetyConflict(): void;
|
|
319
|
-
report(): SelectiveReplayReport;
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
/**
|
|
323
|
-
* Replay context propagation via AsyncLocalStorage.
|
|
324
|
-
*
|
|
325
|
-
* When set, the withSpan wrapper injects experimentId into the span payload
|
|
326
|
-
* so that new spans created during replay are linked to the experiment.
|
|
327
|
-
* Optionally carries a mock tree so child spans can return historical
|
|
328
|
-
* outputs instead of executing.
|
|
329
|
-
*/
|
|
330
|
-
|
|
331
|
-
/**
|
|
332
|
-
* A single span entry in the mock tree.
|
|
333
|
-
*
|
|
334
|
-
* Under the eager path (`mock: "all"`) `output`/`outputMeta` are populated
|
|
335
|
-
* inline, even when overrides are present. Under a non-`all` path that needs a
|
|
336
|
-
* tree (`marked`, or `none` with overrides), they are absent and the recorded
|
|
337
|
-
* output is fetched on demand via `externalSpanId` - see
|
|
338
|
-
* {@link ReplayContext.fetchSpanOutput}.
|
|
339
|
-
*/
|
|
340
|
-
interface MockSpan {
|
|
341
|
-
sourceSpanId: string;
|
|
342
|
-
/** Row id accepted by `getExternalSpan`, for the lazy per-span output fetch. */
|
|
343
|
-
externalSpanId?: string;
|
|
344
|
-
output?: unknown;
|
|
345
|
-
outputMeta?: unknown;
|
|
346
|
-
}
|
|
347
|
-
/**
|
|
348
|
-
* Per-item DB branch resolved by the Bitfab service from the source
|
|
349
|
-
* trace's `dbSnapshotRef`. Carried on the replay context so that
|
|
350
|
-
* customer code reads `databaseUrl` through `getCurrentReplayBranch()`, and so
|
|
351
|
-
* the process-isolated replay runner can materialize it into a `.env`
|
|
352
|
-
* overlay file before customer code initializes its DB client.
|
|
353
|
-
*
|
|
354
|
-
* `neonBranchId` is the literal Neon branch id; passing it to
|
|
355
|
-
* `releaseDbBranchLease` deletes that branch.
|
|
356
|
-
*/
|
|
357
|
-
interface DbBranchLease {
|
|
358
|
-
neonBranchId: string;
|
|
359
|
-
/** Env var name the customer's app reads, e.g. "DATABASE_URL". */
|
|
360
|
-
envKey: string;
|
|
361
|
-
databaseUrl: string;
|
|
362
|
-
expiresAt: string;
|
|
363
|
-
/**
|
|
364
|
-
* The instant the branch was pinned to (the source trace's wall clock).
|
|
365
|
-
* Echoed back in `db_snapshot_usage` on the replayed trace's completion.
|
|
366
|
-
*/
|
|
367
|
-
snapshotTimestamp?: string;
|
|
368
|
-
providerConsoleUrl?: string;
|
|
369
|
-
readOnly?: boolean;
|
|
370
|
-
/**
|
|
371
|
-
* The branch's region, e.g. `aws-us-east-1`. A compute runs in its project's
|
|
372
|
-
* region, so a runner elsewhere pays that round trip on every query.
|
|
373
|
-
*/
|
|
374
|
-
region?: string;
|
|
375
|
-
}
|
|
376
|
-
/**
|
|
377
|
-
* How long each phase of provisioning one replay branch took, measured
|
|
378
|
-
* server-side. A runner in another region sees these plus its own round trip.
|
|
379
|
-
*
|
|
380
|
-
* Durations, not instants: an instant is approximately `startedAt` plus the
|
|
381
|
-
* running sum, and per-phase wall-clock stamps would make clock skew between
|
|
382
|
-
* the server and your runner look like latency. Approximately, because
|
|
383
|
-
* `totalMs` is the resolve's true wall time and covers a little work no phase
|
|
384
|
-
* owns, so the phases account for it without summing to it exactly.
|
|
385
|
-
*
|
|
386
|
-
* `startedAt` and `totalMs` are always present. The phases are optional
|
|
387
|
-
* because a failed resolve reports only the ones it reached, and `totalMs` is
|
|
388
|
-
* then time-to-failure. On success every phase is present except `warmupMs`,
|
|
389
|
-
* which is absent when no warm-up SQL was supplied.
|
|
390
|
-
*/
|
|
391
|
-
interface DbBranchTimings {
|
|
392
|
-
/** When the resolve began, ISO. */
|
|
393
|
-
startedAt: string;
|
|
394
|
-
/** Resolving the project, plus its retention and region reads. */
|
|
395
|
-
projectResolveMs?: number;
|
|
396
|
-
/** Creating the branch, through its provider operations reaching terminal. */
|
|
397
|
-
branchCreateMs?: number;
|
|
398
|
-
/** Resolving the connection URI. 0 when the provider returns one inline. */
|
|
399
|
-
connectionUriMs?: number;
|
|
400
|
-
/** The compute accepting a connection. */
|
|
401
|
-
computeConnectMs?: number;
|
|
402
|
-
/** The branch answering a readiness query. */
|
|
403
|
-
baseProbeMs?: number;
|
|
404
|
-
/** Your warm-up SQL. Absent when you supplied none. */
|
|
405
|
-
warmupMs?: number;
|
|
406
|
-
/** The whole resolve, or time-to-failure when it threw. */
|
|
407
|
-
totalMs: number;
|
|
408
|
-
}
|
|
409
|
-
/**
|
|
410
|
-
* Wire shape of the caller's `ReplayOptions.dbBranch`, sent to
|
|
411
|
-
* `/api/sdk/replay/start` and applied per lease.
|
|
412
|
-
*/
|
|
413
|
-
interface DbBranchSettings {
|
|
414
|
-
minCu?: number;
|
|
415
|
-
maxCu?: number;
|
|
416
|
-
warmupSql?: string;
|
|
417
|
-
}
|
|
418
|
-
/**
|
|
419
|
-
* Pre-built lookup table of historical span outputs.
|
|
420
|
-
* Keys are `${traceFunctionKey}:${spanName}:${callIndex}` so that repeated
|
|
421
|
-
* calls with the same (key, name) are matched by call order, but spans
|
|
422
|
-
* sharing only the traceFunctionKey (different name) do not collide.
|
|
423
|
-
*/
|
|
424
|
-
interface MockTree {
|
|
425
|
-
spans: Map<string, MockSpan>;
|
|
426
|
-
}
|
|
427
|
-
interface ReplayContext {
|
|
428
|
-
experimentId: string;
|
|
429
|
-
selectiveReplay?: SelectiveReplayRuntime;
|
|
430
|
-
traceId?: string;
|
|
431
|
-
inputSourceSpanId?: string;
|
|
432
|
-
/**
|
|
433
|
-
* External trace ID from `external_traces.id`. Used for span-chain
|
|
434
|
-
* lookup against the source platform's trace tree (Braintrust, etc.).
|
|
435
|
-
* NOT the same as the Bitfab `traceId` - see `sourceBitfabTraceId`.
|
|
436
|
-
*/
|
|
437
|
-
inputSourceTraceId?: string;
|
|
438
|
-
/**
|
|
439
|
-
* The Bitfab `traces.id` of the historical trace that produced this
|
|
440
|
-
* replay item's input. This is what customer-facing surfaces (e.g.
|
|
441
|
-
* `ReplayBranch.traceId`) should expose, since it's the ID the
|
|
442
|
-
* customer sees in the Bitfab dashboard.
|
|
443
|
-
*/
|
|
444
|
-
sourceBitfabTraceId?: string;
|
|
445
|
-
replayAttempt?: number;
|
|
446
|
-
mockTree?: MockTree;
|
|
447
|
-
callCounters?: Map<string, number>;
|
|
448
|
-
mockStrategy?: "none" | "all" | "marked";
|
|
449
|
-
/**
|
|
450
|
-
* Resolved override chain for this replay, per-call overrides first then
|
|
451
|
-
* registered ones (first matcher wins). Empty/absent when no overrides apply.
|
|
452
|
-
*/
|
|
453
|
-
mockOverrides?: MockOverride[];
|
|
454
|
-
/**
|
|
455
|
-
* Memoized lazy fetch of a span's recorded output (deserialized), keyed by
|
|
456
|
-
* `externalSpanId`. Present ONLY on a non-`all` path that needs a tree
|
|
457
|
-
* (`marked`, or `none` with overrides); absent under `mock: "all"`, where
|
|
458
|
-
* outputs are inline even when overrides are present. Its presence is the
|
|
459
|
-
* signal that outputs must be fetched rather than read inline.
|
|
460
|
-
*/
|
|
461
|
-
fetchSpanOutput?: (externalSpanId: string) => Promise<unknown>;
|
|
462
|
-
/** Internal framework state captured on the historical root span. */
|
|
463
|
-
integrationFixtures?: Map<string, unknown>;
|
|
464
|
-
dbBranchLease?: DbBranchLease;
|
|
465
|
-
/**
|
|
466
|
-
* Server-measured provisioning timings for this item's branch, echoed back
|
|
467
|
-
* on the trace completion so the trace records what it cost to set up. Kept
|
|
468
|
-
* off `ReplayBranch`: customer code reads that mid-replay to reach the
|
|
469
|
-
* branch, and provisioning latency is a property of the run, not of the
|
|
470
|
-
* connection.
|
|
471
|
-
*/
|
|
472
|
-
dbBranchTimings?: DbBranchTimings;
|
|
473
|
-
/**
|
|
474
|
-
* Set to true by `ReplayBranch` the first time customer code actually
|
|
475
|
-
* obtains `databaseUrl` for this item. Reported on the trace completion inside
|
|
476
|
-
* `db_snapshot_usage` so the server can distinguish "branch was
|
|
477
|
-
* provisioned and exposed" from "branch URL was actually consumed".
|
|
478
|
-
* Only an explicit `databaseUrl` read may set it. A path that hands the URL
|
|
479
|
-
* over by other means (e.g. a process-isolated runner writing an env
|
|
480
|
-
* overlay) must leave it alone: setting it there would make every such
|
|
481
|
-
* replay report `accessed` for free, and the flag would stop separating
|
|
482
|
-
* "branch was used" from "branch was offered".
|
|
483
|
-
*/
|
|
484
|
-
dbSnapshotAccessed?: boolean;
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
type FlagScalar = string | number | boolean | null;
|
|
488
|
-
type FlagValue = FlagScalar | FlagScalar[];
|
|
489
|
-
type ReplayEnvironment = "local" | "ci" | "cloud_replay";
|
|
490
|
-
interface ReplayInvocation {
|
|
491
|
-
flags: Record<string, FlagValue>;
|
|
492
|
-
sdk: {
|
|
493
|
-
language: "typescript";
|
|
494
|
-
version: string;
|
|
495
|
-
};
|
|
496
|
-
environment: ReplayEnvironment;
|
|
497
|
-
}
|
|
498
|
-
|
|
499
|
-
interface SimulationPlanNode {
|
|
500
|
-
traceFunctionKey: string;
|
|
501
|
-
name: string;
|
|
502
|
-
captureContent: boolean;
|
|
503
|
-
}
|
|
504
|
-
interface SimulationPlanResponse {
|
|
505
|
-
nodes: SimulationPlanNode[];
|
|
506
|
-
}
|
|
507
|
-
type SimulationPlanSubmit = (payload: Record<string, unknown>) => void;
|
|
508
|
-
|
|
509
|
-
/**
|
|
510
|
-
* Wait for pending fire-and-forget requests AND every live span transport to
|
|
511
|
-
* deliver, within one total deadline. Useful in tests and scripts to ensure all
|
|
512
|
-
* data has been sent before asserting or exiting.
|
|
513
|
-
*
|
|
514
|
-
* Returns `false` when delivery failed or the deadline expired, so a caller
|
|
515
|
-
* that depends on persistence (replay does) can react instead of assuming a
|
|
516
|
-
* drained queue means the server has the data.
|
|
517
|
-
*
|
|
518
|
-
* @param timeoutMs - Maximum total time to wait in milliseconds (default: 5000)
|
|
519
|
-
*/
|
|
520
|
-
declare function flushTraces(timeoutMs?: number): Promise<boolean>;
|
|
521
|
-
/**
|
|
522
|
-
* How the API key is supplied internally: either a literal string or a
|
|
523
|
-
* function resolved each time the key is needed (at request/send time). The
|
|
524
|
-
* function form is what defers key resolution past module-load construction
|
|
525
|
-
* so an env var loaded after the client is built (the ESM dotenv-hoisting
|
|
526
|
-
* case) is still picked up.
|
|
527
|
-
*/
|
|
528
|
-
type ApiKeyInput = string | (() => string | undefined);
|
|
529
|
-
interface HttpClientConfig {
|
|
530
|
-
apiKey?: ApiKeyInput;
|
|
531
|
-
serviceUrl: string;
|
|
532
|
-
timeout?: number;
|
|
533
|
-
}
|
|
534
|
-
type SpanOccurrence = "first" | "last" | number;
|
|
535
|
-
declare const SPAN_TYPES: readonly ["llm", "agent", "function", "guardrail", "handoff", "custom"];
|
|
536
|
-
/**
|
|
537
|
-
* Span types matching the backend enum.
|
|
538
|
-
* - llm: LLM API calls
|
|
539
|
-
* - agent: Autonomous orchestrators
|
|
540
|
-
* - function: Tool implementations
|
|
541
|
-
* - guardrail: Safety/validation checks
|
|
542
|
-
* - handoff: Agent-to-agent transfers
|
|
543
|
-
* - custom: Application-specific tracing (default)
|
|
544
|
-
*/
|
|
545
|
-
type SpanType = (typeof SPAN_TYPES)[number];
|
|
546
|
-
/**
|
|
547
|
-
* Which span `getTraceSpan` returns: `id` alone, or `name` and/or `type`
|
|
548
|
-
* with an optional `occurrence` (`"first"`, `"last"` by default, or a
|
|
549
|
-
* zero-based index into the matches in the order they ran).
|
|
550
|
-
*/
|
|
551
|
-
type SpanLookup = {
|
|
552
|
-
id: string;
|
|
553
|
-
name?: never;
|
|
554
|
-
type?: never;
|
|
555
|
-
occurrence?: never;
|
|
556
|
-
} | {
|
|
557
|
-
name: string;
|
|
558
|
-
type?: SpanType;
|
|
559
|
-
id?: never;
|
|
560
|
-
occurrence?: SpanOccurrence;
|
|
561
|
-
} | {
|
|
562
|
-
type: SpanType;
|
|
563
|
-
name?: string;
|
|
564
|
-
id?: never;
|
|
565
|
-
occurrence?: SpanOccurrence;
|
|
566
|
-
};
|
|
567
|
-
interface CapturedSpan {
|
|
568
|
-
id: string;
|
|
569
|
-
traceId: string;
|
|
570
|
-
parentSpanId: string | null;
|
|
571
|
-
name: string | null;
|
|
572
|
-
type: string;
|
|
573
|
-
input: unknown;
|
|
574
|
-
output: unknown;
|
|
575
|
-
contexts: Record<string, unknown>[];
|
|
576
|
-
prompt: string | null;
|
|
577
|
-
metadata: Record<string, unknown>;
|
|
578
|
-
metrics: Record<string, unknown> | null;
|
|
579
|
-
errors: unknown;
|
|
580
|
-
startedAt: string | null;
|
|
581
|
-
endedAt: string | null;
|
|
582
|
-
}
|
|
583
|
-
/** What a caller learns about one tracked trace once it takes it back. */
|
|
584
|
-
interface DeliveryReport {
|
|
585
|
-
spanCount: number;
|
|
586
|
-
/** A closing carrier was submitted, so the expected set is final. */
|
|
587
|
-
closed: boolean;
|
|
588
|
-
/** Every carrier submitted under this trace came back accepted. */
|
|
589
|
-
delivered: boolean;
|
|
590
|
-
/**
|
|
591
|
-
* The server's assigned `traces.id`, read back from the OTLP ingest response.
|
|
592
|
-
* Absent when talking to a server that predates this field, or before any
|
|
593
|
-
* carrier for the trace has been acked.
|
|
594
|
-
*/
|
|
595
|
-
serverTraceId?: string;
|
|
596
|
-
}
|
|
597
|
-
declare class HttpClient {
|
|
598
|
-
private readonly apiKey;
|
|
599
|
-
private readonly serviceUrl;
|
|
600
|
-
private readonly timeout;
|
|
601
|
-
private traceTransport;
|
|
602
|
-
private readonly traceDeliveries;
|
|
603
|
-
private readonly deferredWork;
|
|
604
|
-
readonly traceCompletion: TraceCompletion;
|
|
605
|
-
private closed;
|
|
606
|
-
private closing;
|
|
607
|
-
externalSpanTransform: ((payload: Record<string, unknown>, submit: SimulationPlanSubmit, drop: () => void) => void) | undefined;
|
|
608
|
-
externalTraceTransform: ((payload: Record<string, unknown>, submit: SimulationPlanSubmit) => void) | undefined;
|
|
609
|
-
releaseHeldExternalSpans: ((timeoutMs: number) => Promise<boolean> | Promise<void>) | undefined;
|
|
610
|
-
stopSimulationPlan: (() => void) | undefined;
|
|
611
|
-
constructor(config: HttpClientConfig);
|
|
612
|
-
/**
|
|
613
|
-
* Resolve the API key at the moment it is needed (request time), invoking
|
|
614
|
-
* the function form if one was supplied. Never read at construction.
|
|
615
|
-
*/
|
|
616
|
-
private resolveApiKey;
|
|
617
|
-
/** @internal Whether this client currently has credentials for trace delivery. */
|
|
618
|
-
hasApiKey(): boolean;
|
|
619
|
-
/**
|
|
620
|
-
* This client's span transport, built on first use.
|
|
621
|
-
*
|
|
622
|
-
* Lazy on purpose: a client that never sends a span must never start a batch
|
|
623
|
-
* worker. Every framework integration created from a `Bitfab` client shares
|
|
624
|
-
* the owning client's `HttpClient`, so handlers reuse this one worker instead
|
|
625
|
-
* of each spinning up their own.
|
|
626
|
-
*/
|
|
627
|
-
private getTraceTransport;
|
|
628
|
-
/**
|
|
629
|
-
* Post one encoded batch and decide what the server's answer means, so the
|
|
630
|
-
* transport never reads a response. Rejections and permanent statuses come
|
|
631
|
-
* back as a non-retryable {@link DeliveryError}; anything the server might
|
|
632
|
-
* still accept on a second try comes back retryable.
|
|
633
|
-
*/
|
|
634
|
-
private deliverCarriers;
|
|
635
|
-
/**
|
|
636
|
-
* Start tracking delivery for `traceIds`. Nothing is recorded for a trace
|
|
637
|
-
* that was never tracked, so ordinary tracing costs no bookkeeping at all.
|
|
638
|
-
*/
|
|
639
|
-
trackTraceDeliveries(traceIds: string[]): void;
|
|
640
|
-
/**
|
|
641
|
-
* The server's assigned `traces.id` for a tracked trace if it has already
|
|
642
|
-
* been read back off an ingest response, without stopping tracking. Lets a
|
|
643
|
-
* replay surface the id mid-run for items whose spans already landed.
|
|
644
|
-
*/
|
|
645
|
-
peekServerTraceId(traceId: string): string | undefined;
|
|
646
|
-
/** Whether any tracked trace has had its closing carrier submitted. */
|
|
647
|
-
hasClosedDeliveries(traceIds: string[]): boolean;
|
|
648
|
-
/**
|
|
649
|
-
* Report what each tracked trace submitted and whether the server confirmed
|
|
650
|
-
* it, and stop tracking them. Every id passed is freed, so a caller cannot
|
|
651
|
-
* leak a record for a trace that never closed.
|
|
652
|
-
*
|
|
653
|
-
* `delivered` is only meaningful once a flush has settled: acks land before
|
|
654
|
-
* an export resolves, so a flush that reported success has already collected
|
|
655
|
-
* every ack it is going to collect.
|
|
656
|
-
*/
|
|
657
|
-
takeTraceDeliveries(traceIds: string[]): Record<string, DeliveryReport>;
|
|
658
|
-
/** Build a carrier's meta and record what it adds to its trace's expected set. */
|
|
659
|
-
private recordedMeta;
|
|
660
|
-
private recordSubmittedCarrier;
|
|
661
|
-
/**
|
|
662
|
-
* Ingestion commits every carrier in a request before it answers, so a
|
|
663
|
-
* delivered ref is proof its row exists: the same fact the replay status
|
|
664
|
-
* endpoint would report, already in hand.
|
|
665
|
-
*/
|
|
666
|
-
/**
|
|
667
|
-
* Record the server's assigned `traces.id` for each tracked source trace,
|
|
668
|
-
* read back from the OTLP ingest response. Keyed by source trace id, the same
|
|
669
|
-
* key the delivery ledger uses. Untracked ids are ignored.
|
|
670
|
-
*/
|
|
671
|
-
private recordServerTraceIds;
|
|
672
|
-
private recordDeliveredCarriers;
|
|
673
|
-
/**
|
|
674
|
-
* Track deferred span work so this client's own lifecycle waits for it, and
|
|
675
|
-
* so the process-wide flush and exit hook do too.
|
|
676
|
-
*/
|
|
677
|
-
trackDeferred<T>(promise: Promise<T>): Promise<T>;
|
|
678
|
-
/**
|
|
679
|
-
* Settle only THIS client's deferred span work. Scoped deliberately: the
|
|
680
|
-
* global set can contain another client's long-running finalize, and
|
|
681
|
-
* attributing its timeout here would fail a client whose own work succeeded.
|
|
682
|
-
*/
|
|
683
|
-
settleDeferredWork(timeoutMs?: number): Promise<boolean>;
|
|
684
|
-
/**
|
|
685
|
-
* Wait for spans queued by this client to be delivered, within one deadline.
|
|
686
|
-
* Returns false on delivery failure or timeout.
|
|
687
|
-
*/
|
|
688
|
-
waitForPendingRequests(timeoutMs?: number): Promise<boolean>;
|
|
689
|
-
/**
|
|
690
|
-
* Flush and permanently close this client's tracing transport. Idempotent:
|
|
691
|
-
* a second call joins the first rather than tearing down a pipeline the
|
|
692
|
-
* first call already owns.
|
|
693
|
-
*/
|
|
694
|
-
close(timeoutMs?: number): Promise<boolean>;
|
|
695
|
-
/**
|
|
696
|
-
* Make an HTTP request to the Bitfab API. Defaults to POST; pass
|
|
697
|
-
* `options.method` to use a different verb (e.g. "PATCH").
|
|
698
|
-
*
|
|
699
|
-
* @param endpoint - The API endpoint (without base URL)
|
|
700
|
-
* @param payload - The request body
|
|
701
|
-
* @param options - Optional request options
|
|
702
|
-
* @returns The parsed JSON response
|
|
703
|
-
* @throws {BitfabError} If the request fails
|
|
704
|
-
*/
|
|
705
|
-
request<T>(endpoint: string, payload: Record<string, unknown>, options?: {
|
|
706
|
-
timeout?: number;
|
|
707
|
-
method?: "POST" | "PATCH" | "PUT" | "DELETE";
|
|
708
|
-
}): Promise<T>;
|
|
709
|
-
/**
|
|
710
|
-
* POST an already-encoded body. The span transport encodes its own batches,
|
|
711
|
-
* so routing them back through {@link HttpClient.request} would encode the
|
|
712
|
-
* same data twice.
|
|
713
|
-
*/
|
|
714
|
-
sendEncoded<T>(endpoint: string, body: string, options?: {
|
|
715
|
-
timeout?: number;
|
|
716
|
-
method?: "POST" | "PATCH" | "PUT" | "DELETE";
|
|
717
|
-
}): Promise<T>;
|
|
718
|
-
private sendPrepared;
|
|
719
|
-
/**
|
|
720
|
-
* Look up a function by name.
|
|
721
|
-
* Blocks until complete - needed for function execution.
|
|
722
|
-
*/
|
|
723
|
-
lookupFunction<T>(name: string): Promise<T>;
|
|
724
|
-
getSimulationPlan(): Promise<SimulationPlanResponse>;
|
|
725
|
-
getTraceSpan(traceId: string, lookup: SpanLookup): Promise<CapturedSpan | null>;
|
|
726
|
-
getSpan(spanId: string): Promise<CapturedSpan | null>;
|
|
727
|
-
/**
|
|
728
|
-
* GET a JSON endpoint on the service with the client's API key. Throws a
|
|
729
|
-
* `BitfabError` carrying the status text for any non-2xx response.
|
|
730
|
-
*/
|
|
731
|
-
get<T>(endpoint: string, timeoutMs?: number, extraHeaders?: Record<string, string>): Promise<T>;
|
|
732
|
-
/**
|
|
733
|
-
* Queue an internal trace (from local BAML execution via `call()`) onto this
|
|
734
|
-
* client's batching transport. `functionId` moves into the payload because
|
|
735
|
-
* the OTLP carrier has no path to carry it.
|
|
736
|
-
*/
|
|
737
|
-
sendInternalTrace(functionId: string, payload: Record<string, unknown>): void;
|
|
738
|
-
sendExternalSpan(payload: Record<string, unknown>): void;
|
|
739
|
-
submitExternalSpan(payload: Record<string, unknown>): void;
|
|
740
|
-
/**
|
|
741
|
-
* Queue an external trace completion (from OpenAI tracing) onto this
|
|
742
|
-
* client's batching transport. Fire-and-forget for the same reason as
|
|
743
|
-
* {@link HttpClient.sendExternalSpan}; replay confirms persistence with the
|
|
744
|
-
* server-authoritative barrier in `replay.ts`, not by awaiting this call.
|
|
745
|
-
*/
|
|
746
|
-
sendExternalTrace(rawPayload: Record<string, unknown>): void;
|
|
747
|
-
private sendCountedExternalTrace;
|
|
748
|
-
submitExternalTrace(payload: Record<string, unknown>): void;
|
|
749
|
-
/**
|
|
750
|
-
* Partial update of an existing trace identified by its Bitfab trace ID.
|
|
751
|
-
* Used by the detached `client.getTrace(id)` handle.
|
|
752
|
-
*
|
|
753
|
-
* Blocking, like the other trace-API calls: it resolves once the server has
|
|
754
|
-
* applied the change and rejects if the server refused it. A patch targets a
|
|
755
|
-
* trace that is already closed, so there is no batch for it to ride along
|
|
756
|
-
* with and no later signal that would reveal a silent failure.
|
|
757
|
-
*/
|
|
758
|
-
patchTrace(traceId: string, payload: {
|
|
759
|
-
appendContexts?: Record<string, unknown>[];
|
|
760
|
-
mergeMetadata?: Record<string, unknown>;
|
|
761
|
-
setSessionId?: string;
|
|
762
|
-
setName?: string;
|
|
763
|
-
}): Promise<void>;
|
|
764
|
-
/**
|
|
765
|
-
* Start a replay session by fetching historical traces.
|
|
766
|
-
* Blocking call - creates an experiment and returns lightweight item references.
|
|
767
|
-
*/
|
|
768
|
-
startReplay(traceFunctionKey: string, limit: number | undefined, traceIds?: string[], name?: string, codeChangeDescription?: string | null, codeChangeFiles?: CodeChangeFile[] | null, includeDbBranchLease?: boolean, experimentGroupId?: string, datasetIds?: string[], graderIds?: string[], dbBranchSettings?: DbBranchSettings, attempts?: number, includeOriginalMetadata?: boolean, onlyWithAssertions?: boolean, skipAssertionJudging?: boolean, experimentalSelectiveReplay?: SelectiveReplayOptions, notes?: string, metadata?: Record<string, string>, dryRun?: boolean, experimentGroupName?: string, invocation?: ReplayInvocation): Promise<StartReplayResponse>;
|
|
769
|
-
resumeReplay(experimentId: string, traceFunctionKey: string, codeChangeFiles?: CodeChangeFile[] | null, force?: boolean): Promise<ResumeReplayResponse>;
|
|
770
|
-
/**
|
|
771
|
-
* Fetch an external span by ID.
|
|
772
|
-
* Blocking GET request.
|
|
773
|
-
* The replay view limits rawData to input/output serialization fields.
|
|
774
|
-
*/
|
|
775
|
-
getExternalSpan(spanId: string, options?: {
|
|
776
|
-
view?: "full" | "replay";
|
|
777
|
-
}): Promise<ExternalSpanResponse>;
|
|
778
|
-
/**
|
|
779
|
-
* Fetch the span tree for a root span.
|
|
780
|
-
* Blocking GET request.
|
|
781
|
-
*
|
|
782
|
-
* Pass `includeOutputs: false` for a payload-free tree (structure +
|
|
783
|
-
* `externalSpanId` only), so recorded outputs are fetched lazily per mocked
|
|
784
|
-
* span instead of all up front. Omit it (default eager) for `mock: "all"`.
|
|
785
|
-
* Pass `includeRootOutput: false` when the root was already fetched.
|
|
786
|
-
*/
|
|
787
|
-
getSpanTree(externalSpanId: string, options?: {
|
|
788
|
-
includeOutputs?: boolean;
|
|
789
|
-
includeRootOutput?: boolean;
|
|
790
|
-
}): Promise<SpanTreeResponse>;
|
|
791
|
-
/**
|
|
792
|
-
* Read which of a replay run's traces the server has fully persisted.
|
|
793
|
-
*
|
|
794
|
-
* With `expectedSpanCounts`, a trace appears in the response only once it
|
|
795
|
-
* has a final status AND at least that many persisted spans, which is what
|
|
796
|
-
* makes this a real barrier rather than a "the row exists" check.
|
|
797
|
-
*/
|
|
798
|
-
getReplayStatus(experimentId: string, expectedSpanCounts: Record<string, number>): Promise<ReplayStatusResponse>;
|
|
799
|
-
/**
|
|
800
|
-
* Mark a replay experiment as completed.
|
|
801
|
-
* Blocking call.
|
|
802
|
-
*/
|
|
803
|
-
completeReplay(experimentId: string): Promise<CompleteReplayResponse>;
|
|
804
|
-
interruptReplay(experimentId: string): Promise<boolean>;
|
|
805
|
-
heartbeatReplay(experimentId: string): Promise<void>;
|
|
806
|
-
getReseedSource(traceId: string): Promise<ReseedSourceResponse>;
|
|
807
|
-
reseedTrace(traceId: string, runTraceId: string): Promise<ReseedTraceResponse>;
|
|
808
|
-
/**
|
|
809
|
-
* Ask the server to materialize a per-trace DB branch lease from a
|
|
810
|
-
* captured `dbSnapshotRef`. Blocking - the resolver creates a Neon
|
|
811
|
-
* snapshot + preview branch and polls operations to readiness, which
|
|
812
|
-
* can take seconds.
|
|
813
|
-
*/
|
|
814
|
-
resolveDbBranchLease(experimentId: string, traceId: string, dbBranchSettings?: DbBranchSettings, attempt?: number): Promise<{
|
|
815
|
-
dbSnapshotRef: DbSnapshotRef | null;
|
|
816
|
-
lease: DbBranchLease | null;
|
|
817
|
-
leaseError: {
|
|
818
|
-
code: string;
|
|
819
|
-
message: string;
|
|
820
|
-
} | null;
|
|
821
|
-
timings: DbBranchTimings | null;
|
|
822
|
-
}>;
|
|
823
|
-
/** Release a previously-resolved DB branch by deleting its Neon branch. Idempotent server-side. */
|
|
824
|
-
releaseDbBranchLease(neonBranchId: string): Promise<void>;
|
|
825
|
-
}
|
|
826
|
-
interface TokenUsage {
|
|
827
|
-
input: number | null;
|
|
828
|
-
output: number | null;
|
|
829
|
-
cached: number | null;
|
|
830
|
-
total: number | null;
|
|
831
|
-
}
|
|
832
|
-
/**
|
|
833
|
-
* Describes a single file edited as part of a code change.
|
|
834
|
-
*
|
|
835
|
-
* - `path`: file path (relative to the repo root, or any consistent root)
|
|
836
|
-
* - `before`: file contents before the change ("" for newly created files)
|
|
837
|
-
* - `after`: file contents after the change ("" for deleted files)
|
|
838
|
-
*/
|
|
839
|
-
interface CodeChangeFile {
|
|
840
|
-
path: string;
|
|
841
|
-
before: string;
|
|
842
|
-
after: string;
|
|
843
|
-
}
|
|
844
|
-
interface StartReplayResponse {
|
|
845
|
-
/** Null for a dry run, which the server records no experiment for. */
|
|
846
|
-
experimentId: string | null;
|
|
847
|
-
experimentUrl: string | null;
|
|
848
|
-
attempts?: number;
|
|
849
|
-
items: Array<{
|
|
850
|
-
selectiveReplayPlan?: SelectiveReplayPlan;
|
|
851
|
-
/** Bitfab trace ID of the original (historical) trace being replayed. */
|
|
852
|
-
originalTraceId?: string;
|
|
853
|
-
/** External span ID the recorded inputs were read from (the original root span). */
|
|
854
|
-
originalSpanId?: string;
|
|
855
|
-
/** @deprecated alias for `originalTraceId`; the only key emitted by servers that predate the rename. */
|
|
856
|
-
sourceTraceId: string;
|
|
857
|
-
/** @deprecated alias for `originalSpanId`; the only key emitted by servers that predate the rename. */
|
|
858
|
-
sourceSpanId: string;
|
|
859
|
-
durationMs: number | null;
|
|
860
|
-
tokens: TokenUsage | null;
|
|
861
|
-
model: string | null;
|
|
862
|
-
/**
|
|
863
|
-
* The DB snapshot ref captured by the SDK at trace open. Surfaced so
|
|
864
|
-
* the SDK can pass it to the lease-resolver step (or report when no
|
|
865
|
-
* snapshot was captured for this trace).
|
|
866
|
-
*/
|
|
867
|
-
dbSnapshotRef?: DbSnapshotRef;
|
|
868
|
-
/**
|
|
869
|
-
* Populated once the server-side resolver has materialized a per-item
|
|
870
|
-
* branch from `dbSnapshotRef`. The SDK exposes this to customer code
|
|
871
|
-
* via `getCurrentReplayBranch()`. Absent until the resolver lands.
|
|
872
|
-
*/
|
|
873
|
-
dbBranchLease?: DbBranchLease;
|
|
874
|
-
/**
|
|
875
|
-
* Why the branch could not be resolved, when one was requested and the
|
|
876
|
-
* attempt failed. Distinct from both fields being absent, which means the
|
|
877
|
-
* trace carried no snapshot ref so nothing was attempted.
|
|
878
|
-
*/
|
|
879
|
-
dbBranchLeaseError?: {
|
|
880
|
-
code: string;
|
|
881
|
-
message: string;
|
|
882
|
-
};
|
|
883
|
-
/**
|
|
884
|
-
* How long provisioning took, per phase. Sits beside the two fields above
|
|
885
|
-
* rather than inside either: it is reported on both outcomes, complete on
|
|
886
|
-
* success and partial up to the failing phase on error. Absent from
|
|
887
|
-
* servers that predate it.
|
|
888
|
-
*/
|
|
889
|
-
dbBranchTimings?: DbBranchTimings;
|
|
890
|
-
originalMetadata?: Record<string, unknown>;
|
|
891
|
-
}>;
|
|
892
|
-
}
|
|
893
|
-
type StartReplayItem = StartReplayResponse["items"][number];
|
|
894
|
-
interface ResumeReplayWorkPair {
|
|
895
|
-
itemIndex: number;
|
|
896
|
-
attempt: number;
|
|
897
|
-
}
|
|
898
|
-
interface ResumeReplayFinishedItem {
|
|
899
|
-
originalTraceId: string;
|
|
900
|
-
attempt: number;
|
|
901
|
-
traceId: string;
|
|
902
|
-
hasError: boolean;
|
|
903
|
-
}
|
|
904
|
-
interface ResumeReplayResponse {
|
|
905
|
-
experimentId: string;
|
|
906
|
-
experimentUrl: string;
|
|
907
|
-
attempts: number;
|
|
908
|
-
items: StartReplayItem[];
|
|
909
|
-
work: ResumeReplayWorkPair[];
|
|
910
|
-
finished: ResumeReplayFinishedItem[];
|
|
911
|
-
skippedOriginalTraceIds: string[];
|
|
912
|
-
includeDbBranchLease: boolean;
|
|
913
|
-
dbBranchSettings: DbBranchSettings | null;
|
|
914
|
-
}
|
|
915
|
-
interface ExternalSpanResponse {
|
|
916
|
-
id: string;
|
|
917
|
-
externalTraceId: string;
|
|
918
|
-
rawData: {
|
|
919
|
-
span_data: {
|
|
920
|
-
input: unknown;
|
|
921
|
-
output: unknown;
|
|
922
|
-
input_meta?: unknown;
|
|
923
|
-
output_meta?: unknown;
|
|
924
|
-
input_serialized?: {
|
|
925
|
-
json: unknown;
|
|
926
|
-
meta: unknown;
|
|
927
|
-
};
|
|
928
|
-
output_serialized?: {
|
|
929
|
-
json: unknown;
|
|
930
|
-
meta: unknown;
|
|
931
|
-
};
|
|
932
|
-
};
|
|
933
|
-
};
|
|
934
|
-
}
|
|
935
|
-
interface ReplayStatusResponse {
|
|
936
|
-
/**
|
|
937
|
-
* Local replay trace id -> server trace row id, for the traces the server
|
|
938
|
-
* considers fully persisted. Traces still short of their expected span count
|
|
939
|
-
* are simply absent.
|
|
940
|
-
*/
|
|
941
|
-
traceIds?: Record<string, string>;
|
|
942
|
-
}
|
|
943
|
-
interface CompleteReplayResponse {
|
|
944
|
-
id: string;
|
|
945
|
-
status: string;
|
|
946
|
-
traceIds?: Record<string, string>;
|
|
947
|
-
/**
|
|
948
|
-
* Per-replay-trace token usage, keyed by the server trace id (the values of
|
|
949
|
-
* `traceIds`). Aggregated server-side from the freshly-uploaded replay spans,
|
|
950
|
-
* so it's the REPLAYED run's tokens (the same source the dashboard reads), not the
|
|
951
|
-
* original trace's. The SDK maps each item onto this to set
|
|
952
|
-
* `ReplayItem.tokens`. Absent on servers that predate this field.
|
|
953
|
-
*/
|
|
954
|
-
tokens?: Record<string, TokenUsage | null>;
|
|
955
|
-
/**
|
|
956
|
-
* Number of traces the server has persisted for this experiment at
|
|
957
|
-
* completion time. Lets the SDK distinguish "uploads failed" from
|
|
958
|
-
* "server never saw them" when the trace-ID mapping is incomplete.
|
|
959
|
-
*/
|
|
960
|
-
traceCount?: number;
|
|
961
|
-
}
|
|
962
|
-
interface ReseedSourceResponse {
|
|
963
|
-
traceId: string;
|
|
964
|
-
traceFunctionKey: string;
|
|
965
|
-
ingestionType: string;
|
|
966
|
-
name: string | null;
|
|
967
|
-
sessionId: string | null;
|
|
968
|
-
metadata: Record<string, unknown>;
|
|
969
|
-
input: unknown;
|
|
970
|
-
inputMeta: unknown;
|
|
971
|
-
inputSerialized: unknown;
|
|
972
|
-
}
|
|
973
|
-
interface ReseedTraceResponse {
|
|
974
|
-
traceId: string;
|
|
975
|
-
previousRunTraceId: string;
|
|
976
|
-
}
|
|
977
|
-
interface SpanTreeNode {
|
|
978
|
-
/** Upstream platform span id. Stable structural identity; NOT the row id. */
|
|
979
|
-
sourceSpanId: string;
|
|
980
|
-
/**
|
|
981
|
-
* The `externalSpans` row id, accepted by {@link HttpClient.getExternalSpan}.
|
|
982
|
-
* Distinct from `sourceSpanId`; used to lazily fetch this node's output when
|
|
983
|
-
* the tree was fetched with `includeOutputs: false`. Optional so trees from
|
|
984
|
-
* older servers (which omit it) still deserialize.
|
|
985
|
-
*/
|
|
986
|
-
externalSpanId?: string;
|
|
987
|
-
traceFunctionKey: string;
|
|
988
|
-
spanName: string;
|
|
989
|
-
type: string;
|
|
990
|
-
/** Omitted when the tree was fetched payload-free (`includeOutputs: false`). */
|
|
991
|
-
output?: unknown;
|
|
992
|
-
outputMeta?: unknown;
|
|
993
|
-
children: SpanTreeNode[];
|
|
994
|
-
}
|
|
995
|
-
interface SpanTreeResponse {
|
|
996
|
-
root: SpanTreeNode;
|
|
997
|
-
}
|
|
998
|
-
|
|
999
|
-
type CategorySummary = {
|
|
1000
|
-
id: string;
|
|
1001
|
-
title: string;
|
|
1002
|
-
description: string;
|
|
1003
|
-
};
|
|
1004
|
-
type Category = CategorySummary & {
|
|
1005
|
-
organizationId: string;
|
|
1006
|
-
createdAt: string;
|
|
1007
|
-
updatedAt: string;
|
|
1008
|
-
};
|
|
1009
|
-
type SaveCategoryParams = {
|
|
1010
|
-
id?: string;
|
|
1011
|
-
title: string;
|
|
1012
|
-
description?: string;
|
|
1013
|
-
};
|
|
1014
|
-
/** @deprecated Use {@link CategorySummary}. Same shape. */
|
|
1015
|
-
type AssertionCategorySummary = CategorySummary;
|
|
1016
|
-
/** @deprecated Use {@link Category}. Same shape. */
|
|
1017
|
-
type AssertionCategory = Category;
|
|
1018
|
-
/** @deprecated Use {@link SaveCategoryParams}. Same shape. */
|
|
1019
|
-
type SaveAssertionCategoryParams = SaveCategoryParams;
|
|
1020
|
-
/** Organization-scoped groupings for assertions and scenarios. */
|
|
1021
|
-
declare class CategoriesClient {
|
|
1022
|
-
private readonly httpClient;
|
|
1023
|
-
constructor(httpClient: HttpClient);
|
|
1024
|
-
/** Create a category, or update it by ID. Omitted descriptions are preserved. */
|
|
1025
|
-
save(params: SaveCategoryParams): Promise<Category>;
|
|
1026
|
-
/** Read one category in the API key's organization. */
|
|
1027
|
-
get(id: string): Promise<Category>;
|
|
1028
|
-
/** List the organization's categories ordered by title. */
|
|
1029
|
-
list(): Promise<Category[]>;
|
|
1030
|
-
/** Delete a category and clear its assignments, preserving assertions and verdicts. */
|
|
1031
|
-
delete(id: string): Promise<Category>;
|
|
1032
|
-
}
|
|
1033
|
-
/**
|
|
1034
|
-
* @deprecated Use {@link CategoriesClient}. Same class.
|
|
1035
|
-
* @alias
|
|
1036
|
-
*/
|
|
1037
|
-
declare const AssertionCategoriesClient: typeof CategoriesClient;
|
|
1038
|
-
/**
|
|
1039
|
-
* @deprecated Use {@link CategoriesClient}. Same class.
|
|
1040
|
-
* @alias
|
|
1041
|
-
*/
|
|
1042
|
-
type AssertionCategoriesClient = CategoriesClient;
|
|
1043
|
-
|
|
1044
|
-
/**
|
|
1045
|
-
* Claude Agent SDK handler for Bitfab tracing.
|
|
1046
|
-
*
|
|
1047
|
-
* Hooks into the Claude Agent SDK's lifecycle to capture LLM turns,
|
|
1048
|
-
* tool invocations, and subagent execution as Bitfab spans.
|
|
1049
|
-
*
|
|
1050
|
-
* Uses two integration surfaces:
|
|
1051
|
-
* 1. SDK hooks (PreToolUse, PostToolUse, etc.) for tool/subagent lifecycle
|
|
1052
|
-
* 2. Stream wrapping for LLM turn capture from the message stream
|
|
1053
|
-
*/
|
|
1054
|
-
|
|
1055
|
-
interface ActiveSpanContext$2 {
|
|
1056
|
-
traceId: string;
|
|
1057
|
-
spanId: string;
|
|
1058
|
-
}
|
|
1059
|
-
/**
|
|
1060
|
-
* Claude Agent SDK handler that sends traces to Bitfab.
|
|
1061
|
-
*
|
|
1062
|
-
* Captures LLM turns, tool invocations, and subagent execution as
|
|
1063
|
-
* Bitfab spans with proper parent-child hierarchy.
|
|
1064
|
-
*
|
|
1065
|
-
* The TypeScript Claude Agent SDK exposes a single `query()` entry point (there
|
|
1066
|
-
* is no `ClaudeSDKClient` class - that exists only in the Python SDK). Wrap the
|
|
1067
|
-
* `query()` async iterator with `wrapQuery`; tool and subagent spans come from
|
|
1068
|
-
* the hooks injected by `instrumentOptions`.
|
|
1069
|
-
*
|
|
1070
|
-
* ```typescript
|
|
1071
|
-
* import { Bitfab } from "@bitfab/sdk";
|
|
1072
|
-
* import { query } from "@anthropic-ai/claude-agent-sdk";
|
|
1073
|
-
*
|
|
1074
|
-
* const bitfab = new Bitfab({ apiKey: "..." });
|
|
1075
|
-
* const handler = bitfab.getClaudeAgentHandler("my-agent");
|
|
1076
|
-
*
|
|
1077
|
-
* const options = handler.instrumentOptions({
|
|
1078
|
-
* model: "claude-sonnet-4-5-...",
|
|
1079
|
-
* });
|
|
1080
|
-
*
|
|
1081
|
-
* for await (const message of handler.wrapQuery(
|
|
1082
|
-
* query({ prompt: "Do something", options })
|
|
1083
|
-
* )) {
|
|
1084
|
-
* // process messages normally
|
|
1085
|
-
* }
|
|
1086
|
-
* ```
|
|
1087
|
-
*/
|
|
1088
|
-
declare class BitfabClaudeAgentHandler {
|
|
1089
|
-
private readonly httpClient;
|
|
1090
|
-
private readonly ownsHttpClient;
|
|
1091
|
-
private readonly traceFunctionKey;
|
|
1092
|
-
private readonly getActiveSpanContext;
|
|
1093
|
-
private readonly shouldRecord;
|
|
1094
|
-
private runToSpan;
|
|
1095
|
-
private traceId;
|
|
1096
|
-
private rootSpanId;
|
|
1097
|
-
private activeContext;
|
|
1098
|
-
private traceStartedAt;
|
|
1099
|
-
private conversationHistory;
|
|
1100
|
-
private pendingMessages;
|
|
1101
|
-
private currentLlmSpanId;
|
|
1102
|
-
private currentLlmMessageId;
|
|
1103
|
-
private currentLlmContent;
|
|
1104
|
-
private currentLlmModel;
|
|
1105
|
-
private currentLlmUsage;
|
|
1106
|
-
private currentLlmStartedAt;
|
|
1107
|
-
private currentLlmHistorySnapshot;
|
|
1108
|
-
private activeSubagentSpans;
|
|
1109
|
-
private hasRootInput;
|
|
1110
|
-
private rootInput;
|
|
1111
|
-
private rootOutput;
|
|
1112
|
-
constructor(config: {
|
|
1113
|
-
apiKey?: ApiKeyInput;
|
|
1114
|
-
traceFunctionKey: string;
|
|
1115
|
-
serviceUrl?: string;
|
|
1116
|
-
timeout?: number;
|
|
1117
|
-
getActiveSpanContext?: () => ActiveSpanContext$2 | null;
|
|
1118
|
-
shouldRecord?: () => boolean;
|
|
1119
|
-
/**
|
|
1120
|
-
* The owning `Bitfab` client's HTTP client. Supplied by
|
|
1121
|
-
* `getClaudeAgentHandler()` so this handler shares that client's single
|
|
1122
|
-
* span-transport worker instead of starting a second one.
|
|
1123
|
-
* @internal
|
|
1124
|
-
*/
|
|
1125
|
-
_httpClient?: HttpClient;
|
|
1126
|
-
});
|
|
1127
|
-
/**
|
|
1128
|
-
* Flush and release the span transport this handler started. A no-op when
|
|
1129
|
-
* the handler borrowed a `Bitfab` client's HTTP client: that client's
|
|
1130
|
-
* `close()` owns the worker's lifetime.
|
|
1131
|
-
*/
|
|
1132
|
-
close(timeoutMs?: number): Promise<boolean>;
|
|
1133
|
-
private ensureTrace;
|
|
1134
|
-
private getParentId;
|
|
1135
|
-
private maybeStartRootSpan;
|
|
1136
|
-
private completeRootSpan;
|
|
1137
|
-
private startSpan;
|
|
1138
|
-
private completeSpan;
|
|
1139
|
-
private sendSpan;
|
|
1140
|
-
private sendTraceCompletion;
|
|
1141
|
-
private preToolUseHook;
|
|
1142
|
-
private postToolUseHook;
|
|
1143
|
-
private postToolUseFailureHook;
|
|
1144
|
-
private subagentStartHook;
|
|
1145
|
-
private subagentStopHook;
|
|
1146
|
-
/**
|
|
1147
|
-
* Inject Bitfab tracing hooks into Claude Agent SDK options.
|
|
1148
|
-
*
|
|
1149
|
-
* Modifies the options object and returns it for convenience.
|
|
1150
|
-
* The SDK's `HookMatcher` is constructed as a plain object
|
|
1151
|
-
* (`{ matcher: null, hooks: [callback] }`) to avoid requiring
|
|
1152
|
-
* `@anthropic-ai/claude-agent-sdk` as a dependency.
|
|
1153
|
-
*
|
|
1154
|
-
* @param options - Options object with a `hooks` property
|
|
1155
|
-
* @returns The modified options object with Bitfab hooks injected
|
|
1156
|
-
*/
|
|
1157
|
-
instrumentOptions<T extends Record<string, unknown>>(options: T): T;
|
|
1158
|
-
/**
|
|
1159
|
-
* Wrap any Claude Agent SDK message stream to capture LLM turns.
|
|
1160
|
-
*
|
|
1161
|
-
* Yields every message unchanged while capturing assistant message
|
|
1162
|
-
* content as LLM turn spans. Kept for naming symmetry with the Python
|
|
1163
|
-
* SDK's `wrapResponse` (which wraps `ClaudeSDKClient.receiveResponse()`);
|
|
1164
|
-
* in TypeScript, prefer `wrapQuery` around `query()`.
|
|
1165
|
-
*
|
|
1166
|
-
* Pass `{ input }` (the prompt) to record a replayable root span - see
|
|
1167
|
-
* `wrapQuery`.
|
|
1168
|
-
*/
|
|
1169
|
-
wrapResponse(stream: AsyncIterable<unknown>, opts?: {
|
|
1170
|
-
input?: unknown;
|
|
1171
|
-
}): AsyncIterable<unknown>;
|
|
1172
|
-
/**
|
|
1173
|
-
* Wrap a `query()` async iterator to capture LLM turns.
|
|
1174
|
-
*
|
|
1175
|
-
* Tool and subagent spans are captured separately via the hooks injected
|
|
1176
|
-
* by `instrumentOptions` into the `options` passed to `query()`.
|
|
1177
|
-
*
|
|
1178
|
-
* Pass `{ input }` - the prompt (or the serializable args that produced it)
|
|
1179
|
-
* - to make a handler-only run replayable: the handler records a root `agent`
|
|
1180
|
-
* span with that input, so `replay(key, fn)` can re-feed it. Omit it only
|
|
1181
|
-
* when an enclosing `withSpan` already supplies the replayable root.
|
|
1182
|
-
*
|
|
1183
|
-
* ```typescript
|
|
1184
|
-
* handler.wrapQuery(query({ prompt, options }), { input: prompt })
|
|
1185
|
-
* ```
|
|
1186
|
-
*/
|
|
1187
|
-
wrapQuery(stream: AsyncIterable<unknown>, opts?: {
|
|
1188
|
-
input?: unknown;
|
|
1189
|
-
}): AsyncIterable<unknown>;
|
|
1190
|
-
private setRootInput;
|
|
1191
|
-
private processStream;
|
|
1192
|
-
private processMessage;
|
|
1193
|
-
private handleAssistantMessage;
|
|
1194
|
-
private handleUserMessage;
|
|
1195
|
-
private handleResultMessage;
|
|
1196
|
-
private flushLlmTurn;
|
|
1197
|
-
private resetState;
|
|
1198
|
-
}
|
|
1199
|
-
|
|
1200
|
-
/**
|
|
1201
|
-
* Vercel AI SDK integration for Bitfab tracing.
|
|
1202
|
-
*
|
|
1203
|
-
* The Vercel AI SDK (`ai`) routes every `generateText` / `streamText` /
|
|
1204
|
-
* `generateObject` / `streamObject` call through a language model. Bitfab hooks
|
|
1205
|
-
* that model with a *language-model middleware* (`wrapLanguageModel`), so each
|
|
1206
|
-
* model call is captured as a keyed `llm` span with no hand-written `withSpan`:
|
|
1207
|
-
*
|
|
1208
|
-
* ```typescript
|
|
1209
|
-
* import { Bitfab } from "@bitfab/sdk";
|
|
1210
|
-
* import { wrapLanguageModel, streamText } from "ai";
|
|
1211
|
-
* import { openai } from "@ai-sdk/openai";
|
|
1212
|
-
*
|
|
1213
|
-
* const bitfab = new Bitfab({ apiKey: "..." });
|
|
1214
|
-
*
|
|
1215
|
-
* const model = wrapLanguageModel({
|
|
1216
|
-
* model: openai("gpt-4o"),
|
|
1217
|
-
* middleware: bitfab.getVercelAiMiddleware("chat-turn"),
|
|
1218
|
-
* });
|
|
1219
|
-
*
|
|
1220
|
-
* const result = streamText({ model, messages });
|
|
1221
|
-
* return result.toUIMessageStreamResponse(); // live stream untouched
|
|
1222
|
-
* ```
|
|
1223
|
-
*
|
|
1224
|
-
* The span records the call parameters (the prompt/messages) as its input and a
|
|
1225
|
-
* serializable summary (`{ text, toolCalls, usage, finishReason }`) as its
|
|
1226
|
-
* output. Streaming is handled by passing the model's stream through a
|
|
1227
|
-
* reader that accumulates text and usage as the caller consumes it. Parts,
|
|
1228
|
-
* upstream errors, and cancellation propagate to the caller unchanged; the
|
|
1229
|
-
* span finalizes on completion, error, or cancellation with partial output.
|
|
1230
|
-
*
|
|
1231
|
-
* The middleware is fully duck-typed (no static or dynamic `ai` import), so it
|
|
1232
|
-
* adds no dependency and is browser-safe. It works with `ai` v5 and v6.
|
|
1233
|
-
*/
|
|
1234
|
-
type LlmSpanOptions = {
|
|
1235
|
-
type: "llm";
|
|
1236
|
-
finalize: (result: unknown) => unknown | Promise<unknown>;
|
|
1237
|
-
surface: "inherit";
|
|
1238
|
-
instrumentation: "vercel-ai";
|
|
1239
|
-
};
|
|
1240
|
-
type WithSpanFn$1 = <TArgs extends unknown[], TReturn>(traceFunctionKey: string, options: LlmSpanOptions, fn: (...args: TArgs) => TReturn) => (...args: TArgs) => TReturn;
|
|
1241
|
-
/**
|
|
1242
|
-
* Duck-typed subset of the Vercel AI SDK language-model call parameters. The
|
|
1243
|
-
* only field we read for the span input is `prompt` (the messages), but the
|
|
1244
|
-
* whole object is recorded so the call is reconstructable on replay.
|
|
1245
|
-
*/
|
|
1246
|
-
interface VercelCallParams {
|
|
1247
|
-
prompt?: unknown;
|
|
1248
|
-
[key: string]: unknown;
|
|
1249
|
-
}
|
|
1250
|
-
/** A content part of a non-streaming `doGenerate` result. */
|
|
1251
|
-
interface VercelContentPart {
|
|
1252
|
-
type: string;
|
|
1253
|
-
text?: string;
|
|
1254
|
-
toolCallId?: string;
|
|
1255
|
-
toolName?: string;
|
|
1256
|
-
input?: unknown;
|
|
1257
|
-
args?: unknown;
|
|
1258
|
-
}
|
|
1259
|
-
/** Duck-typed subset of a non-streaming `doGenerate` result. */
|
|
1260
|
-
interface VercelGenerateResult {
|
|
1261
|
-
content?: VercelContentPart[];
|
|
1262
|
-
text?: string;
|
|
1263
|
-
usage?: unknown;
|
|
1264
|
-
finishReason?: unknown;
|
|
1265
|
-
[key: string]: unknown;
|
|
1266
|
-
}
|
|
1267
|
-
/** Duck-typed subset of a single streaming part from `doStream`. */
|
|
1268
|
-
interface VercelStreamPart {
|
|
1269
|
-
type: string;
|
|
1270
|
-
error?: unknown;
|
|
1271
|
-
delta?: string;
|
|
1272
|
-
textDelta?: string;
|
|
1273
|
-
toolCallId?: string;
|
|
1274
|
-
toolName?: string;
|
|
1275
|
-
input?: unknown;
|
|
1276
|
-
args?: unknown;
|
|
1277
|
-
usage?: unknown;
|
|
1278
|
-
finishReason?: unknown;
|
|
1279
|
-
}
|
|
1280
|
-
/** Duck-typed subset of a streaming `doStream` result. */
|
|
1281
|
-
interface VercelStreamResult {
|
|
1282
|
-
stream: ReadableStream<VercelStreamPart>;
|
|
1283
|
-
[key: string]: unknown;
|
|
1284
|
-
}
|
|
1285
|
-
interface MiddlewareCall<TResult> {
|
|
1286
|
-
doGenerate: () => PromiseLike<VercelGenerateResult>;
|
|
1287
|
-
doStream: () => PromiseLike<VercelStreamResult>;
|
|
1288
|
-
params: VercelCallParams;
|
|
1289
|
-
model: unknown;
|
|
1290
|
-
__result?: TResult;
|
|
1291
|
-
}
|
|
1292
|
-
/**
|
|
1293
|
-
* The structural shape of a Vercel AI SDK language-model middleware. Matches
|
|
1294
|
-
* `LanguageModelV3Middleware` from `@ai-sdk/provider` without importing it, so
|
|
1295
|
-
* the SDK stays dependency-free. `wrapLanguageModel` only reads the method
|
|
1296
|
-
* fields (it ignores `specificationVersion`), so this object drops straight in.
|
|
1297
|
-
*/
|
|
1298
|
-
interface BitfabLanguageModelMiddleware {
|
|
1299
|
-
specificationVersion: "v3";
|
|
1300
|
-
wrapGenerate: (options: MiddlewareCall<VercelGenerateResult>) => Promise<any>;
|
|
1301
|
-
wrapStream: (options: MiddlewareCall<VercelStreamResult>) => Promise<any>;
|
|
1302
|
-
}
|
|
1303
|
-
/**
|
|
1304
|
-
* Vercel AI SDK middleware that records each language-model call as a keyed
|
|
1305
|
-
* `llm` span. Obtain it from {@link BitfabClient.getVercelAiMiddleware} rather
|
|
1306
|
-
* than constructing it directly.
|
|
1307
|
-
*/
|
|
1308
|
-
declare class BitfabVercelAiHandler {
|
|
1309
|
-
private readonly traceFunctionKey;
|
|
1310
|
-
private readonly withSpanFn;
|
|
1311
|
-
constructor(config: {
|
|
1312
|
-
traceFunctionKey: string;
|
|
1313
|
-
withSpan: WithSpanFn$1;
|
|
1314
|
-
});
|
|
1315
|
-
/** The `wrapLanguageModel` middleware object for this trace function key. */
|
|
1316
|
-
get middleware(): BitfabLanguageModelMiddleware;
|
|
1317
|
-
}
|
|
1318
|
-
|
|
1319
|
-
interface DatasetGraderRef {
|
|
1320
|
-
id: string;
|
|
1321
|
-
name: string | null;
|
|
1322
|
-
}
|
|
1323
|
-
interface Dataset {
|
|
1324
|
-
id: string;
|
|
1325
|
-
traceFunctionKey: string;
|
|
1326
|
-
name: string;
|
|
1327
|
-
description: string | null;
|
|
1328
|
-
traceCount: number;
|
|
1329
|
-
graders: DatasetGraderRef[];
|
|
1330
|
-
/**
|
|
1331
|
-
* Whether the dataset is marked golden for the whole organization, as a
|
|
1332
|
-
* trusted reference set. Golden datasets are listed first in the Bitfab app.
|
|
1333
|
-
* Reported as `false` by servers that predate golden datasets.
|
|
1334
|
-
*/
|
|
1335
|
-
golden: boolean;
|
|
1336
|
-
createdAt: string;
|
|
1337
|
-
updatedAt: string;
|
|
1338
|
-
}
|
|
1339
|
-
interface SaveDatasetParams {
|
|
1340
|
-
traceFunctionKey: string;
|
|
1341
|
-
name: string;
|
|
1342
|
-
description?: string;
|
|
1343
|
-
/**
|
|
1344
|
-
* `true` marks the dataset as golden for the whole organization, `false`
|
|
1345
|
-
* unmarks it, and omitting it leaves it as it is (a new dataset starts not
|
|
1346
|
-
* golden).
|
|
1347
|
-
*/
|
|
1348
|
-
golden?: boolean;
|
|
1349
|
-
}
|
|
1350
|
-
interface UpdateDatasetParams {
|
|
1351
|
-
name?: string;
|
|
1352
|
-
description?: string;
|
|
1353
|
-
/**
|
|
1354
|
-
* `true` marks the dataset as golden for the whole organization, `false`
|
|
1355
|
-
* unmarks it, and omitting it leaves it as it is.
|
|
1356
|
-
*/
|
|
1357
|
-
golden?: boolean;
|
|
1358
|
-
}
|
|
1359
|
-
interface SaveDatasetResult {
|
|
1360
|
-
dataset: Dataset;
|
|
1361
|
-
created: boolean;
|
|
1362
|
-
}
|
|
1363
|
-
interface ListDatasetsParams {
|
|
1364
|
-
traceFunctionKey?: string;
|
|
1365
|
-
}
|
|
1366
|
-
interface DatasetTraceIds {
|
|
1367
|
-
datasetId: string;
|
|
1368
|
-
traceIds: string[];
|
|
1369
|
-
}
|
|
1370
|
-
interface AddDatasetTracesResult {
|
|
1371
|
-
dataset: Dataset;
|
|
1372
|
-
addedTraceIds: string[];
|
|
1373
|
-
alreadyPresentTraceIds: string[];
|
|
1374
|
-
skippedTraceIds: string[];
|
|
1375
|
-
}
|
|
1376
|
-
interface RemoveDatasetTracesResult {
|
|
1377
|
-
dataset: Dataset;
|
|
1378
|
-
removedTraceIds: string[];
|
|
1379
|
-
notPresentTraceIds: string[];
|
|
1380
|
-
}
|
|
1381
|
-
interface AddDatasetGradersResult {
|
|
1382
|
-
dataset: Dataset;
|
|
1383
|
-
addedGraderIds: string[];
|
|
1384
|
-
alreadyAssignedGraderIds: string[];
|
|
1385
|
-
skippedGraderIds: string[];
|
|
1386
|
-
}
|
|
1387
|
-
interface RemoveDatasetGradersResult {
|
|
1388
|
-
dataset: Dataset;
|
|
1389
|
-
removedGraderIds: string[];
|
|
1390
|
-
notAssignedGraderIds: string[];
|
|
1391
|
-
}
|
|
1392
|
-
type GraderRerunStatus = "pending" | "running" | "completed" | "errored";
|
|
1393
|
-
interface GraderRerunProgress {
|
|
1394
|
-
completedTraces: number;
|
|
1395
|
-
totalTraces: number;
|
|
1396
|
-
graderCount: number;
|
|
1397
|
-
}
|
|
1398
|
-
interface GraderRerunResult {
|
|
1399
|
-
tracesGraded: number;
|
|
1400
|
-
gradersRun: number;
|
|
1401
|
-
}
|
|
1402
|
-
interface GraderRerun {
|
|
1403
|
-
id: string;
|
|
1404
|
-
status: GraderRerunStatus;
|
|
1405
|
-
graderIds: string[];
|
|
1406
|
-
progress: GraderRerunProgress | null;
|
|
1407
|
-
result: GraderRerunResult | null;
|
|
1408
|
-
error: string | null;
|
|
1409
|
-
createdAt: string;
|
|
1410
|
-
updatedAt: string;
|
|
1411
|
-
}
|
|
1412
|
-
interface RerunGradersOptions {
|
|
1413
|
-
graderIds?: string[];
|
|
1414
|
-
timeoutMs?: number;
|
|
1415
|
-
}
|
|
1416
|
-
interface RerunGradersResult {
|
|
1417
|
-
run: GraderRerun;
|
|
1418
|
-
joinedExisting: boolean;
|
|
1419
|
-
}
|
|
1420
|
-
declare class GraderRerunError extends Error {
|
|
1421
|
-
readonly run: GraderRerun;
|
|
1422
|
-
constructor(message: string, run: GraderRerun);
|
|
1423
|
-
}
|
|
1424
|
-
declare class GraderRerunTimeoutError extends GraderRerunError {
|
|
1425
|
-
constructor(datasetId: string, run: GraderRerun, timeoutMs: number);
|
|
1426
|
-
}
|
|
1427
|
-
/**
|
|
1428
|
-
* Dataset operations for the authenticated organization, reached as
|
|
1429
|
-
* `client.datasets`. A dataset is a named bucket of traces scoped to one trace
|
|
1430
|
-
* function. Experiments replay against it and its graders score its members.
|
|
1431
|
-
*/
|
|
1432
|
-
declare class DatasetsClient {
|
|
1433
|
-
private readonly httpClient;
|
|
1434
|
-
constructor(httpClient: HttpClient);
|
|
1435
|
-
/**
|
|
1436
|
-
* Create a dataset, or update the one already named this way under the same
|
|
1437
|
-
* trace function. `created` reports which happened. An omitted description
|
|
1438
|
-
* leaves an existing one untouched.
|
|
1439
|
-
*/
|
|
1440
|
-
save(params: SaveDatasetParams): Promise<SaveDatasetResult>;
|
|
1441
|
-
/**
|
|
1442
|
-
* Rename a dataset or change its description in place. The dataset keeps its
|
|
1443
|
-
* id, traces, and graders. An empty `description` clears it. Rejects with a
|
|
1444
|
-
* 400 `BitfabError` when the new name is already used by another dataset on
|
|
1445
|
-
* the same trace function, and a 404 when the dataset is not in this
|
|
1446
|
-
* organization.
|
|
1447
|
-
*/
|
|
1448
|
-
update(datasetId: string, params: UpdateDatasetParams): Promise<Dataset>;
|
|
1449
|
-
/**
|
|
1450
|
-
* List datasets, scoped to one trace function when `traceFunctionKey` is
|
|
1451
|
-
* given and organization-wide otherwise. Fetches all pages automatically.
|
|
1452
|
-
*/
|
|
1453
|
-
list(params?: ListDatasetsParams): Promise<Dataset[]>;
|
|
1454
|
-
/** Fetch one dataset by id. Rejects with a 404 `BitfabError` when it is not in this organization. */
|
|
1455
|
-
get(datasetId: string): Promise<Dataset>;
|
|
1456
|
-
/** Fetches all trace-ID pages, the same membership a replay with `datasetId` selects. */
|
|
1457
|
-
listTraces(datasetId: string): Promise<DatasetTraceIds>;
|
|
1458
|
-
/**
|
|
1459
|
-
* Add traces to the dataset (1 to 100 ids per call). Traces outside the
|
|
1460
|
-
* organization or under another trace function are reported in
|
|
1461
|
-
* `skippedTraceIds` rather than failing the call.
|
|
1462
|
-
*/
|
|
1463
|
-
addTraces(datasetId: string, traceIds: string[]): Promise<AddDatasetTracesResult>;
|
|
1464
|
-
/** Remove traces from the dataset. The traces themselves are never deleted. */
|
|
1465
|
-
removeTraces(datasetId: string, traceIds: string[]): Promise<RemoveDatasetTracesResult>;
|
|
1466
|
-
/**
|
|
1467
|
-
* Assign graders to the dataset (1 to 100 ids per call). Graders outside the
|
|
1468
|
-
* organization or under another trace function are reported in
|
|
1469
|
-
* `skippedGraderIds` rather than failing the call.
|
|
1470
|
-
*/
|
|
1471
|
-
addGraders(datasetId: string, graderIds: string[]): Promise<AddDatasetGradersResult>;
|
|
1472
|
-
/** Unassign graders from the dataset. */
|
|
1473
|
-
removeGraders(datasetId: string, graderIds: string[]): Promise<RemoveDatasetGradersResult>;
|
|
1474
|
-
rerunGraders(datasetId: string, options?: RerunGradersOptions): Promise<RerunGradersResult>;
|
|
1475
|
-
/**
|
|
1476
|
-
* The dataset's active grader re-run, or the run named by `runId`. Returns
|
|
1477
|
-
* `null` when nothing is active or the run does not belong to this dataset.
|
|
1478
|
-
*/
|
|
1479
|
-
getGraderRerun(datasetId: string, runId?: string): Promise<GraderRerun | null>;
|
|
1480
|
-
}
|
|
1481
|
-
|
|
1482
|
-
type ExperimentStatus = "pending" | "completed" | "failed" | "interrupted";
|
|
1483
|
-
type ExperimentRunBy = {
|
|
1484
|
-
id: string;
|
|
1485
|
-
fullName: string | null;
|
|
1486
|
-
email: string | null;
|
|
1487
|
-
imageUrl: string | null;
|
|
1488
|
-
};
|
|
1489
|
-
type ExperimentCodeChangeTotals = {
|
|
1490
|
-
files: number;
|
|
1491
|
-
added: number;
|
|
1492
|
-
removed: number;
|
|
1493
|
-
};
|
|
1494
|
-
type Experiment = {
|
|
1495
|
-
id: string;
|
|
1496
|
-
name: string | null;
|
|
1497
|
-
notes: string | null;
|
|
1498
|
-
status: ExperimentStatus;
|
|
1499
|
-
traceFunctionKey: string | null;
|
|
1500
|
-
createdAt: string;
|
|
1501
|
-
completedAt: string | null;
|
|
1502
|
-
codeChangeDescription: string | null;
|
|
1503
|
-
codeChangeTotals: ExperimentCodeChangeTotals | null;
|
|
1504
|
-
experimentGroupId: string | null;
|
|
1505
|
-
experimentGroupName: string | null;
|
|
1506
|
-
datasetId: string | null;
|
|
1507
|
-
datasetName: string | null;
|
|
1508
|
-
datasetIds: string[];
|
|
1509
|
-
datasetNames: string[];
|
|
1510
|
-
attempts: number;
|
|
1511
|
-
plannedTraceCount: number | null;
|
|
1512
|
-
runBy: ExperimentRunBy | null;
|
|
1513
|
-
githubEmail: string | null;
|
|
1514
|
-
gitBranch: string | null;
|
|
1515
|
-
commitSha: string | null;
|
|
1516
|
-
baseSha: string | null;
|
|
1517
|
-
experimentSha: string | null;
|
|
1518
|
-
metadata: Record<string, string>;
|
|
1519
|
-
};
|
|
1520
|
-
type ExperimentPage = {
|
|
1521
|
-
experiments: Experiment[];
|
|
1522
|
-
nextCursor: string | null;
|
|
1523
|
-
hasMore: boolean;
|
|
1524
|
-
};
|
|
1525
|
-
type ListExperimentsParams = {
|
|
1526
|
-
datasetId?: string;
|
|
1527
|
-
traceFunctionKey?: string;
|
|
1528
|
-
gitBranch?: string;
|
|
1529
|
-
experimentGroupId?: string;
|
|
1530
|
-
status?: ExperimentStatus;
|
|
1531
|
-
metadata?: Record<string, string>;
|
|
1532
|
-
createdAfter?: Date | string;
|
|
1533
|
-
createdBefore?: Date | string;
|
|
1534
|
-
cursor?: string;
|
|
1535
|
-
limit?: number;
|
|
1536
|
-
};
|
|
1537
|
-
type ExperimentTally = {
|
|
1538
|
-
fixed: number;
|
|
1539
|
-
stillPassing: number;
|
|
1540
|
-
regressed: number;
|
|
1541
|
-
stillFailing: number;
|
|
1542
|
-
originalUnlabeled: number;
|
|
1543
|
-
unpaired: number;
|
|
1544
|
-
classified: number;
|
|
1545
|
-
passing: number;
|
|
1546
|
-
failing: number;
|
|
1547
|
-
/** Failing verdicts where the judge abstained on every attempt because it could not find the evidence. */
|
|
1548
|
-
abstained: number;
|
|
1549
|
-
};
|
|
1550
|
-
type ExperimentTotals = {
|
|
1551
|
-
total: number;
|
|
1552
|
-
traces: number;
|
|
1553
|
-
succeeded: number;
|
|
1554
|
-
failed: number;
|
|
1555
|
-
pending: number;
|
|
1556
|
-
awaitingLabels: number;
|
|
1557
|
-
errored: number;
|
|
1558
|
-
withErrors: number;
|
|
1559
|
-
ungradable: number;
|
|
1560
|
-
judging: number;
|
|
1561
|
-
skipped: number;
|
|
1562
|
-
};
|
|
1563
|
-
type ExperimentCategoryTally = {
|
|
1564
|
-
category: CategorySummary;
|
|
1565
|
-
labels: ExperimentTally;
|
|
1566
|
-
};
|
|
1567
|
-
type ExperimentJitter = {
|
|
1568
|
-
disagreeing: number;
|
|
1569
|
-
attempts: number;
|
|
1570
|
-
traces: number;
|
|
1571
|
-
};
|
|
1572
|
-
type ExperimentRollup = {
|
|
1573
|
-
traces: ExperimentTally;
|
|
1574
|
-
skippedTraces: number;
|
|
1575
|
-
labels: ExperimentTally;
|
|
1576
|
-
checks: ExperimentTally;
|
|
1577
|
-
categories: ExperimentCategoryTally[];
|
|
1578
|
-
jitter: ExperimentJitter;
|
|
1579
|
-
};
|
|
1580
|
-
type ExperimentRollupResult = {
|
|
1581
|
-
experimentId: string;
|
|
1582
|
-
totals: ExperimentTotals;
|
|
1583
|
-
rollup: ExperimentRollup;
|
|
1584
|
-
};
|
|
1585
|
-
declare class ExperimentsClient {
|
|
1586
|
-
private readonly httpClient;
|
|
1587
|
-
constructor(httpClient: HttpClient);
|
|
1588
|
-
list(params?: ListExperimentsParams): Promise<ExperimentPage>;
|
|
1589
|
-
get(id: string): Promise<Experiment>;
|
|
1590
|
-
getRollup(id: string): Promise<ExperimentRollupResult>;
|
|
1591
|
-
getRollupAll(ids: string[]): Promise<ExperimentRollupResult[]>;
|
|
1592
|
-
}
|
|
1593
|
-
|
|
1594
|
-
type TraceTargetOccurrence = "first" | "last" | number;
|
|
1595
|
-
type TraceTarget = {
|
|
1596
|
-
kind: "output";
|
|
1597
|
-
} | {
|
|
1598
|
-
kind: "span";
|
|
1599
|
-
name: string;
|
|
1600
|
-
occurrence?: TraceTargetOccurrence;
|
|
1601
|
-
};
|
|
1602
|
-
type TraceAssertionSource = "human" | "agent";
|
|
1603
|
-
type TraceSearchFilters = {
|
|
1604
|
-
callerMetadata?: Record<string, string>;
|
|
1605
|
-
name?: string;
|
|
1606
|
-
nameContains?: string;
|
|
1607
|
-
/** Only the replay traces this experiment produced. */
|
|
1608
|
-
experimentId?: string;
|
|
1609
|
-
};
|
|
1610
|
-
type TraceSearchParams = TraceSearchFilters & ({
|
|
1611
|
-
callerMetadata: Record<string, string>;
|
|
1612
|
-
} | {
|
|
1613
|
-
name: string;
|
|
1614
|
-
} | {
|
|
1615
|
-
nameContains: string;
|
|
1616
|
-
} | {
|
|
1617
|
-
experimentId: string;
|
|
1618
|
-
}) & {
|
|
1619
|
-
traceFunctionKey?: string;
|
|
1620
|
-
cursor?: string | null;
|
|
1621
|
-
limit?: number;
|
|
1622
|
-
};
|
|
1623
|
-
type TraceSearchResult = {
|
|
1624
|
-
traces: Array<{
|
|
1625
|
-
id: string;
|
|
1626
|
-
traceFunctionKey: string | null;
|
|
1627
|
-
name: string | null;
|
|
1628
|
-
status: "pending" | "completed" | "failed" | "dropped";
|
|
1629
|
-
createdAt: string;
|
|
1630
|
-
callerMetadata: Record<string, string>;
|
|
1631
|
-
/** The experiment that produced this trace, or null for a captured trace. */
|
|
1632
|
-
experimentId: string | null;
|
|
1633
|
-
/** Wall-clock time of the whole trace, or null when it has no end time. */
|
|
1634
|
-
durationMs: number | null;
|
|
1635
|
-
/** Token usage summed across the trace, or null when none was recorded. */
|
|
1636
|
-
tokens: TokenUsage | null;
|
|
1637
|
-
}>;
|
|
1638
|
-
nextCursor: string | null;
|
|
1639
|
-
hasMore: boolean;
|
|
1640
|
-
};
|
|
1641
|
-
type TraceAssertion = {
|
|
1642
|
-
id: string;
|
|
1643
|
-
traceId: string;
|
|
1644
|
-
assertion: string;
|
|
1645
|
-
humanNote: string | null;
|
|
1646
|
-
category_id: string | null;
|
|
1647
|
-
/** @deprecated Use {@link category_id}. Same value. */
|
|
1648
|
-
category_assertion_id: string | null;
|
|
1649
|
-
category: CategorySummary | null;
|
|
1650
|
-
passCriteria: string | null;
|
|
1651
|
-
failCriteria: string | null;
|
|
1652
|
-
targetOnEvaluatedTrace: TraceTarget | null;
|
|
1653
|
-
source: TraceAssertionSource;
|
|
1654
|
-
/** Why this assertion is right for its trace, span by span. */
|
|
1655
|
-
justification: Justification | null;
|
|
1656
|
-
/** Why its assigned category is the right one, span by span. */
|
|
1657
|
-
categoryJustification: Justification | null;
|
|
1658
|
-
/** Who is on the hook for reviewing this assertion, or null for nobody. */
|
|
1659
|
-
assignee: Assignee | null;
|
|
1660
|
-
createdAt: string;
|
|
1661
|
-
updatedAt: string;
|
|
1662
|
-
} & ApprovalFields;
|
|
1663
|
-
type AssertionEvidenceParameter = {
|
|
1664
|
-
name: string;
|
|
1665
|
-
type: "string" | "number" | "boolean" | "object" | "array" | "null" | "unknown";
|
|
1666
|
-
};
|
|
1667
|
-
type AssertionLabelEvidence = {
|
|
1668
|
-
spanId: string;
|
|
1669
|
-
text: string;
|
|
1670
|
-
sourceField: string;
|
|
1671
|
-
spanName: string | null;
|
|
1672
|
-
parameters: AssertionEvidenceParameter[];
|
|
1673
|
-
spanType: "llm" | "agent" | "function" | "guardrail" | "handoff" | "custom";
|
|
1674
|
-
isMocked: boolean;
|
|
1675
|
-
};
|
|
1676
|
-
type SaveAssertion = {
|
|
1677
|
-
/** Omit to preserve the category, or pass null to clear it. */
|
|
1678
|
-
category_id?: string | null;
|
|
1679
|
-
/** @deprecated Use {@link category_id}. Ignored when `category_id` is also set. */
|
|
1680
|
-
category_assertion_id?: string | null;
|
|
1681
|
-
id?: string;
|
|
1682
|
-
assertion: string;
|
|
1683
|
-
humanNote?: string | null;
|
|
1684
|
-
passCriteria?: string | null;
|
|
1685
|
-
failCriteria?: string | null;
|
|
1686
|
-
targetOnEvaluatedTrace?: TraceTarget | null;
|
|
1687
|
-
/**
|
|
1688
|
-
* Why this assertion is right for this trace, one entry per span it rests on.
|
|
1689
|
-
* Cite spans on this trace only. Omit on an edit to preserve it.
|
|
1690
|
-
*/
|
|
1691
|
-
justification?: Justification | null;
|
|
1692
|
-
/**
|
|
1693
|
-
* Why `category_id` is the right category. Changing the category
|
|
1694
|
-
* clears it. Omit on an edit to preserve it.
|
|
1695
|
-
*/
|
|
1696
|
-
categoryJustification?: Justification | null;
|
|
1697
|
-
/**
|
|
1698
|
-
* Email of the organization member who should review this assertion. Routing
|
|
1699
|
-
* only, so it leaves the approval state alone. Omit on an edit to preserve
|
|
1700
|
-
* the current assignee, or pass null to leave it to nobody.
|
|
1701
|
-
*/
|
|
1702
|
-
assigneeEmail?: string | null;
|
|
1703
|
-
};
|
|
1704
|
-
type ScenarioSource = {
|
|
1705
|
-
spanId: string;
|
|
1706
|
-
sourceField: "input" | "output";
|
|
1707
|
-
text: string;
|
|
1708
|
-
};
|
|
1709
|
-
type ScenarioTrigger = {
|
|
1710
|
-
inputId: number | null;
|
|
1711
|
-
inputType: string;
|
|
1712
|
-
channel: string;
|
|
1713
|
-
from: string;
|
|
1714
|
-
to: string | null;
|
|
1715
|
-
cc: string | null;
|
|
1716
|
-
subject: string | null;
|
|
1717
|
-
type: string;
|
|
1718
|
-
eventTime: string;
|
|
1719
|
-
eventTimeSource: "message date" | "run start";
|
|
1720
|
-
source: "request" | "run instructions";
|
|
1721
|
-
content: string;
|
|
1722
|
-
contentPassages: string[];
|
|
1723
|
-
contentIsWholeMessage: boolean;
|
|
1724
|
-
sources: ScenarioSource[];
|
|
1725
|
-
};
|
|
1726
|
-
type ScenarioDataOrigin = {
|
|
1727
|
-
section: string;
|
|
1728
|
-
channel: string | null;
|
|
1729
|
-
from: string | null;
|
|
1730
|
-
to: string | null;
|
|
1731
|
-
cc: string | null;
|
|
1732
|
-
subject: string | null;
|
|
1733
|
-
sentAtText: string | null;
|
|
1734
|
-
};
|
|
1735
|
-
type ScenarioHistoryEntry = {
|
|
1736
|
-
role: "thread start" | "fact";
|
|
1737
|
-
kind: string;
|
|
1738
|
-
quote: string;
|
|
1739
|
-
summary: string | null;
|
|
1740
|
-
origin: ScenarioDataOrigin | null;
|
|
1741
|
-
sentAt: string | null;
|
|
1742
|
-
weekday: string | null;
|
|
1743
|
-
sincePrevious: string | null;
|
|
1744
|
-
source: ScenarioSource | null;
|
|
1745
|
-
};
|
|
1746
|
-
type ScenarioData = {
|
|
1747
|
-
hasHistory: boolean;
|
|
1748
|
-
topic: string;
|
|
1749
|
-
history: ScenarioHistoryEntry[];
|
|
1750
|
-
missing: string[];
|
|
1751
|
-
};
|
|
1752
|
-
/** One thing the run did, such as sending a message or calling a tool. */
|
|
1753
|
-
type ScenarioAction = {
|
|
1754
|
-
/** The action, for example `"send_sms_message"`. */
|
|
1755
|
-
name: string;
|
|
1756
|
-
/** The arguments the action was called with. */
|
|
1757
|
-
details: Record<string, unknown>;
|
|
1758
|
-
/** What came back from the action, or null when nothing did. */
|
|
1759
|
-
result: string | null;
|
|
1760
|
-
/** True when the action did not go through. */
|
|
1761
|
-
failed: boolean;
|
|
1762
|
-
/** When the action happened, as an ISO 8601 time, or null when unknown. */
|
|
1763
|
-
at: string | null;
|
|
1764
|
-
};
|
|
1765
|
-
/**
|
|
1766
|
-
* What the run actually did in this scenario, one example of a response. It
|
|
1767
|
-
* never says what the run should have done. Bitfab never fills it in. The
|
|
1768
|
-
* caller saves it with `saveScenario`.
|
|
1769
|
-
*/
|
|
1770
|
-
type ScenarioActualOutcome = {
|
|
1771
|
-
/** What the run did, in a few sentences. */
|
|
1772
|
-
summary: string;
|
|
1773
|
-
/** The actions the run took, in order. */
|
|
1774
|
-
actions: ScenarioAction[];
|
|
1775
|
-
/**
|
|
1776
|
-
* The span the outcome was read from, on a trace linked to this scenario,
|
|
1777
|
-
* or null when it was not read from one.
|
|
1778
|
-
*/
|
|
1779
|
-
source: ScenarioSource | null;
|
|
1780
|
-
};
|
|
1781
|
-
type Scenario = {
|
|
1782
|
-
id: string;
|
|
1783
|
-
createdAt: string;
|
|
1784
|
-
updatedAt: string;
|
|
1785
|
-
organizationId: string;
|
|
1786
|
-
traceFunctionId: string;
|
|
1787
|
-
sourceTraceId: string;
|
|
1788
|
-
extractor: "noah";
|
|
1789
|
-
name: string;
|
|
1790
|
-
description: string;
|
|
1791
|
-
trigger: ScenarioTrigger;
|
|
1792
|
-
data: ScenarioData;
|
|
1793
|
-
triggerSummary: string;
|
|
1794
|
-
dataSummary: string;
|
|
1795
|
-
/** What the run actually did, or null when none has been saved. */
|
|
1796
|
-
actualOutcome: ScenarioActualOutcome | null;
|
|
1797
|
-
/** Every category the scenario is filed under, sorted by title. */
|
|
1798
|
-
categories: CategorySummary[];
|
|
1799
|
-
};
|
|
1800
|
-
type SaveScenarioHistoryEntry = Omit<ScenarioHistoryEntry, "source"> & {
|
|
1801
|
-
source?: ScenarioSource | null;
|
|
1802
|
-
};
|
|
1803
|
-
type SaveScenarioData = Omit<ScenarioData, "history"> & {
|
|
1804
|
-
history: SaveScenarioHistoryEntry[];
|
|
1805
|
-
};
|
|
1806
|
-
/**
|
|
1807
|
-
* What the run actually did, as sent to `saveScenario`. `source` may be left
|
|
1808
|
-
* out, which saves it as null.
|
|
1809
|
-
*/
|
|
1810
|
-
type SaveScenarioActualOutcome = Omit<ScenarioActualOutcome, "source"> & {
|
|
1811
|
-
source?: ScenarioSource | null;
|
|
1812
|
-
};
|
|
1813
|
-
type ScenarioDataAppend = {
|
|
1814
|
-
history?: SaveScenarioHistoryEntry[];
|
|
1815
|
-
missing?: string[];
|
|
1816
|
-
};
|
|
1817
|
-
type SaveScenarioParams = {
|
|
1818
|
-
traceId: string;
|
|
1819
|
-
name?: string;
|
|
1820
|
-
description?: string;
|
|
1821
|
-
trigger?: ScenarioTrigger;
|
|
1822
|
-
/**
|
|
1823
|
-
* What the run actually did, never what it should have done. Replaces the
|
|
1824
|
-
* saved value whole. Pass null to clear it, or leave it out to keep it.
|
|
1825
|
-
* Sending only this field, null included, is a valid save.
|
|
1826
|
-
*/
|
|
1827
|
-
actualOutcome?: SaveScenarioActualOutcome | null;
|
|
1828
|
-
/**
|
|
1829
|
-
* Categories to file the scenario under. Replaces the whole set. Leave it
|
|
1830
|
-
* out to keep the current set, or pass an empty array to clear it. An id
|
|
1831
|
-
* outside the organization fails the save.
|
|
1832
|
-
*/
|
|
1833
|
-
categoryIds?: string[];
|
|
1834
|
-
} & ({
|
|
1835
|
-
data?: SaveScenarioData;
|
|
1836
|
-
appendData?: never;
|
|
1837
|
-
} | {
|
|
1838
|
-
appendData: ScenarioDataAppend;
|
|
1839
|
-
data?: never;
|
|
1840
|
-
}) & ({
|
|
1841
|
-
name: string;
|
|
1842
|
-
} | {
|
|
1843
|
-
description: string;
|
|
1844
|
-
} | {
|
|
1845
|
-
trigger: ScenarioTrigger;
|
|
1846
|
-
} | {
|
|
1847
|
-
data: SaveScenarioData;
|
|
1848
|
-
} | {
|
|
1849
|
-
appendData: ScenarioDataAppend;
|
|
1850
|
-
} | {
|
|
1851
|
-
actualOutcome: SaveScenarioActualOutcome | null;
|
|
1852
|
-
} | {
|
|
1853
|
-
categoryIds: string[];
|
|
1854
|
-
});
|
|
1855
|
-
type AssertionGenerationStatus = "running" | "completed" | "failed";
|
|
1856
|
-
type AssertionGeneration = {
|
|
1857
|
-
status: AssertionGenerationStatus;
|
|
1858
|
-
error: string | null;
|
|
1859
|
-
assertionIds: string[];
|
|
1860
|
-
startedAt: string;
|
|
1861
|
-
finishedAt: string | null;
|
|
1862
|
-
};
|
|
1863
|
-
type TraceAssertionsResult = {
|
|
1864
|
-
assertions: TraceAssertion[];
|
|
1865
|
-
inheritedFrom: string | null;
|
|
1866
|
-
generation: AssertionGeneration | null;
|
|
1867
|
-
};
|
|
1868
|
-
type TraceAssertionsUpdate = {
|
|
1869
|
-
traceId: string;
|
|
1870
|
-
assertions: SaveAssertion[];
|
|
1871
|
-
};
|
|
1872
|
-
type SaveAssertionsParams = {
|
|
1873
|
-
traceId: string;
|
|
1874
|
-
assertions: SaveAssertion[];
|
|
1875
|
-
source?: TraceAssertionSource;
|
|
1876
|
-
};
|
|
1877
|
-
type SaveAssertionsAllParams = {
|
|
1878
|
-
updates: TraceAssertionsUpdate[];
|
|
1879
|
-
source?: TraceAssertionSource;
|
|
1880
|
-
};
|
|
1881
|
-
type ArchiveAssertionsParams = {
|
|
1882
|
-
traceId: string;
|
|
1883
|
-
assertionIds: string[];
|
|
1884
|
-
};
|
|
1885
|
-
type GenerateAssertionsOptions = {
|
|
1886
|
-
guidance?: string;
|
|
1887
|
-
categoryIds?: string[];
|
|
1888
|
-
regenerate?: boolean;
|
|
1889
|
-
timeoutMs?: number;
|
|
1890
|
-
};
|
|
1891
|
-
declare class AssertionGenerationError extends Error {
|
|
1892
|
-
readonly traceId: string;
|
|
1893
|
-
constructor(message: string, traceId: string);
|
|
1894
|
-
}
|
|
1895
|
-
declare class AssertionGenerationTimeoutError extends AssertionGenerationError {
|
|
1896
|
-
constructor(traceId: string, timeoutMs: number);
|
|
1897
|
-
}
|
|
1898
|
-
declare class TracesClient {
|
|
1899
|
-
private readonly httpClient;
|
|
1900
|
-
constructor(httpClient: HttpClient);
|
|
1901
|
-
search(params: TraceSearchParams): Promise<TraceSearchResult>;
|
|
1902
|
-
getAssertions(traceId: string): Promise<TraceAssertionsResult>;
|
|
1903
|
-
saveAssertions(params: SaveAssertionsParams): Promise<TraceAssertion[]>;
|
|
1904
|
-
saveAssertionsAll(params: SaveAssertionsAllParams): Promise<TraceAssertion[]>;
|
|
1905
|
-
generateAssertions(traceId: string, options?: GenerateAssertionsOptions): Promise<TraceAssertion[]>;
|
|
1906
|
-
archiveAssertions(params: ArchiveAssertionsParams): Promise<string[]>;
|
|
1907
|
-
getScenario(traceId: string): Promise<Scenario>;
|
|
1908
|
-
saveScenario({ traceId, ...edit }: SaveScenarioParams): Promise<Scenario>;
|
|
1909
|
-
}
|
|
1910
|
-
|
|
1911
|
-
type LabelConfidence = "VeryLow" | "Low" | "Medium" | "High" | "VeryHigh";
|
|
1912
|
-
type LabelAction = "set" | "archived" | "no-active-label" | "skipped" | "abstained";
|
|
1913
|
-
type LabelStatus = "labeled" | "skipped" | "unlabeled";
|
|
1914
|
-
type LabelSource = "human" | "agent";
|
|
1915
|
-
/** An assertion verdict. `abstained` means the judge could not find the evidence the assertion needs; it counts as a fail. */
|
|
1916
|
-
type Verdict = "pass" | "fail" | "abstained";
|
|
1917
|
-
type LabelAddress = {
|
|
1918
|
-
traceId: string;
|
|
1919
|
-
originalTraceId?: never;
|
|
1920
|
-
attempt?: never;
|
|
1921
|
-
} | {
|
|
1922
|
-
originalTraceId: string;
|
|
1923
|
-
attempt?: number;
|
|
1924
|
-
traceId?: never;
|
|
1925
|
-
};
|
|
1926
|
-
type LabelTarget = LabelAddress & {
|
|
1927
|
-
assertionId: string;
|
|
1928
|
-
};
|
|
1929
|
-
type ArchiveLabelTarget = LabelAddress & {
|
|
1930
|
-
assertionId?: string;
|
|
1931
|
-
};
|
|
1932
|
-
type LabelVerdict = {
|
|
1933
|
-
label: boolean;
|
|
1934
|
-
annotation: string;
|
|
1935
|
-
confidence?: LabelConfidence;
|
|
1936
|
-
evidence?: Justification | null;
|
|
1937
|
-
} | {
|
|
1938
|
-
skip: true;
|
|
1939
|
-
annotation?: string;
|
|
1940
|
-
} | {
|
|
1941
|
-
abstain: true;
|
|
1942
|
-
annotation?: string;
|
|
1943
|
-
};
|
|
1944
|
-
type LabelUpdate = (LabelTarget & LabelVerdict) | (ArchiveLabelTarget & {
|
|
1945
|
-
archive: true;
|
|
1946
|
-
});
|
|
1947
|
-
type LabelOutcome = {
|
|
1948
|
-
key: string;
|
|
1949
|
-
traceId: string;
|
|
1950
|
-
action: LabelAction;
|
|
1951
|
-
};
|
|
1952
|
-
type HumanLabelUpdate = {
|
|
1953
|
-
traceId: string;
|
|
1954
|
-
assertionId: string;
|
|
1955
|
-
label: boolean;
|
|
1956
|
-
annotation: string;
|
|
1957
|
-
confidence?: LabelConfidence;
|
|
1958
|
-
evidence?: Justification | null;
|
|
1959
|
-
};
|
|
1960
|
-
type HumanLabelOutcome = {
|
|
1961
|
-
traceId: string;
|
|
1962
|
-
assertionId: string | null;
|
|
1963
|
-
label: boolean;
|
|
1964
|
-
action: "set";
|
|
1965
|
-
};
|
|
1966
|
-
type AssertionVerdict = {
|
|
1967
|
-
assertionId: string;
|
|
1968
|
-
assertion: string | null;
|
|
1969
|
-
labelStatus: LabelStatus;
|
|
1970
|
-
label: boolean | null;
|
|
1971
|
-
/** The verdict, or null when there is none. `label` carries the same verdict as a boolean, false for both fail and abstained. */
|
|
1972
|
-
verdict: Verdict | null;
|
|
1973
|
-
annotation: string | null;
|
|
1974
|
-
evidence: Justification | null;
|
|
1975
|
-
confidence: LabelConfidence | null;
|
|
1976
|
-
labelSource: LabelSource;
|
|
1977
|
-
approved: boolean;
|
|
1978
|
-
};
|
|
1979
|
-
type TraceLabels = {
|
|
1980
|
-
traceId: string;
|
|
1981
|
-
labelStatus: LabelStatus;
|
|
1982
|
-
label: boolean | null;
|
|
1983
|
-
annotation: string | null;
|
|
1984
|
-
evidence: Justification | null;
|
|
1985
|
-
approved: boolean;
|
|
1986
|
-
passed: number;
|
|
1987
|
-
failed: number;
|
|
1988
|
-
/** How many of the failed assertion verdicts are abstentions, where the judge could not find the evidence. */
|
|
1989
|
-
abstained: number;
|
|
1990
|
-
assertions: AssertionVerdict[];
|
|
1991
|
-
};
|
|
1992
|
-
declare class LabelsClient {
|
|
1993
|
-
private readonly httpClient;
|
|
1994
|
-
constructor(httpClient: HttpClient);
|
|
1995
|
-
save(update: LabelUpdate, experimentId?: string): Promise<LabelOutcome>;
|
|
1996
|
-
saveAll(updates: LabelUpdate[], experimentId?: string): Promise<LabelOutcome[]>;
|
|
1997
|
-
skip(target: LabelTarget & {
|
|
1998
|
-
annotation?: string;
|
|
1999
|
-
}, experimentId?: string): Promise<LabelOutcome>;
|
|
2000
|
-
/**
|
|
2001
|
-
* Record that the evidence does not show the part of the run this assertion
|
|
2002
|
-
* checks, so there was nothing to judge. Counted as a fail in every pass
|
|
2003
|
-
* rate, but stored and read back as an abstention, apart from a fail. Pass
|
|
2004
|
-
* `annotation` to say what the evidence would have had to show.
|
|
2005
|
-
*/
|
|
2006
|
-
abstain(target: LabelTarget & {
|
|
2007
|
-
annotation?: string;
|
|
2008
|
-
}, experimentId?: string): Promise<LabelOutcome>;
|
|
2009
|
-
archive(target: ArchiveLabelTarget, experimentId?: string): Promise<LabelOutcome>;
|
|
2010
|
-
saveHuman(update: HumanLabelUpdate): Promise<HumanLabelOutcome>;
|
|
2011
|
-
saveHumanAll(updates: HumanLabelUpdate[]): Promise<HumanLabelOutcome[]>;
|
|
2012
|
-
get(traceId: string): Promise<TraceLabels | null>;
|
|
2013
|
-
getAll(traceIds: string[]): Promise<TraceLabels[]>;
|
|
2014
|
-
/** Generate suggested evidence for an assertion on an original or replay trace. */
|
|
2015
|
-
generateLabelEvidence(traceId: string, assertionId: string): Promise<AssertionLabelEvidence[]>;
|
|
2016
|
-
}
|
|
2017
|
-
|
|
2018
|
-
type GraderLabelSource = "human" | "live_grader";
|
|
2019
|
-
type GraderLabel = {
|
|
2020
|
-
traceId: string;
|
|
2021
|
-
graderId: string;
|
|
2022
|
-
graderName: string | null;
|
|
2023
|
-
graderStatus: string;
|
|
2024
|
-
graderType: string;
|
|
2025
|
-
label: boolean | null;
|
|
2026
|
-
labelReason: string | null;
|
|
2027
|
-
failureDiagnostic: string | null;
|
|
2028
|
-
labelConfidence: LabelConfidence | null;
|
|
2029
|
-
source: GraderLabelSource;
|
|
2030
|
-
evaluatedAt: string | null;
|
|
2031
|
-
};
|
|
2032
|
-
type GetGraderLabelsParams = {
|
|
2033
|
-
traceIds?: string[];
|
|
2034
|
-
graderId?: string;
|
|
2035
|
-
limit?: number;
|
|
2036
|
-
};
|
|
2037
|
-
type GraderLabelUpdate = {
|
|
2038
|
-
graderId: string;
|
|
2039
|
-
traceId: string;
|
|
2040
|
-
label: boolean;
|
|
2041
|
-
reason?: string;
|
|
2042
|
-
failureDiagnostic?: string;
|
|
2043
|
-
confidence?: LabelConfidence;
|
|
2044
|
-
};
|
|
2045
|
-
type GraderLabelOutcome = {
|
|
2046
|
-
graderId: string;
|
|
2047
|
-
traceId: string;
|
|
2048
|
-
label: boolean;
|
|
2049
|
-
action: "set";
|
|
2050
|
-
};
|
|
2051
|
-
declare class GradersClient {
|
|
2052
|
-
private readonly httpClient;
|
|
2053
|
-
constructor(httpClient: HttpClient);
|
|
2054
|
-
getLabels(params: GetGraderLabelsParams): Promise<GraderLabel[]>;
|
|
2055
|
-
saveLabel(update: GraderLabelUpdate): Promise<GraderLabelOutcome>;
|
|
2056
|
-
saveLabelAll(updates: GraderLabelUpdate[]): Promise<GraderLabelOutcome[]>;
|
|
2057
|
-
}
|
|
2058
|
-
|
|
2059
|
-
/**
|
|
2060
|
-
* LangGraph/LangChain callback handler for Bitfab tracing.
|
|
2061
|
-
*
|
|
2062
|
-
* Hooks into LangGraph's callback system to capture graph node execution,
|
|
2063
|
-
* LLM calls, and tool invocations as Bitfab spans, without requiring users
|
|
2064
|
-
* to wrap their functions with withSpan (which fails on non-serializable args).
|
|
2065
|
-
*
|
|
2066
|
-
* Duck-typed to match LangChain.js's BaseCallbackHandler interface.
|
|
2067
|
-
* No @langchain/core dependency required.
|
|
2068
|
-
*/
|
|
2069
|
-
|
|
2070
|
-
interface ActiveSpanContext$1 {
|
|
2071
|
-
traceId: string;
|
|
2072
|
-
spanId: string;
|
|
2073
|
-
}
|
|
2074
|
-
/**
|
|
2075
|
-
* LangChain/LangGraph callback handler that sends traces to Bitfab.
|
|
2076
|
-
*
|
|
2077
|
-
* Duck-typed to match LangChain.js's BaseCallbackHandler, so no
|
|
2078
|
-
* `@langchain/core` dependency is required. Pass as a callback:
|
|
2079
|
-
*
|
|
2080
|
-
* ```typescript
|
|
2081
|
-
* const handler = bitfab.getLangGraphCallbackHandler("my-agent");
|
|
2082
|
-
* const result = await agent.invoke(
|
|
2083
|
-
* { messages: [...] },
|
|
2084
|
-
* { callbacks: [handler] },
|
|
2085
|
-
* );
|
|
2086
|
-
* ```
|
|
2087
|
-
*/
|
|
2088
|
-
declare class BitfabLangGraphCallbackHandler {
|
|
2089
|
-
name: string;
|
|
2090
|
-
ignoreRetry: boolean;
|
|
2091
|
-
ignoreRetriever: boolean;
|
|
2092
|
-
ignoreCustomEvent: boolean;
|
|
2093
|
-
private readonly httpClient;
|
|
2094
|
-
private readonly ownsHttpClient;
|
|
2095
|
-
private readonly traceFunctionKey;
|
|
2096
|
-
private readonly getActiveSpanContext;
|
|
2097
|
-
private readonly shouldRecord;
|
|
2098
|
-
private readonly captureTools;
|
|
2099
|
-
private runToSpan;
|
|
2100
|
-
private invocations;
|
|
2101
|
-
constructor(config: {
|
|
2102
|
-
apiKey?: ApiKeyInput;
|
|
2103
|
-
traceFunctionKey: string;
|
|
2104
|
-
serviceUrl?: string;
|
|
2105
|
-
timeout?: number;
|
|
2106
|
-
getActiveSpanContext?: () => ActiveSpanContext$1 | null;
|
|
2107
|
-
shouldRecord?: () => boolean;
|
|
2108
|
-
/** Whether callback tool events should emit spans. Defaults to true. */
|
|
2109
|
-
captureTools?: boolean;
|
|
2110
|
-
/**
|
|
2111
|
-
* The owning `Bitfab` client's HTTP client. Supplied by
|
|
2112
|
-
* `getLangGraphCallbackHandler()` so this handler shares that client's
|
|
2113
|
-
* single span-transport worker instead of starting a second one.
|
|
2114
|
-
* @internal
|
|
2115
|
-
*/
|
|
2116
|
-
_httpClient?: HttpClient;
|
|
2117
|
-
});
|
|
2118
|
-
/**
|
|
2119
|
-
* Flush and release the span transport this handler started. A no-op when
|
|
2120
|
-
* the handler borrowed a `Bitfab` client's HTTP client: that client's
|
|
2121
|
-
* `close()` owns the worker's lifetime.
|
|
2122
|
-
*/
|
|
2123
|
-
close(timeoutMs?: number): Promise<boolean>;
|
|
2124
|
-
private startSpan;
|
|
2125
|
-
private completeSpan;
|
|
2126
|
-
private sendSpan;
|
|
2127
|
-
private sendTraceCompletion;
|
|
2128
|
-
private sendTraceStart;
|
|
2129
|
-
handleChainStart(chain: Record<string, unknown> | null | undefined, inputs: Record<string, unknown>, runId: string, parentRunIdOrRunType?: string, tags?: string[], metadata?: Record<string, unknown>, runTypeOrRunName?: string, runNameOrParentRunId?: string): Promise<void>;
|
|
2130
|
-
handleChainEnd(outputs: Record<string, unknown>, runId: string): Promise<void>;
|
|
2131
|
-
handleChainError(error: unknown, runId: string): Promise<void>;
|
|
2132
|
-
handleChatModelStart(llm: Record<string, unknown> | null | undefined, messages: unknown[][], runId: string, parentRunId?: string, _extraParams?: Record<string, unknown>, tags?: string[], metadata?: Record<string, unknown>, runName?: string): Promise<void>;
|
|
2133
|
-
handleLLMStart(llm: Record<string, unknown> | null | undefined, prompts: string[], runId: string, parentRunId?: string, _extraParams?: Record<string, unknown>, tags?: string[], metadata?: Record<string, unknown>, runName?: string): Promise<void>;
|
|
2134
|
-
handleLLMEnd(output: Record<string, unknown>, runId: string): Promise<void>;
|
|
2135
|
-
handleLLMError(error: unknown, runId: string): Promise<void>;
|
|
2136
|
-
handleLLMNewToken(): Promise<void>;
|
|
2137
|
-
handleToolStart(tool: Record<string, unknown> | null | undefined, input: string, runId: string, parentRunId?: string, tags?: string[], metadata?: Record<string, unknown>, runName?: string): Promise<void>;
|
|
2138
|
-
handleToolEnd(output: unknown, runId: string): Promise<void>;
|
|
2139
|
-
handleToolError(error: unknown, runId: string): Promise<void>;
|
|
2140
|
-
handleRetrieverStart(retriever: Record<string, unknown> | null | undefined, query: string, runId: string, parentRunId?: string, tags?: string[], metadata?: Record<string, unknown>, runName?: string): Promise<void>;
|
|
2141
|
-
handleRetrieverEnd(documents: unknown, runId: string): Promise<void>;
|
|
2142
|
-
handleRetrieverError(error: unknown, runId: string): Promise<void>;
|
|
2143
|
-
}
|
|
2144
|
-
|
|
2145
|
-
interface SpanClient {
|
|
2146
|
-
withSpan<TArgs extends unknown[], TReturn>(traceFunctionKey: string, options: {
|
|
2147
|
-
name?: string;
|
|
2148
|
-
type?: "agent" | "function";
|
|
2149
|
-
captureWhen?: "always" | "nested";
|
|
2150
|
-
mockOnReplay?: boolean;
|
|
2151
|
-
finalize?: (result: unknown) => unknown | Promise<unknown>;
|
|
2152
|
-
surface?: "inherit";
|
|
2153
|
-
instrumentation?: "langgraph";
|
|
2154
|
-
}, fn: (...args: TArgs) => TReturn): (...args: TArgs) => TReturn;
|
|
2155
|
-
}
|
|
2156
|
-
interface LangGraphTool {
|
|
2157
|
-
readonly name?: string;
|
|
2158
|
-
invoke(input: unknown, ...rest: unknown[]): unknown;
|
|
2159
|
-
}
|
|
2160
|
-
interface ConfiguredLangGraphRunnable<TInput, TConfig, TReturn> {
|
|
2161
|
-
invoke(input: TInput, config?: TConfig): TReturn;
|
|
2162
|
-
}
|
|
2163
|
-
interface LangGraphRunnable<TInput, TConfig, TReturn> {
|
|
2164
|
-
withConfig(config: {
|
|
2165
|
-
callbacks: unknown[];
|
|
2166
|
-
}): ConfiguredLangGraphRunnable<TInput, TConfig, TReturn>;
|
|
2167
|
-
}
|
|
2168
|
-
/**
|
|
2169
|
-
* Options for first-class LangGraph ToolNode replay interception.
|
|
2170
|
-
*
|
|
2171
|
-
* @experimental This API may change before it is stable.
|
|
2172
|
-
*/
|
|
2173
|
-
interface LangGraphIntegrationOptions {
|
|
2174
|
-
/**
|
|
2175
|
-
* Tools marked for recorded-output mocking under replay's default
|
|
2176
|
-
* `mock: "marked"` strategy. Defaults to every wrapped tool. Pass a list to
|
|
2177
|
-
* mark only those tool names, or `false` to require `mock: "all"` or an
|
|
2178
|
-
* override.
|
|
2179
|
-
*/
|
|
2180
|
-
mockToolsOnReplay?: boolean | readonly string[];
|
|
2181
|
-
}
|
|
2182
|
-
/**
|
|
2183
|
-
* First-class LangGraph integration for callback tracing and replayable
|
|
2184
|
-
* `ToolNode` tools.
|
|
2185
|
-
*
|
|
2186
|
-
* @experimental This API may change before it is stable.
|
|
2187
|
-
*/
|
|
2188
|
-
declare class BitfabLangGraphIntegration {
|
|
2189
|
-
/** Callback handler for the compiled graph's invocation config. */
|
|
2190
|
-
readonly callbackHandler: any;
|
|
2191
|
-
private readonly client;
|
|
2192
|
-
private readonly traceFunctionKey;
|
|
2193
|
-
private readonly mockToolsOnReplay;
|
|
2194
|
-
constructor(config: {
|
|
2195
|
-
client: SpanClient;
|
|
2196
|
-
traceFunctionKey: string;
|
|
2197
|
-
callbackHandler: BitfabLangGraphCallbackHandler;
|
|
2198
|
-
options?: LangGraphIntegrationOptions;
|
|
2199
|
-
});
|
|
2200
|
-
/**
|
|
2201
|
-
* Wrap tools before passing the same returned array to both
|
|
2202
|
-
* `model.bindTools()` and `new ToolNode()`. Each invocation becomes an
|
|
2203
|
-
* independently mockable child span.
|
|
2204
|
-
*/
|
|
2205
|
-
wrapTools<T extends readonly LangGraphTool[]>(tools: T): T;
|
|
2206
|
-
/**
|
|
2207
|
-
* Create the normal graph entry point. The returned function adds Bitfab's
|
|
2208
|
-
* callback handler, preserves invocation config, and records only the graph
|
|
2209
|
-
* input as the replayable root input.
|
|
2210
|
-
*
|
|
2211
|
-
* @experimental This API may change before it is stable.
|
|
2212
|
-
*/
|
|
2213
|
-
createInvoker<TInput, TConfig, TReturn>(graph: LangGraphRunnable<TInput, TConfig, TReturn>): (input: TInput, config?: TConfig) => TReturn;
|
|
2214
|
-
private wrapTool;
|
|
2215
|
-
private assertReplayToolResultExists;
|
|
2216
|
-
/** Wrap the function that invokes the compiled graph as the replay root. */
|
|
2217
|
-
wrapInvoke<TArgs extends unknown[], TReturn>(fn: (...args: TArgs) => TReturn): (...args: TArgs) => TReturn;
|
|
2218
|
-
}
|
|
2219
|
-
|
|
2220
|
-
/**
|
|
2221
|
-
* OpenAI Agents SDK handler for Bitfab tracing.
|
|
2222
|
-
*
|
|
2223
|
-
* The OpenAI Agents SDK is instrumented in two layers:
|
|
2224
|
-
*
|
|
2225
|
-
* 1. A process-wide `TracingProcessor` (see `getOpenAiTracingProcessor` /
|
|
2226
|
-
* `BitfabOpenAITracingProcessor`) registered once with `addTraceProcessor`.
|
|
2227
|
-
* It captures everything *inside* a run - LLM calls, tool calls, handoffs -
|
|
2228
|
-
* as Bitfab spans.
|
|
2229
|
-
* 2. This handler's `wrapRun`, which owns the *root*. The processor never sees
|
|
2230
|
-
* the caller's input (the SDK's trace events don't carry it), so a
|
|
2231
|
-
* processor-only run records a root span with an empty input and is not
|
|
2232
|
-
* replayable. `wrapRun` is a thin drop-in for `run()` that opens a
|
|
2233
|
-
* `withSpan` root carrying the input and final output, so the run is
|
|
2234
|
-
* replayable with no hand-written `withSpan`. The processor's spans nest
|
|
2235
|
-
* underneath it automatically (it remaps onto the active span context).
|
|
2236
|
-
*
|
|
2237
|
-
* When `wrapRun` runs inside an enclosing Bitfab span (the replay auto-wrap, or
|
|
2238
|
-
* a caller's own `withSpan`), that span is already the replayable root: the
|
|
2239
|
-
* handler skips opening a second one and lets the processor nest the run's spans
|
|
2240
|
-
* under the existing root. This mirrors the Claude Agent SDK and LangGraph
|
|
2241
|
-
* handlers, which no-op their root span under an enclosing span, and keeps a
|
|
2242
|
-
* replayed run's span tree identical to the original (no doubled root agent
|
|
2243
|
-
* span).
|
|
2244
|
-
*
|
|
2245
|
-
* Use both together: register the processor once at startup, then call
|
|
2246
|
-
* `handler.wrapRun(agent, input)` instead of `run(agent, input)`.
|
|
2247
|
-
*/
|
|
2248
|
-
type AgentLike = any;
|
|
2249
|
-
type RunInput = string | unknown[] | Record<string, unknown>;
|
|
2250
|
-
type RunOptions = {
|
|
2251
|
-
stream?: boolean;
|
|
2252
|
-
} & Record<string, unknown>;
|
|
2253
|
-
type RunResultLike = {
|
|
2254
|
-
finalOutput?: unknown;
|
|
2255
|
-
completed?: Promise<void>;
|
|
2256
|
-
};
|
|
2257
|
-
type RootSpanOptions = {
|
|
2258
|
-
type: "agent";
|
|
2259
|
-
finalize: (result: unknown) => unknown | Promise<unknown>;
|
|
2260
|
-
surface: "inherit";
|
|
2261
|
-
instrumentation: "openai-agents";
|
|
2262
|
-
};
|
|
2263
|
-
type WithSpanFn = <TArgs extends unknown[], TReturn>(traceFunctionKey: string, options: RootSpanOptions, fn: (...args: TArgs) => TReturn) => (...args: TArgs) => TReturn;
|
|
2264
|
-
type GetActiveSpanContextFn = () => unknown | null;
|
|
2265
|
-
/**
|
|
2266
|
-
* OpenAI Agents SDK handler that records a replayable root span around a run.
|
|
2267
|
-
*
|
|
2268
|
-
* ```typescript
|
|
2269
|
-
* import { Bitfab } from "@bitfab/sdk";
|
|
2270
|
-
* import { addTraceProcessor, Agent, run } from "@openai/agents";
|
|
2271
|
-
*
|
|
2272
|
-
* const bitfab = new Bitfab({ apiKey: "..." });
|
|
2273
|
-
* addTraceProcessor(bitfab.getOpenAiTracingProcessor()); // captures internals
|
|
2274
|
-
*
|
|
2275
|
-
* const agent = new Agent({ name: "Researcher", instructions: "..." });
|
|
2276
|
-
* const handler = bitfab.getOpenAiAgentHandler("research-topic");
|
|
2277
|
-
*
|
|
2278
|
-
* // Swap run(agent, input) -> handler.wrapRun(agent, input)
|
|
2279
|
-
* const result = await handler.wrapRun(agent, "Find X");
|
|
2280
|
-
* return result.finalOutput;
|
|
2281
|
-
* ```
|
|
2282
|
-
*/
|
|
2283
|
-
declare class BitfabOpenAIAgentHandler {
|
|
2284
|
-
private readonly traceFunctionKey;
|
|
2285
|
-
private readonly withSpanFn;
|
|
2286
|
-
private readonly getActiveSpanContext?;
|
|
2287
|
-
constructor(config: {
|
|
2288
|
-
traceFunctionKey: string;
|
|
2289
|
-
withSpan: WithSpanFn;
|
|
2290
|
-
getActiveSpanContext?: GetActiveSpanContextFn;
|
|
2291
|
-
});
|
|
2292
|
-
/**
|
|
2293
|
-
* Drop-in replacement for the OpenAI Agents SDK's `run()` that records a
|
|
2294
|
-
* replayable root `agent` span.
|
|
2295
|
-
*
|
|
2296
|
-
* The `input` is captured as the root span's input (as a single positional
|
|
2297
|
-
* argument, so `replay(key, fn)` re-feeds it), and the run's `finalOutput`
|
|
2298
|
-
* is recorded as the root output. For streaming runs (`{ stream: true }`),
|
|
2299
|
-
* the result is handed back immediately and the final output is recorded
|
|
2300
|
-
* once the stream completes - first-byte latency is untouched.
|
|
2301
|
-
*
|
|
2302
|
-
* The process-wide tracing processor (`getOpenAiTracingProcessor`) must still
|
|
2303
|
-
* be registered: it captures the LLM/tool/handoff spans that nest beneath
|
|
2304
|
-
* this root.
|
|
2305
|
-
*/
|
|
2306
|
-
wrapRun(agent: AgentLike, input: RunInput, options?: RunOptions): Promise<RunResultLike>;
|
|
2307
|
-
}
|
|
2308
|
-
|
|
2309
|
-
type OrganizationMember = {
|
|
2310
|
-
id: string;
|
|
2311
|
-
fullName: string | null;
|
|
2312
|
-
email: string | null;
|
|
2313
|
-
imageUrl: string | null;
|
|
2314
|
-
};
|
|
2315
|
-
/** The people in the API key's organization. */
|
|
2316
|
-
declare class OrganizationMembersClient {
|
|
2317
|
-
private readonly httpClient;
|
|
2318
|
-
constructor(httpClient: HttpClient);
|
|
2319
|
-
/**
|
|
2320
|
-
* List the organization's members, ordered by name. Read it to find the
|
|
2321
|
-
* address to pass as an assertion's `assigneeEmail`.
|
|
2322
|
-
*/
|
|
2323
|
-
list(): Promise<OrganizationMember[]>;
|
|
2324
|
-
}
|
|
2325
|
-
|
|
2326
|
-
type ReplayInterruptSignal = "SIGINT" | "SIGTERM";
|
|
2327
|
-
interface ReplayInterrupt {
|
|
2328
|
-
experimentId: string;
|
|
2329
|
-
experimentUrl: string;
|
|
2330
|
-
signal: ReplayInterruptSignal;
|
|
2331
|
-
markedInterrupted: boolean;
|
|
2332
|
-
}
|
|
2333
|
-
|
|
2334
|
-
/**
|
|
2335
|
-
* Replay historical traces through a function and create an experiment.
|
|
2336
|
-
*
|
|
2337
|
-
* The replay flow has three phases:
|
|
2338
|
-
* 1. Start: fetches historical traces from the server and creates an experiment
|
|
2339
|
-
* 2. Execute: re-runs each trace's inputs through the provided function locally
|
|
2340
|
-
* 3. Complete: marks the experiment as completed on the server
|
|
2341
|
-
*/
|
|
2342
|
-
|
|
2343
|
-
type MockStrategy = "none" | "all" | "marked";
|
|
2344
|
-
type TraceIngestionType = "captured" | "seeded";
|
|
2345
|
-
/**
|
|
2346
|
-
* How the DB-snapshot branch each replay item runs against is sized and warmed.
|
|
2347
|
-
*
|
|
2348
|
-
* Every field is optional, so this object is only worth passing when you want
|
|
2349
|
-
* to override the mirror project's own sizing. To branch with those defaults,
|
|
2350
|
-
* pass `dbBranch: true` instead of an empty object.
|
|
2351
|
-
*/
|
|
2352
|
-
interface DbBranchOptions {
|
|
2353
|
-
/**
|
|
2354
|
-
* Autoscaling floor for the branch's compute, in Neon Compute Units (0.25 to
|
|
2355
|
-
* 56). Omit to keep the mirror project's own default. Raise it when the
|
|
2356
|
-
* mirror is provisioned smaller than the database it stands in for, so
|
|
2357
|
-
* replay latency reflects your code rather than a cold, undersized branch.
|
|
2358
|
-
*/
|
|
2359
|
-
minCu?: number;
|
|
2360
|
-
/**
|
|
2361
|
-
* Autoscaling ceiling for the branch's compute, in Neon Compute Units.
|
|
2362
|
-
* Setting it equal to `minCu` pins the size, which keeps items comparable:
|
|
2363
|
-
* otherwise a later item can run against an endpoint that has already
|
|
2364
|
-
* scaled up and post a better number for the same code.
|
|
2365
|
-
*/
|
|
2366
|
-
maxCu?: number;
|
|
2367
|
-
/**
|
|
2368
|
-
* SQL that warms the branch's cache. The server appends it to the branch's
|
|
2369
|
-
* readiness check, so it runs BEFORE your function sees the branch and its
|
|
2370
|
-
* time is not charged to the replayed call. Invalid SQL fails the branch
|
|
2371
|
-
* rather than silently leaving it cold.
|
|
2372
|
-
*/
|
|
2373
|
-
warmupSql?: string;
|
|
2374
|
-
}
|
|
2375
|
-
interface ReplayOptions {
|
|
2376
|
-
/**
|
|
2377
|
-
* Experimental selective replay. Explicit required nodes
|
|
2378
|
-
* always execute; eligible output-only nodes reuse JSON recordings only
|
|
2379
|
-
* when runtime inputs match. Requires a compatible server, mock: "marked",
|
|
2380
|
-
* and no mock overrides or resume. Does not infer changes from codeChangeFiles.
|
|
2381
|
-
* The server reads active/inherited assertions and protects their scoped
|
|
2382
|
-
* execution evidence. Coverage in item.selectiveReplay is not a verdict.
|
|
2383
|
-
*/
|
|
2384
|
-
experimentalSelectiveReplay?: SelectiveReplayOptions;
|
|
2385
|
-
/**
|
|
2386
|
-
* Maximum number of traces to replay (1-5,000, default 5). Ignored when
|
|
2387
|
-
* `traceIds` is passed (with a warning), or when `datasetId` / `datasetIds`
|
|
2388
|
-
* is passed, because either source already determines how many traces replay.
|
|
2389
|
-
* To replay part of a dataset selection, name the members to run in
|
|
2390
|
-
* `traceIds` rather than bounding it here.
|
|
2391
|
-
*/
|
|
2392
|
-
limit?: number;
|
|
2393
|
-
attempts?: number;
|
|
2394
|
-
/**
|
|
2395
|
-
* Optional list of specific trace IDs to replay (max 100). Passed alongside
|
|
2396
|
-
* `datasetId` or `datasetIds` it pins which members of that selection replay,
|
|
2397
|
-
* and the server rejects any ID none of those datasets contains. On its own
|
|
2398
|
-
* it selects by ID with no dataset attribution.
|
|
2399
|
-
*/
|
|
2400
|
-
traceIds?: string[];
|
|
2401
|
-
/**
|
|
2402
|
-
* What this run is testing, in a few words, such as 'baseline' or 'shorter
|
|
2403
|
-
* system prompt'. Bitfab records the commit, branch, tree state, datasets,
|
|
2404
|
-
* and who ran it with every experiment, so do not repeat them here.
|
|
2405
|
-
*/
|
|
2406
|
-
name?: string;
|
|
2407
|
-
/**
|
|
2408
|
-
* Run conditions Bitfab cannot see on its own, such as an environment
|
|
2409
|
-
* override or a forced feature flag. Kept on the experiment next to its name.
|
|
2410
|
-
*/
|
|
2411
|
-
notes?: string;
|
|
2412
|
-
metadata?: Record<string, string>;
|
|
2413
|
-
/** Positive integer limiting parallel items. Set to 1 for sequential. Default 10. */
|
|
2414
|
-
maxConcurrency?: number;
|
|
2415
|
-
/**
|
|
2416
|
-
* Description of the code change being tested in this replay. Stored on
|
|
2417
|
-
* the resulting experiment so the change can be reviewed alongside results.
|
|
2418
|
-
* Supplying a description without `codeChangeFiles` preserves this text
|
|
2419
|
-
* while the SDK automatically captures the files.
|
|
2420
|
-
* Pass `null` when the replay should have no description; use
|
|
2421
|
-
* `codeChangeFiles: null` to suppress automatic file capture.
|
|
2422
|
-
*/
|
|
2423
|
-
codeChangeDescription?: string | null;
|
|
2424
|
-
/**
|
|
2425
|
-
* Files edited as part of this code change. Each entry holds the file path
|
|
2426
|
-
* and the full `before`/`after` contents. The agent reads each file before
|
|
2427
|
-
* and after editing and passes the two strings. Use `""` for newly created
|
|
2428
|
-
* files (`before`) or deleted files (`after`). Omit this field to capture
|
|
2429
|
-
* automatically, or pass `null` to suppress file capture for this replay.
|
|
2430
|
-
*/
|
|
2431
|
-
codeChangeFiles?: CodeChangeFile[] | null;
|
|
2432
|
-
/**
|
|
2433
|
-
* Mock strategy for child spans during replay.
|
|
2434
|
-
* - "marked": only spans tagged with { mockOnReplay: true } in SpanOptions are mocked (default)
|
|
2435
|
-
* - "none": everything runs real code
|
|
2436
|
-
* - "all": every matched recorded child withSpan returns historical output;
|
|
2437
|
-
* a missing occurrence fails the replay item closed
|
|
2438
|
-
*/
|
|
2439
|
-
mock?: MockStrategy;
|
|
2440
|
-
/**
|
|
2441
|
-
* Selective mock overrides: inject custom values into specific spans during
|
|
2442
|
-
* replay, so downstream real code runs against the substituted output. Each
|
|
2443
|
-
* Pass `{ match, value }` pairs, or one resolver invoked for every child span.
|
|
2444
|
-
* A resolver can route on `ctx.node.traceFunctionKey` and return
|
|
2445
|
-
* `NO_MOCK_OVERRIDE` to continue to the next override and base strategy.
|
|
2446
|
-
* Per-call overrides take precedence over registrations on the client.
|
|
2447
|
-
*/
|
|
2448
|
-
mockOverride?: MockOverrideInput | MockOverrideInput[];
|
|
2449
|
-
/**
|
|
2450
|
-
* Run each item against a database branch restored to the state its source
|
|
2451
|
-
* trace saw. Pass `true` to branch with the mirror project's own sizing, or a
|
|
2452
|
-
* {@link DbBranchOptions} object to tune how each branch is sized and warmed.
|
|
2453
|
-
* `false` and omission leave branching off. Inside `fn`, read the resolved
|
|
2454
|
-
* branch with `getCurrentReplayBranch()`.
|
|
2455
|
-
*
|
|
2456
|
-
* Items whose source trace carried no DB snapshot reference get no branch and
|
|
2457
|
-
* use the app's normal database path. Unsafe calls on that path still require
|
|
2458
|
-
* replay mocking. An item whose branch was requested but could not be resolved
|
|
2459
|
-
* fails instead of running, so replay never silently reports a result that did
|
|
2460
|
-
* not use the historical data you asked for.
|
|
2461
|
-
*/
|
|
2462
|
-
dbBranch?: DbBranchOptions | boolean;
|
|
2463
|
-
/**
|
|
2464
|
-
* Experiment group this replay joins, shown together on the experiments
|
|
2465
|
-
* page. An ID with no group yet creates the group.
|
|
2466
|
-
*/
|
|
2467
|
-
experimentGroupId?: string;
|
|
2468
|
-
/**
|
|
2469
|
-
* Name for the experiment group in `experimentGroupId`. Applied when this
|
|
2470
|
-
* replay creates the group, or when the group exists without a name. A group
|
|
2471
|
-
* that already has a name keeps it. Requires `experimentGroupId`.
|
|
2472
|
-
*/
|
|
2473
|
-
experimentGroupName?: string;
|
|
2474
|
-
/**
|
|
2475
|
-
* Dataset this replay runs against. Mutually exclusive with `datasetIds`.
|
|
2476
|
-
* Alone it replays the dataset's full membership, and with `traceIds` it
|
|
2477
|
-
* replays only those members. When set, the resulting experiment is
|
|
2478
|
-
* durably attributed to the dataset (stored on the experiment), so it appears
|
|
2479
|
-
* under the dataset's experiments even if the trace lineage can't be
|
|
2480
|
-
* reconstructed. Validated server-side against the org.
|
|
2481
|
-
*/
|
|
2482
|
-
datasetId?: string;
|
|
2483
|
-
datasetIds?: string[];
|
|
2484
|
-
/**
|
|
2485
|
-
* Graders to attach directly to this experiment, independent of any
|
|
2486
|
-
* graders already on the dataset. The resulting experiment is graded by the
|
|
2487
|
-
* union of these and the dataset's runnable graders at completion, so use this
|
|
2488
|
-
* to grade a single run with a check you don't want to add to the dataset
|
|
2489
|
-
* permanently. Each id must be an active/live grader belonging to the same
|
|
2490
|
-
* organization and trace function, otherwise the server rejects the replay.
|
|
2491
|
-
*/
|
|
2492
|
-
graderIds?: string[];
|
|
2493
|
-
/**
|
|
2494
|
-
* Replay only the selected traces that carry at least one approved assertion.
|
|
2495
|
-
*
|
|
2496
|
-
* Narrows whatever `limit`, `traceIds` or `datasetIds` selected, so a run that
|
|
2497
|
-
* measures assertion outcomes does not pay to re-execute traces nothing can
|
|
2498
|
-
* grade. The server applies it as part of selection, which is what makes it
|
|
2499
|
-
* compose with `limit`: `{ limit: 10, onlyWithAssertions: true }` is the ten
|
|
2500
|
-
* most recent traces that HAVE approved assertions, not the ten most recent
|
|
2501
|
-
* filtered down to however few do. An archived assertion does not count, and
|
|
2502
|
-
* neither does one still awaiting review: a replay is only ever judged
|
|
2503
|
-
* against assertions a person has approved, so a trace carrying nothing but
|
|
2504
|
-
* drafts has nothing here to measure. A selection that narrows to nothing
|
|
2505
|
-
* replays nothing.
|
|
2506
|
-
*/
|
|
2507
|
-
onlyWithAssertions?: boolean;
|
|
2508
|
-
skipAssertionJudging?: boolean;
|
|
2509
|
-
judgeAssertions?: boolean;
|
|
2510
|
-
/**
|
|
2511
|
-
* Reshape recorded inputs before they are spread into `fn`.
|
|
2512
|
-
*
|
|
2513
|
-
* Replay pulls each trace's inputs exactly as they were captured against the
|
|
2514
|
-
* function's signature AT TRACE TIME. When the function's shape has since
|
|
2515
|
-
* changed (params renamed, reordered, collapsed into an options object, etc.),
|
|
2516
|
-
* the recorded inputs no longer line up and `fn(...inputs)` throws. This hook
|
|
2517
|
-
* lets a caller map the recorded inputs onto the current signature so replay
|
|
2518
|
-
* can still run.
|
|
2519
|
-
*
|
|
2520
|
-
* Receives the deserialized recorded inputs and a per-trace {@link AdaptContext}
|
|
2521
|
-
* (so a table-driven adapter can look up the adapted inputs by `traceId`), and
|
|
2522
|
-
* returns the array actually spread into `fn`. The returned array is also what
|
|
2523
|
-
* `ReplayItem.input` reports, so the experiment shows what was really run.
|
|
2524
|
-
*
|
|
2525
|
-
* Runs per item, inside the same try/catch as `fn`: if the adapter throws, the
|
|
2526
|
-
* failure is surfaced on that item's `error` rather than crashing the run.
|
|
2527
|
-
* Omit it to spread the recorded inputs unchanged.
|
|
2528
|
-
*/
|
|
2529
|
-
adaptInputs?: (inputs: unknown[], ctx: AdaptContext) => unknown[];
|
|
2530
|
-
/**
|
|
2531
|
-
* Resolve every item's inputs and stop, without calling the function.
|
|
2532
|
-
*
|
|
2533
|
-
* Selection, span fetch, deserialization, and `adaptInputs` all run, so each
|
|
2534
|
-
* item reports the exact arguments the function would have received. Nothing
|
|
2535
|
-
* executes and no replay traces are produced, which is the cheap way to check
|
|
2536
|
-
* that recorded inputs still fit the current signature. No experiment is
|
|
2537
|
-
* created either, so the result's `experimentId` and `experimentUrl` are null
|
|
2538
|
-
* and `onExperimentStart` is not called.
|
|
2539
|
-
*/
|
|
2540
|
-
dryRun?: boolean;
|
|
2541
|
-
/**
|
|
2542
|
-
* Called once per item as it finishes, in completion order (not input
|
|
2543
|
-
* order), with running totals for the whole run. Use it to render replay
|
|
2544
|
-
* progress, for example a terminal progress bar. Replay does not know
|
|
2545
|
-
* pass/fail at this point (verdicts are assigned later), so the totals only
|
|
2546
|
-
* distinguish items whose function ran (`succeeded`) from items that threw
|
|
2547
|
-
* (`errored`).
|
|
2548
|
-
*
|
|
2549
|
-
* Invoked synchronously right after each item finishes, so keep it cheap. A
|
|
2550
|
-
* throwing callback never crashes the run: the error is swallowed so progress
|
|
2551
|
-
* UI can't break replay.
|
|
2552
|
-
*/
|
|
2553
|
-
onItemFinish?: (progress: ReplayItemFinishProgress) => void;
|
|
2554
|
-
/**
|
|
2555
|
-
* @deprecated Use {@link onItemFinish}. This compatibility callback receives
|
|
2556
|
-
* the same per-item finish events plus the legacy whole-run `"complete"`
|
|
2557
|
-
* event. It is ignored when `onItemFinish` is also provided.
|
|
2558
|
-
*/
|
|
2559
|
-
onProgress?: (progress: ReplayProgress) => void;
|
|
2560
|
-
/**
|
|
2561
|
-
* Called once when each replay item begins processing, before input loading,
|
|
2562
|
-
* mock preparation, database branching, or the customer function runs. Use
|
|
2563
|
-
* it with {@link onItemFinish} to distinguish queued items from items that
|
|
2564
|
-
* are still in flight. A throwing callback never crashes the run.
|
|
2565
|
-
*/
|
|
2566
|
-
onItemStart?: (progress: ReplayItemStartProgress) => void;
|
|
2567
|
-
/**
|
|
2568
|
-
* Called once, right after the server creates the experiment and before any
|
|
2569
|
-
* item runs, with the experiment ID and its dashboard URL. Use it to show
|
|
2570
|
-
* where results will appear while the replay is still running. A throwing
|
|
2571
|
-
* callback never crashes the run.
|
|
2572
|
-
*/
|
|
2573
|
-
onExperimentStart?: (experiment: ReplayExperiment) => void;
|
|
2574
|
-
onInterrupt?: (interrupt: ReplayInterrupt) => void;
|
|
2575
|
-
resume?: string;
|
|
2576
|
-
force?: boolean;
|
|
2577
|
-
}
|
|
2578
|
-
/** The experiment a replay records into, passed to {@link ReplayOptions.onExperimentStart}. */
|
|
2579
|
-
interface ReplayExperiment {
|
|
2580
|
-
/** Experiment ID created for this replay. */
|
|
2581
|
-
experimentId: string;
|
|
2582
|
-
/** Full URL to view the experiment in the dashboard. */
|
|
2583
|
-
experimentUrl: string;
|
|
2584
|
-
}
|
|
2585
|
-
/** Emitted through {@link ReplayOptions.onItemStart} when a worker starts an item. */
|
|
2586
|
-
interface ReplayItemStartProgress {
|
|
2587
|
-
type: "started";
|
|
2588
|
-
/** Experiment ID created for this replay. Null for a dry run, which creates none. */
|
|
2589
|
-
experimentId: string | null;
|
|
2590
|
-
/** @deprecated Use {@link experimentId}. Same value. */
|
|
2591
|
-
testRunId: string | null;
|
|
2592
|
-
/** Items whose processing has started so far. */
|
|
2593
|
-
started: number;
|
|
2594
|
-
/** Items that have finished so far, whether they succeeded or errored. */
|
|
2595
|
-
completed: number;
|
|
2596
|
-
/** Total number of items in this replay run. */
|
|
2597
|
-
total: number;
|
|
2598
|
-
/** Of the completed items, how many finished without a code or replay error. */
|
|
2599
|
-
succeeded: number;
|
|
2600
|
-
/** Of the completed items, how many finished with an error. */
|
|
2601
|
-
errored: number;
|
|
2602
|
-
/** The historical trace and span whose replay processing just started. */
|
|
2603
|
-
item: {
|
|
2604
|
-
originalTraceId: string;
|
|
2605
|
-
originalSpanId: string;
|
|
2606
|
-
/** @deprecated alias for `originalTraceId`. */
|
|
2607
|
-
sourceTraceId: string;
|
|
2608
|
-
/** @deprecated alias for `originalSpanId`. */
|
|
2609
|
-
sourceSpanId: string;
|
|
2610
|
-
attempt: number;
|
|
2611
|
-
};
|
|
2612
|
-
}
|
|
2613
|
-
/** Running totals reported to {@link ReplayOptions.onItemFinish} as replay proceeds. */
|
|
2614
|
-
interface ReplayItemFinishProgress {
|
|
2615
|
-
/** Event kind, omitted for backward-compatible per-item events. */
|
|
2616
|
-
type?: "item";
|
|
2617
|
-
/** Experiment ID created for this replay. Null for a dry run, which creates none. */
|
|
2618
|
-
experimentId: string | null;
|
|
2619
|
-
/** @deprecated Use {@link experimentId}. Same value. */
|
|
2620
|
-
testRunId: string | null;
|
|
2621
|
-
/** Items that have finished so far, whether they succeeded or errored. */
|
|
2622
|
-
completed: number;
|
|
2623
|
-
/** Total number of items in this replay run. */
|
|
2624
|
-
total: number;
|
|
2625
|
-
/** Of the completed items, how many finished without a code or replay error. */
|
|
2626
|
-
succeeded: number;
|
|
2627
|
-
/** Of the completed items, how many have `item.error` set. */
|
|
2628
|
-
errored: number;
|
|
2629
|
-
/**
|
|
2630
|
-
* The single item that just finished to produce this event. Field-for-field
|
|
2631
|
-
* the same shape and meaning as {@link ReplayItem}, so a progress UI and a
|
|
2632
|
-
* final-result UI read one contract. `traceId` is null at this stage (the
|
|
2633
|
-
* server replay id arrives at completion), and `tokens` is null for the same
|
|
2634
|
-
* reason: the replayed run's usage is aggregated server-side by
|
|
2635
|
-
* completeReplay. `durationMs` is how long this item took to replay; the
|
|
2636
|
-
* `original*` fields describe the trace it replayed.
|
|
2637
|
-
*/
|
|
2638
|
-
item: ReplayProgressItem;
|
|
2639
|
-
}
|
|
2640
|
-
/** See {@link ReplayItemFinishProgress.item}. Mirrors {@link ReplayItem}. */
|
|
2641
|
-
interface ReplayProgressItem {
|
|
2642
|
-
selectiveReplay?: SelectiveReplayReport;
|
|
2643
|
-
/** Trace ID of the new replay trace (null during the run; the server id arrives at completion). */
|
|
2644
|
-
traceId?: string | null;
|
|
2645
|
-
/** Bitfab trace ID of the original (historical) trace being replayed. */
|
|
2646
|
-
originalTraceId: string | null;
|
|
2647
|
-
/** External span ID the recorded inputs were read from (the original root span). */
|
|
2648
|
-
originalSpanId?: string | null;
|
|
2649
|
-
/** @deprecated alias for `originalTraceId`. */
|
|
2650
|
-
sourceTraceId: string | null;
|
|
2651
|
-
/** @deprecated alias for `originalSpanId`. */
|
|
2652
|
-
sourceSpanId?: string | null;
|
|
2653
|
-
attempt: number;
|
|
2654
|
-
/** Deserialized inputs from the original trace. */
|
|
2655
|
-
input?: unknown[];
|
|
2656
|
-
/** The result returned by the replayed function, or undefined on error. */
|
|
2657
|
-
result?: unknown;
|
|
2658
|
-
/** The original output from the historical trace. */
|
|
2659
|
-
originalOutput?: unknown;
|
|
2660
|
-
/** Backward-compatible message for either error kind. */
|
|
2661
|
-
error: string | null;
|
|
2662
|
-
/** The actual value thrown by the replayed customer function. */
|
|
2663
|
-
traceError?: unknown | null;
|
|
2664
|
-
/** The actual value thrown while Bitfab prepared this replay item. */
|
|
2665
|
-
replayError?: unknown | null;
|
|
2666
|
-
/** How long the replayed function took on this run. */
|
|
2667
|
-
durationMs?: number | null;
|
|
2668
|
-
/** The original trace's duration, tokens, and model. */
|
|
2669
|
-
originalDurationMs?: number | null;
|
|
2670
|
-
originalTokens?: TokenUsage | null;
|
|
2671
|
-
originalModel?: string | null;
|
|
2672
|
-
ingestionType?: TraceIngestionType;
|
|
2673
|
-
/** Always null here: the replayed run's usage is only known at completion. */
|
|
2674
|
-
tokens?: TokenUsage | null;
|
|
2675
|
-
/** @deprecated renamed to `originalModel`. */
|
|
2676
|
-
model?: string | null;
|
|
2677
|
-
dbSnapshotRef?: DbSnapshotRef | null;
|
|
2678
|
-
dbBranchTimings?: DbBranchTimings | null;
|
|
2679
|
-
}
|
|
2680
|
-
/**
|
|
2681
|
-
* @deprecated Use {@link ReplayItemFinishProgress}. This legacy shape also
|
|
2682
|
-
* represents the item-less terminal `"complete"` event emitted by
|
|
2683
|
-
* {@link ReplayOptions.onProgress}.
|
|
2684
|
-
*/
|
|
2685
|
-
interface ReplayProgress {
|
|
2686
|
-
type?: "item" | "complete";
|
|
2687
|
-
result?: ReplayResult<unknown>;
|
|
2688
|
-
/** Null for a dry run, which creates no experiment. */
|
|
2689
|
-
experimentId?: string | null;
|
|
2690
|
-
/** @deprecated Use {@link experimentId}. Same value. */
|
|
2691
|
-
testRunId?: string | null;
|
|
2692
|
-
completed: number;
|
|
2693
|
-
total: number;
|
|
2694
|
-
succeeded: number;
|
|
2695
|
-
errored: number;
|
|
2696
|
-
item?: ReplayItemFinishProgress["item"];
|
|
2697
|
-
}
|
|
2698
|
-
/**
|
|
2699
|
-
* Wire prefix the Bitfab plugin scans for. Each line {@link reportReplayProgress}
|
|
2700
|
-
* writes is this prefix followed by the JSON of the running totals (plus the
|
|
2701
|
-
* finished `item`). The plugin polls these lines to report live progress while a
|
|
2702
|
-
* replay runs in the background; keep the SDK emitter and the plugin parser in
|
|
2703
|
-
* sync.
|
|
2704
|
-
*/
|
|
2705
|
-
declare const BITFAB_PROGRESS_PREFIX = "@@bitfab:progress ";
|
|
2706
|
-
/**
|
|
2707
|
-
* A ready-made replay lifecycle callback for replay scripts.
|
|
2708
|
-
* Pass it straight in:
|
|
2709
|
-
*
|
|
2710
|
-
* ```ts
|
|
2711
|
-
* await bitfab.replay("my-fn", fn, {
|
|
2712
|
-
* limit,
|
|
2713
|
-
* onItemStart: reportReplayProgress,
|
|
2714
|
-
* onItemFinish: reportReplayProgress,
|
|
2715
|
-
* })
|
|
2716
|
-
* ```
|
|
2717
|
-
*
|
|
2718
|
-
* It writes `@@bitfab:progress` lines to stderr, which the Bitfab plugin polls
|
|
2719
|
-
* to report in-flight and finished items while replay runs in the background,
|
|
2720
|
-
* so scripts never hand-format the wire protocol.
|
|
2721
|
-
* stdout is left untouched for direct-run ReplayResult JSON. Outside Node (no
|
|
2722
|
-
* `process.stderr`, e.g. a browser) it is a no-op, and a write failure is
|
|
2723
|
-
* swallowed so progress can never crash a run.
|
|
2724
|
-
*/
|
|
2725
|
-
declare function reportReplayProgress(progress: ReplayItemFinishProgress | ReplayItemStartProgress | ReplayProgress): void;
|
|
2726
|
-
/** Per-trace context passed to {@link ReplayOptions.adaptInputs}. */
|
|
2727
|
-
interface AdaptContext {
|
|
2728
|
-
/** Bitfab trace ID of the original (historical) trace being replayed. */
|
|
2729
|
-
originalTraceId: string;
|
|
2730
|
-
/** External span ID the recorded inputs were read from. */
|
|
2731
|
-
originalSpanId: string;
|
|
2732
|
-
/** @deprecated alias for {@link AdaptContext.originalTraceId}. */
|
|
2733
|
-
sourceTraceId: string;
|
|
2734
|
-
/** @deprecated alias for {@link AdaptContext.originalSpanId}. */
|
|
2735
|
-
sourceSpanId: string;
|
|
2736
|
-
metadata: Record<string, unknown>;
|
|
2737
|
-
}
|
|
2738
|
-
/**
|
|
2739
|
-
* The shape an adapter module must export as `adaptInputs`.
|
|
2740
|
-
*
|
|
2741
|
-
* Author an adapter in its own file (e.g. `scripts/replay-adapters/<name>.ts`),
|
|
2742
|
-
* import it in your replay script, and pass it to `replay({ adaptInputs })`:
|
|
2743
|
-
*
|
|
2744
|
-
* ```ts
|
|
2745
|
-
* import type { AdaptInputsFn } from "@bitfab/sdk"
|
|
2746
|
-
* export const adaptInputs: AdaptInputsFn = (inputs, ctx) => [reshape(inputs)]
|
|
2747
|
-
* ```
|
|
2748
|
-
*/
|
|
2749
|
-
type AdaptInputsFn = (inputs: unknown[], ctx: AdaptContext) => unknown[];
|
|
2750
|
-
interface ReplayItem<T> {
|
|
2751
|
-
/** Decisions and changed-code coverage for experimental selective replay. */
|
|
2752
|
-
selectiveReplay?: SelectiveReplayReport;
|
|
2753
|
-
/**
|
|
2754
|
-
* Server trace ID of the new replay trace this item produced. Written in by
|
|
2755
|
-
* `replay()` from the complete-replay response once the server has minted the
|
|
2756
|
-
* trace row; the client-side id used to correlate spans during the run is
|
|
2757
|
-
* never surfaced here. Null until completion, on older servers that omit the
|
|
2758
|
-
* mapping, or if the item produced no trace. Not the verdict-persistence key:
|
|
2759
|
-
* that is the original-trace lineage (`originalTraceId` + `experimentId`).
|
|
2760
|
-
*/
|
|
2761
|
-
traceId: string | null;
|
|
2762
|
-
/** Bitfab trace ID of the original (historical) trace being replayed. */
|
|
2763
|
-
originalTraceId: string;
|
|
2764
|
-
/** External span ID the recorded inputs were read from (the original root span). */
|
|
2765
|
-
originalSpanId: string;
|
|
2766
|
-
/** @deprecated alias for {@link ReplayItem.originalTraceId}. */
|
|
2767
|
-
sourceTraceId: string;
|
|
2768
|
-
/** @deprecated alias for {@link ReplayItem.originalSpanId}. */
|
|
2769
|
-
sourceSpanId: string;
|
|
2770
|
-
attempt: number;
|
|
2771
|
-
/** Deserialized inputs from the original trace. */
|
|
2772
|
-
input: unknown[];
|
|
2773
|
-
/** The result returned by the function during replay, or undefined on error. */
|
|
2774
|
-
result: T | undefined;
|
|
2775
|
-
originalOutput: unknown;
|
|
2776
|
-
/**
|
|
2777
|
-
* How the source trace came to exist. Absent on servers that predate the
|
|
2778
|
-
* field, which only ever served captured traces.
|
|
2779
|
-
*/
|
|
2780
|
-
ingestionType?: TraceIngestionType;
|
|
2781
|
-
/**
|
|
2782
|
-
* Backward-compatible message for either error kind. Prefer `traceError` and
|
|
2783
|
-
* `replayError` when callers need the original exception and its source.
|
|
2784
|
-
*/
|
|
2785
|
-
error: string | null;
|
|
2786
|
-
/** The actual value thrown by the replayed customer function. */
|
|
2787
|
-
traceError: unknown | null;
|
|
2788
|
-
/**
|
|
2789
|
-
* The actual value thrown by replay setup before the customer function ran,
|
|
2790
|
-
* such as database warmup, input hydration, or mock preparation.
|
|
2791
|
-
*/
|
|
2792
|
-
replayError: unknown | null;
|
|
2793
|
-
/**
|
|
2794
|
-
* How long the replayed function took on this run, in ms. Null if the item
|
|
2795
|
-
* failed before it ran. Compare against
|
|
2796
|
-
* {@link ReplayItem.originalDurationMs} for the before/after.
|
|
2797
|
-
*/
|
|
2798
|
-
durationMs: number | null;
|
|
2799
|
-
/** The original trace's duration in ms, or null if its timestamps are missing. */
|
|
2800
|
-
originalDurationMs: number | null;
|
|
2801
|
-
/**
|
|
2802
|
-
* Token usage recorded on the original trace, or null if it captured none.
|
|
2803
|
-
* The "old" side of a token delta; {@link ReplayItem.tokens} is the new one.
|
|
2804
|
-
*/
|
|
2805
|
-
originalTokens: TokenUsage | null;
|
|
2806
|
-
/** Model name from the original trace, or null if not captured. */
|
|
2807
|
-
originalModel: string | null;
|
|
2808
|
-
/**
|
|
2809
|
-
* Token usage from the REPLAYED run (this item's new execution), aggregated
|
|
2810
|
-
* server-side from the spans it produced, or null if the run captured no
|
|
2811
|
-
* token data. Compare against {@link ReplayItem.originalTokens} to see how
|
|
2812
|
-
* the code change moved cost. Matches what the experiments page shows.
|
|
2813
|
-
*
|
|
2814
|
-
* Unprefixed because the replay is this item's subject: anything describing
|
|
2815
|
-
* the trace being replayed carries `original`.
|
|
2816
|
-
*/
|
|
2817
|
-
tokens: TokenUsage | null;
|
|
2818
|
-
/** @deprecated renamed to {@link ReplayItem.originalModel}. */
|
|
2819
|
-
model: string | null;
|
|
2820
|
-
/**
|
|
2821
|
-
* The DB snapshot ref the SDK captured at trace open. Useful for debugging
|
|
2822
|
-
* ("what state was this trace pinned to?") and for customers building
|
|
2823
|
-
* their own resolvers. Undefined when the source trace was captured
|
|
2824
|
-
* without `dbSnapshot` configured.
|
|
2825
|
-
*/
|
|
2826
|
-
dbSnapshotRef: DbSnapshotRef | null;
|
|
2827
|
-
/**
|
|
2828
|
-
* How long this item's DB branch took to provision, per phase, measured
|
|
2829
|
-
* server-side. Present whenever a branch was attempted, on both outcomes:
|
|
2830
|
-
* complete on success, partial up to the failing phase when the resolve
|
|
2831
|
-
* failed. Null when no branch was asked for, when the source trace carried
|
|
2832
|
-
* no snapshot ref, or against a server that predates timings.
|
|
2833
|
-
*/
|
|
2834
|
-
dbBranchTimings: DbBranchTimings | null;
|
|
2835
|
-
carriedOver?: boolean;
|
|
2836
|
-
}
|
|
2837
|
-
|
|
2838
|
-
interface ReplayResult<T> {
|
|
2839
|
-
/** Individual replay items with inputs, results, and comparison data. */
|
|
2840
|
-
items: ReplayItem<T>[];
|
|
2841
|
-
/**
|
|
2842
|
-
* The experiment ID created on the server. Null for a dry run, which
|
|
2843
|
-
* executes nothing and so creates no experiment.
|
|
2844
|
-
*/
|
|
2845
|
-
experimentId: string | null;
|
|
2846
|
-
/** Full URL to view the experiment in the dashboard. Null for a dry run. */
|
|
2847
|
-
experimentUrl: string | null;
|
|
2848
|
-
/** @deprecated Use {@link experimentId}. Same value. */
|
|
2849
|
-
testRunId: string | null;
|
|
2850
|
-
/** @deprecated Use {@link experimentUrl}. Same value. */
|
|
2851
|
-
testRunUrl: string | null;
|
|
2852
|
-
attempts: number;
|
|
2853
|
-
}
|
|
2854
|
-
/**
|
|
2855
|
-
* Whole-run replay failure that preserves every item collected before the run
|
|
2856
|
-
* failed. The original whole-run exception is available as `cause`.
|
|
2857
|
-
*/
|
|
2858
|
-
declare class ReplayError<T = unknown> extends BitfabError {
|
|
2859
|
-
readonly items: ReplayItem<T>[];
|
|
2860
|
-
readonly experimentId: string;
|
|
2861
|
-
readonly experimentUrl: string;
|
|
2862
|
-
readonly cause: unknown;
|
|
2863
|
-
/** @deprecated Use {@link experimentId}. Same value. */
|
|
2864
|
-
readonly testRunId: string;
|
|
2865
|
-
/** @deprecated Use {@link experimentUrl}. Same value. */
|
|
2866
|
-
readonly testRunUrl: string;
|
|
2867
|
-
constructor(message: string, items: ReplayItem<T>[], experimentId: string, experimentUrl: string, cause: unknown);
|
|
2868
|
-
}
|
|
2869
|
-
/**
|
|
2870
|
-
* A database branch requested for one replay item could not be resolved.
|
|
2871
|
-
*
|
|
2872
|
-
* The server's resolver code and message are retained as structured fields so
|
|
2873
|
-
* callers can handle failures such as `branch_create_failed` and
|
|
2874
|
-
* `snapshot_from_replaced_origin` without parsing the compatible `item.error`
|
|
2875
|
-
* string.
|
|
2876
|
-
*/
|
|
2877
|
-
declare class DbBranchReplayError extends BitfabError {
|
|
2878
|
-
readonly code: string;
|
|
2879
|
-
readonly originalTraceId: string;
|
|
2880
|
-
readonly cause?: unknown | undefined;
|
|
2881
|
-
constructor(code: string, message: string, originalTraceId: string, cause?: unknown | undefined);
|
|
2882
|
-
}
|
|
2883
|
-
/**
|
|
2884
|
-
* Serialize a replay result as JSON while retaining structured trace and replay
|
|
2885
|
-
* errors. Use this for direct-run stdout instead of raw `JSON.stringify`, which
|
|
2886
|
-
* drops the useful fields on JavaScript `Error` objects.
|
|
2887
|
-
*/
|
|
2888
|
-
declare function serializeReplayResult<T>(result: ReplayResult<T>): string;
|
|
2889
|
-
|
|
2890
|
-
/**
|
|
2891
|
-
* The database branch a single replay item runs against.
|
|
2892
|
-
*
|
|
2893
|
-
* `getCurrentReplayBranch()` hands you one inside a replayed function when the
|
|
2894
|
-
* source trace carried a DB snapshot reference and the Bitfab service resolved
|
|
2895
|
-
* a branch from it. Outside a replay item, or when no branch was resolved, that
|
|
2896
|
-
* accessor returns null and your code keeps reading `process.env.DATABASE_URL`
|
|
2897
|
-
* the normal way.
|
|
2898
|
-
*
|
|
2899
|
-
* Immutable and scoped to one item: the accessor builds it from the replay
|
|
2900
|
-
* AsyncLocalStorage context, so parallel replay items each see their own branch
|
|
2901
|
-
* and no lease state lives on a long-lived object.
|
|
2902
|
-
*
|
|
2903
|
-
* Internally the resolved per-item state is a `DbBranchLease` (see
|
|
2904
|
-
* replayContext.ts), the SDK/server protocol term. Its useful fields are
|
|
2905
|
-
* exposed directly here so customer code never sees the word.
|
|
2906
|
-
*/
|
|
2907
|
-
|
|
2908
|
-
declare class ReplayBranch implements Omit<DbBranchLease, "databaseUrl"> {
|
|
2909
|
-
#private;
|
|
2910
|
-
/** The provider's own id for this branch, e.g. for correlating with its console. */
|
|
2911
|
-
readonly neonBranchId: string;
|
|
2912
|
-
/** Env var name the customer's app reads, e.g. `DATABASE_URL`. */
|
|
2913
|
-
readonly envKey: string;
|
|
2914
|
-
/** When this branch's URL stops being valid. ISO-8601. */
|
|
2915
|
-
readonly expiresAt: string;
|
|
2916
|
-
/**
|
|
2917
|
-
* The instant this branch is pinned to: the source trace's wall clock, read
|
|
2918
|
-
* just before the traced function ran. Compare it against the trace you meant
|
|
2919
|
-
* to replay to confirm the branch is the right point in history.
|
|
2920
|
-
*/
|
|
2921
|
-
readonly snapshotTimestamp?: string;
|
|
2922
|
-
/** Deep link to the branch in the provider console, if available. */
|
|
2923
|
-
readonly providerConsoleUrl?: string;
|
|
2924
|
-
/**
|
|
2925
|
-
* True if the branch is read-only. Use it to skip write operations during
|
|
2926
|
-
* replay when the provider returned a read-only lease.
|
|
2927
|
-
*/
|
|
2928
|
-
readonly readOnly?: boolean;
|
|
2929
|
-
/**
|
|
2930
|
-
* The branch's region, e.g. `aws-us-east-1`. A compute runs in its project's
|
|
2931
|
-
* region, so a replay runner elsewhere pays that round trip on every query.
|
|
2932
|
-
*/
|
|
2933
|
-
readonly region?: string;
|
|
2934
|
-
/** The historical trace ID that produced the input for this replay item. */
|
|
2935
|
-
readonly traceId: string;
|
|
2936
|
-
/** @internal Built by `getCurrentReplayBranch()`; never constructed by callers. */
|
|
2937
|
-
constructor(lease: DbBranchLease, traceId: string, context: ReplayContext);
|
|
2938
|
-
/**
|
|
2939
|
-
* Connection string for this item's branch. Point your database client at it
|
|
2940
|
-
* instead of the live database for the duration of the replayed call.
|
|
2941
|
-
*
|
|
2942
|
-
* Reading it records on the trace that the replayed code obtained the branch
|
|
2943
|
-
* URL, which is what separates "a branch was provisioned" from "the branch
|
|
2944
|
-
* was actually used". The other fields inspect the lease without exposing the
|
|
2945
|
-
* connection string, so they deliberately do not record anything. That is
|
|
2946
|
-
* also why this is a getter and not a plain field: the URL is absent from
|
|
2947
|
-
* `JSON.stringify(branch)` and from logging the object.
|
|
2948
|
-
*/
|
|
2949
|
-
get databaseUrl(): string;
|
|
2950
|
-
}
|
|
2951
|
-
|
|
2952
|
-
declare const RIVET_GROUP_EFFECT_METHODS: {
|
|
2953
|
-
readonly db: readonly ["execute", "nativeMetrics", "close"];
|
|
2954
|
-
readonly kv: readonly ["get", "put", "delete", "batchPut", "batchGet", "batchDelete", "deleteRange", "listPrefix", "listRange", "list"];
|
|
2955
|
-
readonly schedule: readonly ["after", "at", "cancel", "get", "list"];
|
|
2956
|
-
readonly cron: readonly ["set", "every", "get", "list", "delete", "history"];
|
|
2957
|
-
readonly queue: readonly ["send", "next", "nextBatch", "waitForNames", "waitForAvailable", "complete", "enqueueAndWait", "tryNext", "tryNextBatch"];
|
|
2958
|
-
readonly run: readonly ["setWakeAt"];
|
|
2959
|
-
};
|
|
2960
|
-
declare const RIVET_TOP_LEVEL_EFFECTS: readonly ["broadcast", "saveState", "sleep", "destroy", "step", "tryStep", "try", "loop", "sleepUntil", "rollbackCheckpoint", "join", "race", "removed", "getVersion"];
|
|
2961
|
-
type RivetGroup = keyof typeof RIVET_GROUP_EFFECT_METHODS;
|
|
2962
|
-
type RivetGroupEffectType = {
|
|
2963
|
-
[TGroup in RivetGroup]: `${TGroup}.${(typeof RIVET_GROUP_EFFECT_METHODS)[TGroup][number]}`;
|
|
2964
|
-
}[RivetGroup];
|
|
2965
|
-
/** Rivet context operations supported as replay effects. */
|
|
2966
|
-
type RivetEffectType = RivetGroupEffectType | (typeof RIVET_TOP_LEVEL_EFFECTS)[number] | "db.transaction.execute" | "queue.message.complete";
|
|
2967
|
-
/** Configuration for {@link BitfabRivetIntegration}. */
|
|
2968
|
-
interface RivetIntegrationOptions {
|
|
2969
|
-
/**
|
|
2970
|
-
* Context operations whose recorded outputs replay should serve instead of
|
|
2971
|
-
* executing against a live actor or workflow. Nothing is marked by default.
|
|
2972
|
-
*/
|
|
2973
|
-
mockOnReplay?: readonly RivetEffectType[];
|
|
2974
|
-
}
|
|
2975
|
-
/** Minimal structural context type used when callers do not supply Rivet's type. */
|
|
2976
|
-
interface RivetContextLike {
|
|
2977
|
-
state: unknown;
|
|
2978
|
-
readonly db: {
|
|
2979
|
-
execute(...args: unknown[]): Promise<unknown>;
|
|
2980
|
-
};
|
|
2981
|
-
broadcast(name: string, ...args: unknown[]): void;
|
|
2982
|
-
readonly actorId?: string;
|
|
2983
|
-
readonly name?: string;
|
|
2984
|
-
readonly key?: readonly string[];
|
|
2985
|
-
readonly region?: string;
|
|
2986
|
-
}
|
|
2987
|
-
interface RivetIntegrationRuntime {
|
|
2988
|
-
wrapEffect<TArgs extends unknown[], TReturn>(effect: RivetEffectType, mockOnReplay: boolean, fn: (...args: TArgs) => TReturn, replayOutput?: () => unknown): (...args: TArgs) => TReturn;
|
|
2989
|
-
recordRootFixture(context: Record<string, unknown>, recordedRoots: Set<string>): void;
|
|
2990
|
-
}
|
|
2991
|
-
/**
|
|
2992
|
-
* Binds a live Rivet actor or workflow context to an existing traced function
|
|
2993
|
-
* and exposes replay-aware proxies for supported context operations.
|
|
2994
|
-
*
|
|
2995
|
-
* The integration never creates a trace root. Use it with the same
|
|
2996
|
-
* `BitfabFunction` handle whose `withTrace()` wrapper is the production and
|
|
2997
|
-
* replay root.
|
|
2998
|
-
*/
|
|
2999
|
-
declare class BitfabRivetIntegration<TContext extends object = RivetContextLike> {
|
|
3000
|
-
private readonly traceFunctionKey;
|
|
3001
|
-
private readonly runtime;
|
|
3002
|
-
private readonly markedEffects;
|
|
3003
|
-
private readonly replayProxies;
|
|
3004
|
-
/** @internal Constructed by `Bitfab.getRivetIntegration()`. */
|
|
3005
|
-
constructor(traceFunctionKey: string, runtime: RivetIntegrationRuntime, options?: RivetIntegrationOptions);
|
|
3006
|
-
/**
|
|
3007
|
-
* Run a callback with a live Rivet context bound to the current async chain.
|
|
3008
|
-
* This binds context only; it creates no trace or span.
|
|
3009
|
-
*/
|
|
3010
|
-
withContext<TReturn>(context: TContext, fn: () => TReturn): TReturn;
|
|
3011
|
-
/**
|
|
3012
|
-
* Adapt a traced function into a Rivet handler whose first argument is the
|
|
3013
|
-
* live actor or workflow context. The context is not added to the traced
|
|
3014
|
-
* root's inputs.
|
|
3015
|
-
*/
|
|
3016
|
-
bind<This, TArgs extends unknown[], TReturn>(fn: (this: This, ...args: TArgs) => TReturn): (this: This, context: TContext, ...args: TArgs) => TReturn;
|
|
3017
|
-
/**
|
|
3018
|
-
* Return the current Rivet context proxy. In production it delegates to the
|
|
3019
|
-
* bound live context. During replay it hydrates recorded actor state and
|
|
3020
|
-
* intercepts supported effects without requiring a live Rivet actor or
|
|
3021
|
-
* workflow.
|
|
3022
|
-
*/
|
|
3023
|
-
getContext(): TContext;
|
|
3024
|
-
private createContextProxy;
|
|
3025
|
-
private createDbProxy;
|
|
3026
|
-
private createGroupProxy;
|
|
3027
|
-
private createMethodProxy;
|
|
3028
|
-
private createEffect;
|
|
3029
|
-
private createTransaction;
|
|
3030
|
-
private decorateQueueMessages;
|
|
3031
|
-
private decorateQueueMessage;
|
|
3032
|
-
private queueCallIsCompletable;
|
|
3033
|
-
private createQueueIterator;
|
|
3034
|
-
}
|
|
3035
|
-
|
|
3036
|
-
type SpanListField = "input" | "output" | "reasoning" | "content" | "errors" | "contexts";
|
|
3037
|
-
type SpanListParams = {
|
|
3038
|
-
spanType?: SpanType;
|
|
3039
|
-
spanName?: string;
|
|
3040
|
-
fields?: SpanListField[];
|
|
3041
|
-
cursor?: string | null;
|
|
3042
|
-
/** Spans per page, 1 to 200, or at most 50 when `fields` is set. The server defaults to 50. */
|
|
3043
|
-
limit?: number;
|
|
3044
|
-
};
|
|
3045
|
-
type ListedSpan = {
|
|
3046
|
-
id: string;
|
|
3047
|
-
traceId: string;
|
|
3048
|
-
parentSpanId: string | null;
|
|
3049
|
-
name: string | null;
|
|
3050
|
-
type: string | null;
|
|
3051
|
-
startedAt: string | null;
|
|
3052
|
-
endedAt: string | null;
|
|
3053
|
-
input?: unknown;
|
|
3054
|
-
output?: unknown;
|
|
3055
|
-
reasoning?: unknown;
|
|
3056
|
-
content?: unknown;
|
|
3057
|
-
errors?: unknown;
|
|
3058
|
-
contexts?: unknown;
|
|
3059
|
-
};
|
|
3060
|
-
type SpanListResult = {
|
|
3061
|
-
spans: ListedSpan[];
|
|
3062
|
-
nextCursor: string | null;
|
|
3063
|
-
hasMore: boolean;
|
|
3064
|
-
};
|
|
3065
|
-
declare class SpansClient {
|
|
3066
|
-
private readonly httpClient;
|
|
3067
|
-
constructor(httpClient: HttpClient);
|
|
3068
|
-
/**
|
|
3069
|
-
* List one trace's spans in the order they ran, one page at a time. No
|
|
3070
|
-
* filter is required: omit `spanType` and `spanName` to page through every
|
|
3071
|
-
* span. Each span carries one extra key per requested `fields` entry. Pass
|
|
3072
|
-
* the returned `nextCursor` back as `cursor` for the next page. Throws a
|
|
3073
|
-
* `BitfabError` with status 404 when the trace is not in your organization.
|
|
3074
|
-
*/
|
|
3075
|
-
list(traceId: string, params?: SpanListParams): Promise<SpanListResult>;
|
|
3076
|
-
}
|
|
3077
|
-
|
|
3078
|
-
/**
|
|
3079
|
-
* Tracing utilities for external trace submission to Bitfab.
|
|
3080
|
-
*
|
|
3081
|
-
* This module provides utilities for sending external traces (e.g., from OpenAI API calls)
|
|
3082
|
-
* to Bitfab for monitoring and analysis.
|
|
3083
|
-
*/
|
|
3084
|
-
|
|
3085
|
-
interface Trace {
|
|
3086
|
-
traceId: string;
|
|
3087
|
-
toJSON(): unknown;
|
|
3088
|
-
}
|
|
3089
|
-
interface Span<_T = any> {
|
|
3090
|
-
spanId?: string;
|
|
3091
|
-
traceId?: string;
|
|
3092
|
-
parentId?: string | null;
|
|
3093
|
-
toJSON(): unknown;
|
|
3094
|
-
spanData?: {
|
|
3095
|
-
type?: string;
|
|
3096
|
-
_input?: unknown;
|
|
3097
|
-
_response?: unknown;
|
|
3098
|
-
} | null;
|
|
3099
|
-
}
|
|
3100
|
-
interface TraceResponse {
|
|
3101
|
-
traceId: string;
|
|
3102
|
-
status: "success";
|
|
3103
|
-
}
|
|
3104
|
-
interface ActiveSpanContext {
|
|
3105
|
-
traceId: string;
|
|
3106
|
-
spanId: string;
|
|
3107
|
-
}
|
|
3108
|
-
/**
|
|
3109
|
-
* TracingProcessor interface from OpenAI Agents SDK v0.3.7
|
|
3110
|
-
*/
|
|
3111
|
-
interface TracingProcessor {
|
|
3112
|
-
onTraceStart(trace: Trace): Promise<void>;
|
|
3113
|
-
onTraceEnd(trace: Trace): Promise<void>;
|
|
3114
|
-
onSpanStart(span: Span<any>): Promise<void>;
|
|
3115
|
-
onSpanEnd(span: Span<any>): Promise<void>;
|
|
3116
|
-
forceFlush(): Promise<void>;
|
|
3117
|
-
shutdown(timeout?: number): Promise<void>;
|
|
3118
|
-
}
|
|
3119
|
-
/**
|
|
3120
|
-
* Tracing processor for OpenAI Agents SDK integration.
|
|
3121
|
-
*
|
|
3122
|
-
* Implements the TracingProcessor interface from the OpenAI Agents SDK to
|
|
3123
|
-
* automatically capture traces and spans and send them to Bitfab for
|
|
3124
|
-
* monitoring and analysis.
|
|
3125
|
-
*
|
|
3126
|
-
* Example usage:
|
|
3127
|
-
* ```typescript
|
|
3128
|
-
* import { Bitfab } from 'bitfab';
|
|
3129
|
-
* import { addTraceProcessor } from '@openai/agents';
|
|
3130
|
-
*
|
|
3131
|
-
* const client = new Bitfab({ apiKey: 'your-api-key' });
|
|
3132
|
-
* const processor = client.getOpenAiTracingProcessor();
|
|
3133
|
-
* addTraceProcessor(processor);
|
|
3134
|
-
* ```
|
|
3135
|
-
*/
|
|
3136
|
-
declare class BitfabOpenAITracingProcessor implements TracingProcessor {
|
|
3137
|
-
private readonly httpClient;
|
|
3138
|
-
private readonly ownsHttpClient;
|
|
3139
|
-
private activeTraces;
|
|
3140
|
-
private readonly getActiveSpanContext;
|
|
3141
|
-
private readonly shouldRecord;
|
|
3142
|
-
private activeSpanMappings;
|
|
3143
|
-
private canonicalTraceIds;
|
|
3144
|
-
private getCanonicalTraceId;
|
|
3145
|
-
/**
|
|
3146
|
-
* Initialize the tracing processor.
|
|
3147
|
-
*
|
|
3148
|
-
* @param config - Configuration options
|
|
3149
|
-
*/
|
|
3150
|
-
constructor(config: {
|
|
3151
|
-
apiKey?: ApiKeyInput;
|
|
3152
|
-
serviceUrl?: string;
|
|
3153
|
-
timeout?: number;
|
|
3154
|
-
getActiveSpanContext?: () => ActiveSpanContext | null;
|
|
3155
|
-
shouldRecord?: () => boolean;
|
|
3156
|
-
/**
|
|
3157
|
-
* The owning `Bitfab` client's HTTP client. Supplied by
|
|
3158
|
-
* `getOpenAiTracingProcessor()` so this processor shares that client's
|
|
3159
|
-
* single span-transport worker instead of starting a second one.
|
|
3160
|
-
* @internal
|
|
3161
|
-
*/
|
|
3162
|
-
_httpClient?: HttpClient;
|
|
3163
|
-
});
|
|
3164
|
-
/**
|
|
3165
|
-
* Flush and release the span transport this processor started. A no-op when
|
|
3166
|
-
* the processor borrowed a `Bitfab` client's HTTP client: that client's
|
|
3167
|
-
* `close()` owns the worker's lifetime.
|
|
3168
|
-
*/
|
|
3169
|
-
close(timeoutMs?: number): Promise<boolean>;
|
|
3170
|
-
/**
|
|
3171
|
-
* Called when a trace is started.
|
|
3172
|
-
* If there's an active withSpan context, the trace ID is remapped to the
|
|
3173
|
-
* outer trace and sent to pre-create the external_traces row on the server.
|
|
3174
|
-
*/
|
|
3175
|
-
onTraceStart(trace: Trace): Promise<void>;
|
|
3176
|
-
/**
|
|
3177
|
-
* Called when a trace is ended.
|
|
3178
|
-
* If mapped to a withSpan trace, sends with remapped ID and completed=false
|
|
3179
|
-
* since the parent withSpan handles completion.
|
|
3180
|
-
*/
|
|
3181
|
-
onTraceEnd(trace: Trace): Promise<void>;
|
|
3182
|
-
private readonly startedSpans;
|
|
3183
|
-
private readonly endedTraces;
|
|
3184
|
-
private cleanupTrace;
|
|
3185
|
-
/**
|
|
3186
|
-
* Called when a span is started. Span payloads are authoritative completed
|
|
3187
|
-
* snapshots, so start notifications do not cross the transport boundary.
|
|
3188
|
-
*/
|
|
3189
|
-
onSpanStart(span: Span<any>): Promise<void>;
|
|
3190
|
-
/**
|
|
3191
|
-
* Called when a span is ended.
|
|
3192
|
-
*
|
|
3193
|
-
* Send the finalized span snapshot to Bitfab for complete trace capture.
|
|
3194
|
-
*/
|
|
3195
|
-
onSpanEnd(span: Span<any>): Promise<void>;
|
|
3196
|
-
/**
|
|
3197
|
-
* Called when a trace is being flushed.
|
|
3198
|
-
*/
|
|
3199
|
-
forceFlush(): Promise<void>;
|
|
3200
|
-
/**
|
|
3201
|
-
* Called when the trace processor is shutting down.
|
|
3202
|
-
*/
|
|
3203
|
-
shutdown(timeout?: number): Promise<void>;
|
|
3204
|
-
/**
|
|
3205
|
-
* Send trace to Bitfab API (fire-and-forget).
|
|
3206
|
-
* When traceIdOverride is provided, the trace ID is remapped to link
|
|
3207
|
-
* the OpenAI trace into an outer withSpan trace.
|
|
3208
|
-
*/
|
|
3209
|
-
private sendTrace;
|
|
3210
|
-
/**
|
|
3211
|
-
* Export span to JSON object, collecting any errors.
|
|
3212
|
-
*/
|
|
3213
|
-
private exportSpan;
|
|
3214
|
-
/**
|
|
3215
|
-
* Extract and add input/response to serialized span, updating errors list.
|
|
3216
|
-
*/
|
|
3217
|
-
private extractSpanInputResponse;
|
|
3218
|
-
/**
|
|
3219
|
-
* If the span's trace is mapped to a withSpan trace, rewrite trace_id and parent_id.
|
|
3220
|
-
*/
|
|
3221
|
-
private applySpanOverrides;
|
|
3222
|
-
/**
|
|
3223
|
-
* Build span payload for the external spans API.
|
|
3224
|
-
*/
|
|
3225
|
-
private buildSpanPayload;
|
|
3226
|
-
/**
|
|
3227
|
-
* Send span to Bitfab API (fire-and-forget).
|
|
3228
|
-
* If the span belongs to a trace mapped to a withSpan trace, the trace_id
|
|
3229
|
-
* and parent_id are rewritten to link the span into the withSpan tree.
|
|
3230
|
-
*/
|
|
3231
|
-
private sendSpan;
|
|
3232
|
-
}
|
|
3233
|
-
|
|
3234
|
-
type CaptureSurface = "opt-in" | "opt-out";
|
|
3235
|
-
|
|
3236
|
-
/**
|
|
3237
|
-
* Options for wrapBAML.
|
|
3238
|
-
*/
|
|
3239
|
-
interface WrapBAMLOptions {
|
|
3240
|
-
/** Called after each BAML invocation with the Collector instance. */
|
|
3241
|
-
onCollector?: (collector: unknown) => void;
|
|
3242
|
-
}
|
|
3243
|
-
/**
|
|
3244
|
-
* A function returned by wrapBAML that exposes the BAML collector from the last call.
|
|
3245
|
-
*/
|
|
3246
|
-
interface WrappedBamlFn<TArgs extends unknown[], TReturn> {
|
|
3247
|
-
(...args: TArgs): Promise<TReturn>;
|
|
3248
|
-
/** The BAML Collector instance from the most recent call. `null` before the first call or if @boundaryml/baml is unavailable. */
|
|
3249
|
-
collector: unknown | null;
|
|
3250
|
-
}
|
|
3251
|
-
/**
|
|
3252
|
-
* A handle to the current active span, allowing context to be added.
|
|
3253
|
-
*/
|
|
3254
|
-
interface CurrentSpan {
|
|
3255
|
-
/** The Bitfab ID for the current span. */
|
|
3256
|
-
readonly id: string;
|
|
3257
|
-
/** The trace ID for the current span. */
|
|
3258
|
-
readonly traceId: string;
|
|
3259
|
-
/**
|
|
3260
|
-
* Add a context entry to this span. Each call appends to the contexts array.
|
|
3261
|
-
* Context entries are stored in span_data.contexts as [{key, value}, ...].
|
|
3262
|
-
*/
|
|
3263
|
-
addContext(context: Record<string, unknown>): void;
|
|
3264
|
-
/**
|
|
3265
|
-
* Set the prompt for this span. Stored in span_data.prompt.
|
|
3266
|
-
* Calling multiple times overwrites the previous value.
|
|
3267
|
-
*/
|
|
3268
|
-
setPrompt(prompt: string): void;
|
|
3269
|
-
}
|
|
3270
|
-
/**
|
|
3271
|
-
* A detached handle to a previously-created trace, looked up by its
|
|
3272
|
-
* canonical Bitfab trace ID.
|
|
3273
|
-
*
|
|
3274
|
-
* Unlike `getCurrentTrace()`, this handle is not tied to AsyncLocalStorage -
|
|
3275
|
-
* each method sends to the server immediately. Useful for adding context
|
|
3276
|
-
* to a trace from a different process, request, or thread (e.g. a forked
|
|
3277
|
-
* agent that wants to annotate the original conversation's trace). Its methods
|
|
3278
|
-
* write whether or not capture is on.
|
|
3279
|
-
*/
|
|
3280
|
-
interface DetachedTrace {
|
|
3281
|
-
/** The canonical Bitfab trace ID this handle resolves. */
|
|
3282
|
-
readonly traceId: string;
|
|
3283
|
-
/**
|
|
3284
|
-
* Append a context entry to this trace. Each call adds one entry to the
|
|
3285
|
-
* server-side contexts array; existing entries are preserved.
|
|
3286
|
-
*
|
|
3287
|
-
* Resolves once the server has applied the change and REJECTS if the server
|
|
3288
|
-
* refused it. A detached patch targets an already-closed trace, so it rides
|
|
3289
|
-
* no batch and no later signal would reveal a silent failure - the caller is
|
|
3290
|
-
* the only one who can react.
|
|
3291
|
-
*/
|
|
3292
|
-
addContext(context: Record<string, unknown>): Promise<void>;
|
|
3293
|
-
/**
|
|
3294
|
-
* Merge metadata into this trace. Server-side shallow-merges the new keys
|
|
3295
|
-
* into the existing metadata object; existing keys are preserved unless
|
|
3296
|
-
* overwritten by the new values. Rejects if the server refused the update.
|
|
3297
|
-
*/
|
|
3298
|
-
setMetadata(metadata: Record<string, unknown>): Promise<void>;
|
|
3299
|
-
/**
|
|
3300
|
-
* Set the sessionId for this trace. Replaces any existing sessionId.
|
|
3301
|
-
* Rejects if the server refused the update.
|
|
3302
|
-
*/
|
|
3303
|
-
setSessionId(sessionId: string): Promise<void>;
|
|
3304
|
-
setName(name: string): Promise<void>;
|
|
3305
|
-
}
|
|
3306
|
-
/**
|
|
3307
|
-
* A handle to the current active trace, allowing trace-level context to be set.
|
|
3308
|
-
*/
|
|
3309
|
-
interface CurrentTrace {
|
|
3310
|
-
/**
|
|
3311
|
-
* Set the session ID for this trace. Stored in the database session_id column.
|
|
3312
|
-
*/
|
|
3313
|
-
setSessionId(sessionId: string): void;
|
|
3314
|
-
setName(name: string): void;
|
|
3315
|
-
/**
|
|
3316
|
-
* Set metadata for this trace. Stored in rawData.metadata.
|
|
3317
|
-
* Subsequent calls merge with existing metadata, with later values taking precedence.
|
|
3318
|
-
*/
|
|
3319
|
-
setMetadata(metadata: Record<string, unknown>): void;
|
|
3320
|
-
/**
|
|
3321
|
-
* Add a context entry to this trace. Each call appends to the contexts array.
|
|
3322
|
-
* Context entries are stored in rawData.contexts as [{key, value}, ...].
|
|
3323
|
-
*/
|
|
3324
|
-
addContext(context: Record<string, unknown>): void;
|
|
3325
|
-
/**
|
|
3326
|
-
* Flag this trace to be dropped. Once flagged, spans that complete afterward
|
|
3327
|
-
* are not uploaded at all, and when the trace completes the server scrubs any
|
|
3328
|
-
* payloads that already raced out (trace, external trace, and sibling spans),
|
|
3329
|
-
* marking it `dropped` instead of `completed` and retaining only a skeleton
|
|
3330
|
-
* audit record. Use this to discard runs you never want stored (e.g. health
|
|
3331
|
-
* checks, or a run you know contains sensitive data). Takes effect
|
|
3332
|
-
* immediately for later spans; the server-side scrub takes effect at trace
|
|
3333
|
-
* completion, so a trace that is flagged but never completes is not scrubbed.
|
|
3334
|
-
*/
|
|
3335
|
-
drop(): void;
|
|
3336
|
-
}
|
|
3337
|
-
/**
|
|
3338
|
-
* Get a handle to the current active span.
|
|
3339
|
-
*
|
|
3340
|
-
* Call this from inside a traced function (wrapped with `withSpan`) to get
|
|
3341
|
-
* a span handle that allows adding context at runtime.
|
|
3342
|
-
*
|
|
3343
|
-
* Returns a no-op object if called outside of a span context (methods do nothing).
|
|
3344
|
-
*/
|
|
3345
|
-
declare function getCurrentSpan(): CurrentSpan;
|
|
3346
|
-
/**
|
|
3347
|
-
* Get the database branch the current replay item is running against.
|
|
3348
|
-
*
|
|
3349
|
-
* Call this from inside a function being replayed with `replay({ dbBranch })`
|
|
3350
|
-
* and point your database client at `branch.databaseUrl` so the replay reads
|
|
3351
|
-
* the data as it was at trace time:
|
|
3352
|
-
*
|
|
3353
|
-
* ```ts
|
|
3354
|
-
* const branch = getCurrentReplayBranch()
|
|
3355
|
-
* const url = branch?.databaseUrl ?? process.env.DATABASE_URL
|
|
3356
|
-
* ```
|
|
3357
|
-
*
|
|
3358
|
-
* Returns null outside a replay item, and for an item whose source trace
|
|
3359
|
-
* carried no DB snapshot reference, so live request code takes the same path
|
|
3360
|
-
* it always did.
|
|
3361
|
-
*/
|
|
3362
|
-
declare function getCurrentReplayBranch(): ReplayBranch | null;
|
|
3363
|
-
/**
|
|
3364
|
-
* Get a handle to the current active trace.
|
|
3365
|
-
*
|
|
3366
|
-
* Call this from inside a traced function (wrapped with `withSpan`) to get
|
|
3367
|
-
* a trace handle that allows setting trace-level context at runtime.
|
|
3368
|
-
*
|
|
3369
|
-
* Returns a no-op object if called outside of a span context (methods do nothing).
|
|
3370
|
-
*/
|
|
3371
|
-
declare function getCurrentTrace(): CurrentTrace;
|
|
3372
|
-
interface SeedCaseOptions {
|
|
3373
|
-
input: unknown[];
|
|
3374
|
-
expected?: unknown;
|
|
3375
|
-
fn?: (...args: any[]) => unknown;
|
|
3376
|
-
metadata?: Record<string, unknown>;
|
|
3377
|
-
sessionId?: string;
|
|
3378
|
-
name?: string;
|
|
3379
|
-
spanName?: string;
|
|
3380
|
-
spanType?: SpanType;
|
|
3381
|
-
}
|
|
3382
|
-
interface SeedRunOptions<TArgs extends unknown[]> {
|
|
3383
|
-
args?: TArgs;
|
|
3384
|
-
metadata?: Record<string, unknown>;
|
|
3385
|
-
sessionId?: string;
|
|
3386
|
-
name?: string;
|
|
3387
|
-
}
|
|
3388
|
-
interface ReseedOptions {
|
|
3389
|
-
traceId: string;
|
|
3390
|
-
}
|
|
3391
|
-
interface ReseedResult {
|
|
3392
|
-
traceId: string;
|
|
3393
|
-
previousRunTraceId: string;
|
|
3394
|
-
}
|
|
3395
|
-
interface BitfabConfig {
|
|
3396
|
-
/**
|
|
3397
|
-
* The API key for Bitfab API authentication. Resolved lazily, the first
|
|
3398
|
-
* time a span actually needs it, not at construction. When it resolves
|
|
3399
|
-
* empty, tracing is disabled (a no-op, unless `strict` is set).
|
|
3400
|
-
*
|
|
3401
|
-
* Accepts either a string or a function returning the key. The function
|
|
3402
|
-
* form is resolved at first use, so it survives the ESM trap where a shim
|
|
3403
|
-
* built at module load runs before the script body's `dotenv.config()`:
|
|
3404
|
-
* `apiKey: () => process.env.BITFAB_API_KEY`. When omitted (or it resolves
|
|
3405
|
-
* empty), the SDK also falls back to reading `BITFAB_API_KEY` from the
|
|
3406
|
-
* environment itself, again at first use.
|
|
3407
|
-
*/
|
|
3408
|
-
apiKey?: string | (() => string | null | undefined);
|
|
3409
|
-
/** The base URL for the Bitfab API (default: https://bitfab.ai) */
|
|
3410
|
-
serviceUrl?: string;
|
|
3411
|
-
/** Request timeout in milliseconds (default: 120000) */
|
|
3412
|
-
timeout?: number;
|
|
3413
|
-
/** Environment variables for LLM provider API keys (only OPENAI_API_KEY is supported) */
|
|
3414
|
-
envVars?: AllowedEnvVars;
|
|
3415
|
-
captureEnabled?: boolean;
|
|
3416
|
-
enabled?: boolean;
|
|
3417
|
-
/**
|
|
3418
|
-
* Fail loud instead of degrading quietly. When true, the first traced call
|
|
3419
|
-
* with no resolvable API key throws a `BitfabError` rather than silently
|
|
3420
|
-
* disabling tracing. Off by default so a missing telemetry key never takes
|
|
3421
|
-
* down the host app; turn it on in standalone scripts where a run that
|
|
3422
|
-
* emits no traces is a failure you want surfaced immediately.
|
|
3423
|
-
*/
|
|
3424
|
-
strict?: boolean;
|
|
3425
|
-
/** The generated BAML client instance (e.g., `b` from your baml_client). Used by wrapBAML() when no explicit client is passed. */
|
|
3426
|
-
bamlClient?: unknown;
|
|
3427
|
-
/**
|
|
3428
|
-
* Per-trace database snapshot config. When set, every root span captures
|
|
3429
|
-
* a wall-clock timestamp (and, if `captureRef` is provided, a provider-
|
|
3430
|
-
* specific point-in-time ref) so the trace can later be replayed against
|
|
3431
|
-
* a branch materialized from that point.
|
|
3432
|
-
*/
|
|
3433
|
-
dbSnapshot?: DbSnapshotConfig;
|
|
3434
|
-
simulationPlan?: boolean;
|
|
3435
|
-
}
|
|
3436
|
-
|
|
3437
|
-
/**
|
|
3438
|
-
* Controls when a span is captured.
|
|
3439
|
-
* - always: Capture the span even when it becomes the root of a new trace.
|
|
3440
|
-
* - nested: Capture the span only when another Bitfab span is already active.
|
|
3441
|
-
*/
|
|
3442
|
-
type CaptureWhen = "always" | "nested";
|
|
3443
|
-
/**
|
|
3444
|
-
* Options for configuring span behavior.
|
|
3445
|
-
*/
|
|
3446
|
-
interface SpanOptions {
|
|
3447
|
-
/**
|
|
3448
|
-
* The name of the span. Defaults to the function name if available,
|
|
3449
|
-
* otherwise falls back to the trace function key.
|
|
3450
|
-
*/
|
|
3451
|
-
name?: string;
|
|
3452
|
-
/**
|
|
3453
|
-
* The type of span. Defaults to "custom" if not specified.
|
|
3454
|
-
*/
|
|
3455
|
-
type?: SpanType;
|
|
3456
|
-
/**
|
|
3457
|
-
* Controls whether this span may start a new trace. Defaults to "always".
|
|
3458
|
-
*
|
|
3459
|
-
* Use "nested" for reusable helpers that should appear inside an existing
|
|
3460
|
-
* trace but should run untraced when called on their own.
|
|
3461
|
-
* Unknown values warn once and default to "always".
|
|
3462
|
-
*/
|
|
3463
|
-
captureWhen?: CaptureWhen;
|
|
3464
|
-
/**
|
|
3465
|
-
* When true, replay will reuse this span's historical output instead of
|
|
3466
|
-
* executing the wrapped function. Read by the "marked" replay strategy;
|
|
3467
|
-
* ignored outside replay and under the "all"/"none" strategies.
|
|
3468
|
-
*
|
|
3469
|
-
* Use this for child spans that are expensive (paid LLM/API calls),
|
|
3470
|
-
* slow, or non-deterministic - the root function still runs real code,
|
|
3471
|
-
* only the marked descendants return their recorded output. If a selected
|
|
3472
|
-
* occurrence is unavailable, replay fails the item without executing the
|
|
3473
|
-
* real child.
|
|
3474
|
-
*/
|
|
3475
|
-
mockOnReplay?: boolean;
|
|
3476
|
-
/**
|
|
3477
|
-
* Experimental selective-replay eligibility, recorded with this span.
|
|
3478
|
-
* Attests that the whole subtree is output-only with JSON value semantics:
|
|
3479
|
-
* no hidden mutable reads, required effects, input mutations, or reliance
|
|
3480
|
-
* on object identity. Include implementation/dependency changes in replay's
|
|
3481
|
-
* mustRun. Both capture-time and current declarations must allow reuse.
|
|
3482
|
-
* Does not affect ordinary replay or replace mockOnReplay safety protection.
|
|
3483
|
-
*/
|
|
3484
|
-
replayReusable?: boolean;
|
|
3485
|
-
/** Optional experiment ID included on the span and on the trace it starts. */
|
|
3486
|
-
experimentId?: string;
|
|
3487
|
-
/**
|
|
3488
|
-
* @deprecated Use {@link experimentId}. Same meaning. Passing both with
|
|
3489
|
-
* different values throws.
|
|
3490
|
-
*/
|
|
3491
|
-
testRunId?: string;
|
|
3492
|
-
/**
|
|
3493
|
-
* Record a serializable view of a non-serializable result (e.g. a live
|
|
3494
|
-
* stream object) as the span output.
|
|
3495
|
-
*
|
|
3496
|
-
* When set, the wrapped function's raw return value is handed back to the
|
|
3497
|
-
* caller unchanged (so streaming and first-byte latency are untouched),
|
|
3498
|
-
* but instead of serializing that raw value, the span records
|
|
3499
|
-
* `await finalize(result)`. Use this to trace functions that return a live
|
|
3500
|
-
* stream consumed by the caller (Vercel AI SDK `streamText`, a
|
|
3501
|
-
* `ReadableStream`, an SSE response) while still capturing a serializable,
|
|
3502
|
-
* replayable output such as `{ text, usage, toolCalls }`.
|
|
3503
|
-
*
|
|
3504
|
-
* Reading from a multi-consumer stream result (like the AI SDK's, which
|
|
3505
|
-
* tees internally) does not disturb the caller's own consumption. For the
|
|
3506
|
-
* Vercel AI SDK shape, pass the prebuilt `finalizers.aiSdk` helper.
|
|
3507
|
-
*
|
|
3508
|
-
* Ignored for async-generator results, which are captured automatically.
|
|
3509
|
-
*/
|
|
3510
|
-
finalize?: (result: any) => unknown | Promise<unknown>;
|
|
3511
|
-
}
|
|
3512
|
-
/**
|
|
3513
|
-
* The standard method context passed to a decorator.
|
|
3514
|
-
*
|
|
3515
|
-
* Defined structurally instead of referencing TypeScript's built-in
|
|
3516
|
-
* `ClassMethodDecoratorContext`, which was added in TypeScript 5.0. This keeps
|
|
3517
|
-
* the SDK's non-decorator APIs consumable by projects on older compilers.
|
|
3518
|
-
*/
|
|
3519
|
-
interface SpanMethodDecoratorContext<TThis, TValue> {
|
|
3520
|
-
readonly kind: "method";
|
|
3521
|
-
readonly name: string | symbol;
|
|
3522
|
-
readonly static: boolean;
|
|
3523
|
-
readonly private: boolean;
|
|
3524
|
-
readonly access: {
|
|
3525
|
-
has(object: TThis): boolean;
|
|
3526
|
-
get(object: TThis): TValue;
|
|
3527
|
-
};
|
|
3528
|
-
addInitializer(initializer: (this: TThis) => void): void;
|
|
3529
|
-
readonly metadata?: Record<PropertyKey, unknown>;
|
|
3530
|
-
}
|
|
3531
|
-
/** A standard ECMAScript method decorator produced by {@link Bitfab.span}. */
|
|
3532
|
-
type SpanMethodDecorator = <TThis, TArgs extends unknown[], TReturn>(originalMethod: (this: TThis, ...args: TArgs) => TReturn, context: SpanMethodDecoratorContext<TThis, (this: TThis, ...args: TArgs) => TReturn>) => (this: TThis, ...args: TArgs) => TReturn;
|
|
3533
|
-
/** Trace-owned configuration for a function discovered beneath `trace()`. */
|
|
3534
|
-
interface NodeOptions extends Omit<SpanOptions, "captureWhen"> {
|
|
3535
|
-
/** Whether the enclosing trace captures this call. Defaults to true. */
|
|
3536
|
-
capture?: boolean;
|
|
3537
|
-
}
|
|
3538
|
-
type StandardNodeMethodDecorator = SpanMethodDecorator;
|
|
3539
|
-
type LegacyNodeMethodDecorator = <TThis, TArgs extends unknown[], TReturn>(target: object, propertyKey: string | symbol, descriptor: TypedPropertyDescriptor<(this: TThis, ...args: TArgs) => TReturn>) => void;
|
|
3540
|
-
/** A method decorator produced by {@link Bitfab.node}. */
|
|
3541
|
-
type NodeMethodDecorator = StandardNodeMethodDecorator & LegacyNodeMethodDecorator;
|
|
3542
|
-
/** Options for experimental automatic subtree tracing. */
|
|
3543
|
-
interface TraceOptions {
|
|
3544
|
-
/** Root span name. Defaults to the decorated or wrapped function name. */
|
|
3545
|
-
name?: string;
|
|
3546
|
-
/** Root span type. Descendants are always `function` spans. */
|
|
3547
|
-
type?: SpanType;
|
|
3548
|
-
/**
|
|
3549
|
-
* Default replay-mocking policy for automatically captured descendants.
|
|
3550
|
-
* When true, those descendants are mocked by the default "marked" strategy
|
|
3551
|
-
* unless a node explicitly sets `mockOnReplay: false`. Defaults to false.
|
|
3552
|
-
*/
|
|
3553
|
-
mockOnReplayDefault?: boolean;
|
|
3554
|
-
/** Maximum number of recorded descendant levels. Defaults to 30. */
|
|
3555
|
-
maxDepth?: number;
|
|
3556
|
-
maxCapturedSubtreeSpans?: number;
|
|
3557
|
-
/** Qualified or simple function names to leave out of the subtree. */
|
|
3558
|
-
exclude?: readonly string[] | ReadonlySet<string>;
|
|
3559
|
-
/** Record rest-argument wrapper functions. Defaults to false. */
|
|
3560
|
-
includeWrappers?: boolean;
|
|
3561
|
-
}
|
|
3562
|
-
type StandardTraceMethodDecorator = <This, TArgs extends unknown[], TReturn>(method: (this: This, ...args: TArgs) => TReturn, context: {
|
|
3563
|
-
kind: "method";
|
|
3564
|
-
name: string | symbol;
|
|
3565
|
-
}) => (this: This, ...args: TArgs) => TReturn;
|
|
3566
|
-
type LegacyTraceMethodDecorator = <This, TArgs extends unknown[], TReturn>(target: object, propertyKey: string | symbol, descriptor: TypedPropertyDescriptor<(this: This, ...args: TArgs) => TReturn>) => void;
|
|
3567
|
-
type TraceMethodDecorator = StandardTraceMethodDecorator & LegacyTraceMethodDecorator;
|
|
3568
|
-
|
|
3569
|
-
/**
|
|
3570
|
-
* Client for making provider-based API calls via BAML.
|
|
3571
|
-
*/
|
|
3572
|
-
declare class Bitfab {
|
|
3573
|
-
private readonly apiKeyConfig;
|
|
3574
|
-
/** Cached only once a non-empty key is found, so an early resolve (before env loaded) can't poison a later one. */
|
|
3575
|
-
private resolvedApiKey;
|
|
3576
|
-
/** Gate the empty-key warning to fire at most once. */
|
|
3577
|
-
private apiKeyWarned;
|
|
3578
|
-
private readonly serviceUrl;
|
|
3579
|
-
private readonly timeout;
|
|
3580
|
-
private readonly envVars;
|
|
3581
|
-
private readonly captureConfiguredInCode;
|
|
3582
|
-
private captureConfiguredFromEnv;
|
|
3583
|
-
private readonly strict;
|
|
3584
|
-
private readonly httpClient;
|
|
3585
|
-
/** Dataset operations for the authenticated organization. */
|
|
3586
|
-
readonly datasets: DatasetsClient;
|
|
3587
|
-
/** Organization-scoped groupings for assertions and scenarios. */
|
|
3588
|
-
readonly categories: CategoriesClient;
|
|
3589
|
-
/** @deprecated Use {@link categories}. Same client. */
|
|
3590
|
-
readonly assertionCategories: CategoriesClient;
|
|
3591
|
-
readonly organizationMembers: OrganizationMembersClient;
|
|
3592
|
-
readonly traces: TracesClient;
|
|
3593
|
-
readonly spans: SpansClient;
|
|
3594
|
-
readonly labels: LabelsClient;
|
|
3595
|
-
readonly graders: GradersClient;
|
|
3596
|
-
readonly experiments: ExperimentsClient;
|
|
3597
|
-
private readonly bamlClient;
|
|
3598
|
-
private readonly dbSnapshot;
|
|
3599
|
-
private readonly simulationPlan;
|
|
3600
|
-
/**
|
|
3601
|
-
* Mock overrides registered via {@link Bitfab.registerMockOverride}, applied
|
|
3602
|
-
* to every `replay` on this client (after any per-call `mockOverride`). In
|
|
3603
|
-
* registration order; first matcher wins within this list.
|
|
3604
|
-
*/
|
|
3605
|
-
private readonly mockOverrides;
|
|
3606
|
-
/**
|
|
3607
|
-
* Initialize the Bitfab client.
|
|
3608
|
-
*
|
|
3609
|
-
* @param config - Configuration options for the client
|
|
3610
|
-
*/
|
|
3611
|
-
constructor(config: BitfabConfig);
|
|
3612
|
-
/**
|
|
3613
|
-
* Decorate a class method as an automatically expanded trace root.
|
|
3614
|
-
*
|
|
3615
|
-
* Build instrumentation turns repository functions called beneath this
|
|
3616
|
-
* method into nested spans that capture inputs, outputs, and errors by
|
|
3617
|
-
* default. Source exclusions and sim-plan capture-off decisions are explicit
|
|
3618
|
-
* opt-outs from that default.
|
|
3619
|
-
* Without a compatible build transform, this still records the decorated
|
|
3620
|
-
* method as a normal rich root span but cannot discover child calls.
|
|
3621
|
-
*
|
|
3622
|
-
* @param traceFunctionKey - Groups traces and their capture controls.
|
|
3623
|
-
* @param options - Root presentation, subtree bounds, and exclusions.
|
|
3624
|
-
* @experimental Automatic child-call instrumentation is experimental.
|
|
3625
|
-
*/
|
|
3626
|
-
trace(traceFunctionKey: string, options?: TraceOptions): TraceMethodDecorator;
|
|
3627
|
-
/**
|
|
3628
|
-
* Wrap a function as an automatically expanded trace root.
|
|
3629
|
-
*
|
|
3630
|
-
* This is the function-oriented equivalent of {@link Bitfab.trace}. Build
|
|
3631
|
-
* instrumentation turns repository functions called beneath the wrapped
|
|
3632
|
-
* function into nested spans that capture inputs, outputs, and errors by
|
|
3633
|
-
* default. Source exclusions and sim-plan capture-off decisions are explicit
|
|
3634
|
-
* opt-outs from that default. Without a compatible transform, this still
|
|
3635
|
-
* records one normal rich root span and runs the function unchanged.
|
|
3636
|
-
*
|
|
3637
|
-
* @param traceFunctionKey - Groups traces and their capture controls.
|
|
3638
|
-
* @param optionsOrFn - Options or the workflow entrypoint to wrap.
|
|
3639
|
-
* @param maybeFn - Workflow entrypoint when options are provided.
|
|
3640
|
-
* @experimental Automatic child-call instrumentation is experimental.
|
|
3641
|
-
*/
|
|
3642
|
-
withTrace<This, TArgs extends unknown[], TReturn>(traceFunctionKey: string, fn: (this: This, ...args: TArgs) => TReturn): (this: This, ...args: TArgs) => TReturn;
|
|
3643
|
-
withTrace<This, TArgs extends unknown[], TReturn>(traceFunctionKey: string, options: TraceOptions, fn: (this: This, ...args: TArgs) => TReturn): (this: This, ...args: TArgs) => TReturn;
|
|
3644
|
-
/**
|
|
3645
|
-
* Configure a transformed class method when it is discovered beneath a
|
|
3646
|
-
* {@link Bitfab.trace} root.
|
|
3647
|
-
*
|
|
3648
|
-
* The decorator creates no span or trace by itself. Beneath an active trace,
|
|
3649
|
-
* it can rename or retype the discovered call, capture its contents, mark it
|
|
3650
|
-
* for recorded-output replay, finalize its output, or omit it while leaving
|
|
3651
|
-
* captured descendants attached to the nearest captured parent.
|
|
3652
|
-
*
|
|
3653
|
-
* @param options - Trace-owned call configuration.
|
|
3654
|
-
* @experimental Automatic child-call instrumentation is experimental.
|
|
3655
|
-
*/
|
|
3656
|
-
node(options?: NodeOptions): NodeMethodDecorator;
|
|
3657
|
-
/**
|
|
3658
|
-
* Configure a transformed standalone function when it is discovered beneath
|
|
3659
|
-
* a {@link Bitfab.trace} or {@link Bitfab.withTrace} root.
|
|
3660
|
-
*
|
|
3661
|
-
* This is the function-oriented equivalent of {@link Bitfab.node}. Without
|
|
3662
|
-
* an active automatic trace, the returned function runs normally and never
|
|
3663
|
-
* creates a span or trace. The function must be named so configuration can
|
|
3664
|
-
* be bound to its transformed definition without leaking to a descendant.
|
|
3665
|
-
*
|
|
3666
|
-
* @param optionsOrFn - Node options or the function to configure.
|
|
3667
|
-
* @param maybeFn - Function to configure when options are provided.
|
|
3668
|
-
* @experimental Automatic child-call instrumentation is experimental.
|
|
3669
|
-
*/
|
|
3670
|
-
withNode<This, TArgs extends unknown[], TReturn>(fn: (this: This, ...args: TArgs) => TReturn): (this: This, ...args: TArgs) => TReturn;
|
|
3671
|
-
withNode<This, TArgs extends unknown[], TReturn>(options: NodeOptions, fn: (this: This, ...args: TArgs) => TReturn): (this: This, ...args: TArgs) => TReturn;
|
|
3672
|
-
private resolveNodeConfiguration;
|
|
3673
|
-
private createAutoTraceNode;
|
|
3674
|
-
private createAutoTraceRoot;
|
|
3675
|
-
private applySimulationPlan;
|
|
3676
|
-
/**
|
|
3677
|
-
* Flush and permanently close this client's tracing resources: its pending
|
|
3678
|
-
* requests and the single span-transport worker shared by its decorators and
|
|
3679
|
-
* framework handlers.
|
|
3680
|
-
*
|
|
3681
|
-
* Resolves `false` when delivery failed or the deadline expired. Long-lived
|
|
3682
|
-
* processes never need this (the transport batches in the background and the
|
|
3683
|
-
* exit hook drains it); scripts and tests that want a hard guarantee should
|
|
3684
|
-
* await it.
|
|
3685
|
-
*
|
|
3686
|
-
* Deliberately not a `Symbol.asyncDispose` method: the SDK targets runtimes
|
|
3687
|
-
* where that symbol may be absent, and a computed key on a missing symbol
|
|
3688
|
-
* throws at class-definition time, taking the whole SDK down on load.
|
|
3689
|
-
*/
|
|
3690
|
-
close(timeoutMs?: number): Promise<boolean>;
|
|
3691
|
-
/**
|
|
3692
|
-
* Resolve the API key lazily, the first time a span actually needs it.
|
|
3693
|
-
*
|
|
3694
|
-
* The key is intentionally NOT read at construction. In ESM, a shim that
|
|
3695
|
-
* does `new Bitfab({ apiKey: process.env.BITFAB_API_KEY })` is hoisted and
|
|
3696
|
-
* evaluated before the importing script's body runs `dotenv.config()`, so
|
|
3697
|
-
* the key would be empty at construction even though it is set moments
|
|
3698
|
-
* later. Resolving here (at first `withSpan` call / first request) reads
|
|
3699
|
-
* the key after env loading has run.
|
|
3700
|
-
*
|
|
3701
|
-
* Resolution order: the configured value (string, or function called each
|
|
3702
|
-
* time it is still unresolved), then a fallback read of `BITFAB_API_KEY`
|
|
3703
|
-
* from the environment. Once a non-empty key is found it is cached, so an
|
|
3704
|
-
* early resolve that found nothing never poisons a later one.
|
|
3705
|
-
*/
|
|
3706
|
-
private findApiKey;
|
|
3707
|
-
private resolveApiKey;
|
|
3708
|
-
private isCaptureConfigured;
|
|
3709
|
-
private isCaptureEnabled;
|
|
3710
|
-
private shouldRecord;
|
|
3711
|
-
private prepareFrameworkIntegration;
|
|
3712
|
-
get captureEnabled(): boolean;
|
|
3713
|
-
/**
|
|
3714
|
-
* Fetch the function with its current version and BAML prompt from the server.
|
|
3715
|
-
*
|
|
3716
|
-
* @param methodName - The name of the method to fetch
|
|
3717
|
-
* @returns The function with current version, BAML prompt, and provider definitions
|
|
3718
|
-
* @throws {BitfabError} If the function is not found or an error occurs
|
|
3719
|
-
*/
|
|
3720
|
-
private fetchFunctionVersion;
|
|
3721
|
-
/**
|
|
3722
|
-
* Call a method with the given named arguments via BAML execution.
|
|
3723
|
-
*
|
|
3724
|
-
* @param methodName - The name of the method to call
|
|
3725
|
-
* @param inputs - Named arguments to pass to the method
|
|
3726
|
-
* @returns The result of the BAML function execution
|
|
3727
|
-
* @throws {BitfabError} If service_url is not set, or if an error occurs
|
|
3728
|
-
*/
|
|
3729
|
-
call<T = unknown>(methodName: string, inputs?: Record<string, unknown>): Promise<T>;
|
|
3730
|
-
/**
|
|
3731
|
-
* Get a tracing processor for OpenAI Agents SDK integration.
|
|
3732
|
-
*
|
|
3733
|
-
* This processor automatically captures traces and spans from the OpenAI Agents SDK
|
|
3734
|
-
* and sends them to Bitfab for monitoring and analysis.
|
|
3735
|
-
*
|
|
3736
|
-
* Example usage:
|
|
3737
|
-
* ```typescript
|
|
3738
|
-
* import { addTraceProcessor } from '@openai/agents';
|
|
3739
|
-
*
|
|
3740
|
-
* const client = new Bitfab({ apiKey: 'your-api-key' });
|
|
3741
|
-
* const processor = client.getOpenAiTracingProcessor();
|
|
3742
|
-
* addTraceProcessor(processor);
|
|
3743
|
-
* ```
|
|
3744
|
-
*
|
|
3745
|
-
* @returns A BitfabOpenAITracingProcessor instance configured for this client
|
|
3746
|
-
*/
|
|
3747
|
-
getOpenAiTracingProcessor(): BitfabOpenAITracingProcessor;
|
|
3748
|
-
/**
|
|
3749
|
-
* Get an OpenAI Agents SDK handler that records a replayable root span.
|
|
3750
|
-
*
|
|
3751
|
-
* The processor from {@link getOpenAiTracingProcessor} captures everything
|
|
3752
|
-
* inside a run (LLM calls, tools, handoffs) but never sees the caller's
|
|
3753
|
-
* input, so a processor-only run records an empty-input root and is not
|
|
3754
|
-
* replayable. This handler's `wrapRun` is a drop-in for `run()` that opens a
|
|
3755
|
-
* `withSpan` root carrying the input and final output; the processor's spans
|
|
3756
|
-
* nest beneath it. Register the processor once at startup, then call
|
|
3757
|
-
* `handler.wrapRun(agent, input)` in place of `run(agent, input)`.
|
|
3758
|
-
*
|
|
3759
|
-
* ```typescript
|
|
3760
|
-
* import { addTraceProcessor, Agent, run } from "@openai/agents";
|
|
3761
|
-
*
|
|
3762
|
-
* addTraceProcessor(client.getOpenAiTracingProcessor());
|
|
3763
|
-
* const handler = client.getOpenAiAgentHandler("research-topic");
|
|
3764
|
-
* const result = await handler.wrapRun(agent, "Find X");
|
|
3765
|
-
* ```
|
|
3766
|
-
*
|
|
3767
|
-
* @param traceFunctionKey - Groups traces under this key in Bitfab
|
|
3768
|
-
* @returns A BitfabOpenAIAgentHandler configured for this client
|
|
3769
|
-
*/
|
|
3770
|
-
getOpenAiAgentHandler(traceFunctionKey: string): BitfabOpenAIAgentHandler;
|
|
3771
|
-
/**
|
|
3772
|
-
* Get a LangGraph/LangChain callback handler for tracing.
|
|
3773
|
-
*
|
|
3774
|
-
* The handler captures graph node execution, LLM calls, and tool
|
|
3775
|
-
* invocations as Bitfab spans with proper parent-child hierarchy.
|
|
3776
|
-
*
|
|
3777
|
-
* ```typescript
|
|
3778
|
-
* const handler = client.getLangGraphCallbackHandler("my-agent");
|
|
3779
|
-
* const result = await agent.invoke(
|
|
3780
|
-
* { messages: [...] },
|
|
3781
|
-
* { callbacks: [handler] },
|
|
3782
|
-
* );
|
|
3783
|
-
* ```
|
|
3784
|
-
*
|
|
3785
|
-
* @param traceFunctionKey - Groups traces under this key in Bitfab
|
|
3786
|
-
* @returns A BitfabLangGraphCallbackHandler configured for this client
|
|
3787
|
-
*/
|
|
3788
|
-
getLangGraphCallbackHandler(traceFunctionKey: string): BitfabLangGraphCallbackHandler;
|
|
3789
|
-
/**
|
|
3790
|
-
* Get a LangChain callback handler for tracing.
|
|
3791
|
-
*
|
|
3792
|
-
* Alias of {@link getLangGraphCallbackHandler}: LangChain chains and
|
|
3793
|
-
* LangGraph graphs share the same callback system, so one handler serves
|
|
3794
|
-
* both.
|
|
3795
|
-
*
|
|
3796
|
-
* ```typescript
|
|
3797
|
-
* const handler = client.getLangChainCallbackHandler("my-chain");
|
|
3798
|
-
* const result = await chain.invoke(input, { callbacks: [handler] });
|
|
3799
|
-
* ```
|
|
3800
|
-
*
|
|
3801
|
-
* @param traceFunctionKey - Groups traces under this key in Bitfab
|
|
3802
|
-
* @returns A BitfabLangGraphCallbackHandler configured for this client
|
|
3803
|
-
*/
|
|
3804
|
-
getLangChainCallbackHandler(traceFunctionKey: string): BitfabLangGraphCallbackHandler;
|
|
3805
|
-
/**
|
|
3806
|
-
* Get the first-class LangGraph integration for tracing and replaying tools
|
|
3807
|
-
* executed by `ToolNode`.
|
|
3808
|
-
*
|
|
3809
|
-
* The integration combines `wrapTools()` for per-tool replay interception
|
|
3810
|
-
* with `createInvoker()` for a callback-configured replayable graph entry
|
|
3811
|
-
* point. Lower-level callback and root wrappers remain available.
|
|
3812
|
-
*
|
|
3813
|
-
* @param traceFunctionKey - Groups traces under this key in Bitfab
|
|
3814
|
-
* @param options - Controls which tools are marked for replay mocking
|
|
3815
|
-
* @experimental This API may change before it is stable.
|
|
3816
|
-
*/
|
|
3817
|
-
getLangGraphIntegration(traceFunctionKey: string, options?: LangGraphIntegrationOptions): BitfabLangGraphIntegration;
|
|
3818
|
-
/**
|
|
3819
|
-
* Get the first-class Rivet actor/workflow-context integration for an
|
|
3820
|
-
* existing trace root. The integration binds context and records effect
|
|
3821
|
-
* spans; it never creates another root.
|
|
3822
|
-
*
|
|
3823
|
-
* @param traceFunctionKey - Key used by the enclosing `withTrace()` root
|
|
3824
|
-
* @param options - Explicit per-context-path replay mocking policy
|
|
3825
|
-
* @experimental This API may change before it is stable.
|
|
3826
|
-
*/
|
|
3827
|
-
getRivetIntegration<TContext extends object = RivetContextLike>(traceFunctionKey: string, options?: RivetIntegrationOptions): BitfabRivetIntegration<TContext>;
|
|
3828
|
-
/**
|
|
3829
|
-
* Get a Claude Agent SDK handler for tracing.
|
|
3830
|
-
*
|
|
3831
|
-
* The handler captures LLM turns, tool invocations, and subagent
|
|
3832
|
-
* execution as Bitfab spans with proper parent-child hierarchy.
|
|
3833
|
-
*
|
|
3834
|
-
* ```typescript
|
|
3835
|
-
* import { query } from "@anthropic-ai/claude-agent-sdk";
|
|
3836
|
-
*
|
|
3837
|
-
* const handler = client.getClaudeAgentHandler("my-agent");
|
|
3838
|
-
* const options = handler.instrumentOptions({
|
|
3839
|
-
* model: "claude-sonnet-4-5-...",
|
|
3840
|
-
* });
|
|
3841
|
-
* for await (const msg of handler.wrapQuery(
|
|
3842
|
-
* query({ prompt: "Do something", options })
|
|
3843
|
-
* )) {
|
|
3844
|
-
* // process messages
|
|
3845
|
-
* }
|
|
3846
|
-
* ```
|
|
3847
|
-
*
|
|
3848
|
-
* @param traceFunctionKey - Groups traces under this key in Bitfab
|
|
3849
|
-
* @returns A BitfabClaudeAgentHandler configured for this client
|
|
3850
|
-
*/
|
|
3851
|
-
getClaudeAgentHandler(traceFunctionKey: string): BitfabClaudeAgentHandler;
|
|
3852
|
-
/**
|
|
3853
|
-
* Get a Vercel AI SDK language-model middleware for tracing.
|
|
3854
|
-
*
|
|
3855
|
-
* Pass it to the AI SDK's `wrapLanguageModel` and use the wrapped model with
|
|
3856
|
-
* `generateText` / `streamText` / `generateObject` / `streamObject`. Every
|
|
3857
|
-
* call through that model is captured as a keyed `llm` span carrying the call
|
|
3858
|
-
* parameters (the prompt) as input and a serializable summary
|
|
3859
|
-
* (`{ text, toolCalls, usage, finishReason }`) as output. Streaming is
|
|
3860
|
-
* captured without disturbing the caller's live stream.
|
|
3861
|
-
*
|
|
3862
|
-
* ```typescript
|
|
3863
|
-
* import { wrapLanguageModel, streamText } from "ai";
|
|
3864
|
-
* import { openai } from "@ai-sdk/openai";
|
|
3865
|
-
*
|
|
3866
|
-
* const model = wrapLanguageModel({
|
|
3867
|
-
* model: openai("gpt-4o"),
|
|
3868
|
-
* middleware: client.getVercelAiMiddleware("chat-turn"),
|
|
3869
|
-
* });
|
|
3870
|
-
* const result = streamText({ model, messages });
|
|
3871
|
-
* ```
|
|
3872
|
-
*
|
|
3873
|
-
* @param traceFunctionKey - Groups traces under this key in Bitfab
|
|
3874
|
-
* @returns A Vercel AI SDK middleware configured for this client
|
|
3875
|
-
*/
|
|
3876
|
-
getVercelAiMiddleware(traceFunctionKey: string): BitfabLanguageModelMiddleware;
|
|
3877
|
-
/**
|
|
3878
|
-
* Wrap a BAML client method to automatically capture prompt and LLM metadata.
|
|
3879
|
-
*
|
|
3880
|
-
* Creates a BAML Collector, calls the method through a tracked client,
|
|
3881
|
-
* then extracts rendered messages and token usage - calling setPrompt()
|
|
3882
|
-
* and addContext() on the current span automatically.
|
|
3883
|
-
*
|
|
3884
|
-
* The BAML client can be provided in the constructor or passed explicitly:
|
|
3885
|
-
*
|
|
3886
|
-
* ```typescript
|
|
3887
|
-
* // Option 1: bamlClient in constructor (use wrapBAML with just the method)
|
|
3888
|
-
* const client = new Bitfab({ apiKey: 'your-api-key', bamlClient: b });
|
|
3889
|
-
* const traced = client.withSpan('classify', { type: 'llm' },
|
|
3890
|
-
* client.wrapBAML(b.ClassifyText)
|
|
3891
|
-
* );
|
|
3892
|
-
*
|
|
3893
|
-
* // Option 2: pass bamlClient at call site
|
|
3894
|
-
* const client = new Bitfab({ apiKey: 'your-api-key' });
|
|
3895
|
-
* const traced = client.withSpan('classify', { type: 'llm' },
|
|
3896
|
-
* client.wrapBAML(b, b.ClassifyText)
|
|
3897
|
-
* );
|
|
3898
|
-
* ```
|
|
3899
|
-
*
|
|
3900
|
-
* @param methodOrClient - Either a BAML method (uses constructor bamlClient) or the BAML client instance
|
|
3901
|
-
* @param maybeMethodOrOptions - The BAML method when the first argument is a client, or WrapBAMLOptions when the first argument is the method
|
|
3902
|
-
* @param maybeOptions - WrapBAMLOptions when using the two-argument (client, method) form
|
|
3903
|
-
* @returns An async function with the same signature that instruments the BAML call
|
|
3904
|
-
*/
|
|
3905
|
-
wrapBAML<TArgs extends unknown[], TReturn>(methodOrClient: unknown, maybeMethodOrOptions?: ((...args: TArgs) => Promise<TReturn>) | WrapBAMLOptions, maybeOptions?: WrapBAMLOptions): WrappedBamlFn<TArgs, TReturn>;
|
|
3906
|
-
/**
|
|
3907
|
-
* Wrap a function to automatically create a span for its inputs and outputs.
|
|
3908
|
-
*
|
|
3909
|
-
* The wrapped function behaves identically to the original, but sends
|
|
3910
|
-
* span data to Bitfab in the background after each call.
|
|
3911
|
-
*
|
|
3912
|
-
* Example usage:
|
|
3913
|
-
* ```typescript
|
|
3914
|
-
* const client = new Bitfab({ apiKey: 'your-api-key' });
|
|
3915
|
-
*
|
|
3916
|
-
* async function processOrder(orderId: string, items: string[]): Promise<{ total: number }> {
|
|
3917
|
-
* // ... process order
|
|
3918
|
-
* return { total: 100 };
|
|
3919
|
-
* }
|
|
3920
|
-
*
|
|
3921
|
-
* // Basic usage (defaults to "custom" span type)
|
|
3922
|
-
* const tracedProcessOrder = client.withSpan('order-processing', processOrder);
|
|
3923
|
-
*
|
|
3924
|
-
* // With explicit span type
|
|
3925
|
-
* const tracedProcessOrder = client.withSpan('order-processing', { type: 'function' }, processOrder);
|
|
3926
|
-
*
|
|
3927
|
-
* // Call the wrapped function normally
|
|
3928
|
-
* const result = await tracedProcessOrder('order-123', ['item-1', 'item-2']);
|
|
3929
|
-
* // Span is automatically sent to Bitfab
|
|
3930
|
-
* ```
|
|
3931
|
-
*
|
|
3932
|
-
* @param traceFunctionKey - A string identifier for grouping spans (e.g., 'order-processing', 'user-auth')
|
|
3933
|
-
* @param optionsOrFn - Either SpanOptions or the function to wrap
|
|
3934
|
-
* @param maybeFn - The function to wrap if options were provided
|
|
3935
|
-
* @returns A wrapped function with the same signature that creates spans for inputs and outputs
|
|
3936
|
-
*/
|
|
3937
|
-
withSpan<TArgs extends unknown[], TReturn>(traceFunctionKey: string, optionsOrFn: SpanOptions | ((...args: TArgs) => TReturn), maybeFn?: (...args: TArgs) => TReturn): (...args: TArgs) => TReturn;
|
|
3938
|
-
/**
|
|
3939
|
-
* Create a standard ECMAScript method decorator that records each invocation
|
|
3940
|
-
* as a span.
|
|
3941
|
-
*
|
|
3942
|
-
* It supports instance, static, and private methods; use
|
|
3943
|
-
* {@link Bitfab.withSpan} for standalone functions, class fields, and
|
|
3944
|
-
* accessors.
|
|
3945
|
-
*
|
|
3946
|
-
* @example
|
|
3947
|
-
* ```typescript
|
|
3948
|
-
* const bitfab = new Bitfab({ apiKey: process.env.BITFAB_API_KEY });
|
|
3949
|
-
*
|
|
3950
|
-
* class OrderService {
|
|
3951
|
-
* @bitfab.span("order-processing", { type: "agent" })
|
|
3952
|
-
* async process(orderId: string) {
|
|
3953
|
-
* return { orderId };
|
|
3954
|
-
* }
|
|
3955
|
-
* }
|
|
3956
|
-
* ```
|
|
3957
|
-
*
|
|
3958
|
-
* @param traceFunctionKey - A string identifier for grouping spans
|
|
3959
|
-
* @param options - Span configuration applied to the decorated method
|
|
3960
|
-
* @returns A standard ECMAScript method decorator
|
|
3961
|
-
*/
|
|
3962
|
-
span(traceFunctionKey: string, options?: SpanOptions): SpanMethodDecorator;
|
|
3963
|
-
/**
|
|
3964
|
-
* Get a detached handle to a previously-created trace, looked up by the
|
|
3965
|
-
* canonical Bitfab trace ID.
|
|
3966
|
-
*
|
|
3967
|
-
* The returned handle is not tied to AsyncLocalStorage - each method sends
|
|
3968
|
-
* to the server immediately. Useful for adding context to a trace from a
|
|
3969
|
-
* different process or thread than the one that created it.
|
|
3970
|
-
*
|
|
3971
|
-
* Throws synchronously if `traceId` is not a valid Bitfab trace ID. The
|
|
3972
|
-
* server returns 404 if no trace exists with that ID in the org; the failure surfaces as a
|
|
3973
|
-
* logged warning (fire-and-forget) or via the awaited promise.
|
|
3974
|
-
*
|
|
3975
|
-
* Example:
|
|
3976
|
-
* ```typescript
|
|
3977
|
-
* const trace = client.getTrace(traceId);
|
|
3978
|
-
* await trace.addContext({ refund_status: "approved" });
|
|
3979
|
-
* await trace.setMetadata({ region: "us-west" });
|
|
3980
|
-
* ```
|
|
3981
|
-
*/
|
|
3982
|
-
getTrace(traceId: string): DetachedTrace;
|
|
3983
|
-
/**
|
|
3984
|
-
* Fetch one persisted span from a trace without loading the full trace.
|
|
3985
|
-
* Look up by `id` alone, or by `name` and/or `type` (both must match when
|
|
3986
|
-
* both are given). Name and type lookups return the last matching span by
|
|
3987
|
-
* default. Pass `occurrence` as `"first"` or a zero-based index to select a
|
|
3988
|
-
* different match. Returns `null` when no span matches.
|
|
3989
|
-
*/
|
|
3990
|
-
getTraceSpan(traceId: string, lookup: SpanLookup): Promise<CapturedSpan | null>;
|
|
3991
|
-
getSpan(spanId: string): Promise<CapturedSpan | null>;
|
|
3992
|
-
/**
|
|
3993
|
-
* Get a function wrapper for a specific trace function key.
|
|
3994
|
-
*
|
|
3995
|
-
* This provides a fluent API alternative to calling withSpan directly,
|
|
3996
|
-
* allowing you to bind the traceFunctionKey once and wrap multiple functions.
|
|
3997
|
-
*
|
|
3998
|
-
* Example usage:
|
|
3999
|
-
* ```typescript
|
|
4000
|
-
* const client = new Bitfab({ apiKey: 'your-api-key' });
|
|
4001
|
-
*
|
|
4002
|
-
* const orderFunc = client.getFunction('order-processing');
|
|
4003
|
-
* const tracedProcessOrder = orderFunc.withSpan(processOrder);
|
|
4004
|
-
* const tracedValidateOrder = orderFunc.withSpan(validateOrder);
|
|
4005
|
-
* ```
|
|
4006
|
-
*
|
|
4007
|
-
* @param traceFunctionKey - A string identifier for grouping spans
|
|
4008
|
-
* @returns A BitfabFunction instance for wrapping functions
|
|
4009
|
-
*/
|
|
4010
|
-
getFunction(traceFunctionKey: string): BitfabFunction;
|
|
4011
|
-
/**
|
|
4012
|
-
* Send trace completion when a root span ends.
|
|
4013
|
-
* Internal method to record trace completion with end time.
|
|
4014
|
-
* Queued on the client's span transport; delivery is the transport's job.
|
|
4015
|
-
*/
|
|
4016
|
-
private sendTraceCompletion;
|
|
4017
|
-
private sendsSpanOnCompletion;
|
|
4018
|
-
private nearestCapturedAncestorSpanIdOf;
|
|
4019
|
-
private contentOffBySimulationPlan;
|
|
4020
|
-
/**
|
|
4021
|
-
* Send a wrapper span from function execution.
|
|
4022
|
-
* Internal method to record spans when using withSpan.
|
|
4023
|
-
* Queued on the client's span transport; delivery is the transport's job.
|
|
4024
|
-
*/
|
|
4025
|
-
private sendWrapperSpan;
|
|
4026
|
-
/**
|
|
4027
|
-
* Replay historical traces through a function and create an experiment.
|
|
4028
|
-
*
|
|
4029
|
-
* Fetches the last N traces for the given trace function key, re-runs each
|
|
4030
|
-
* through the provided function, and returns comparison data.
|
|
4031
|
-
*
|
|
4032
|
-
* Accepts either a `withSpan`-wrapped function (under the same key) or any
|
|
4033
|
-
* plain callable: plain callables are wrapped internally so each replayed
|
|
4034
|
-
* invocation records a trace tied to the experiment. The plain-callable form
|
|
4035
|
-
* is how handler-instrumented workflows (LangGraph/LangChain, Claude Agent
|
|
4036
|
-
* SDK) replay - those record traces under a key with no `withSpan`-wrapped
|
|
4037
|
-
* root in the app.
|
|
4038
|
-
*
|
|
4039
|
-
* @param traceFunctionKey - The trace function key to replay
|
|
4040
|
-
* @param fn - The function to run recorded inputs through
|
|
4041
|
-
* @param options - Optional replay options. When `traceIds` is passed,
|
|
4042
|
-
* `limit` is ignored (with a warning): an explicit ID list already
|
|
4043
|
-
* determines how many traces replay.
|
|
4044
|
-
* @returns ReplayResult with items, experimentId, and experimentUrl
|
|
4045
|
-
*/
|
|
4046
|
-
/**
|
|
4047
|
-
* Register a mock override applied to every subsequent `replay` on this
|
|
4048
|
-
* client, so downstream real code runs against a value you supply for the
|
|
4049
|
-
* matched span. Instance-scoped (no global state); call {@link clearMockOverrides}
|
|
4050
|
-
* to reset. Per-call `replay({ mockOverride })` overrides take precedence, and
|
|
4051
|
-
* both take precedence over the base `mock` strategy.
|
|
4052
|
-
*
|
|
4053
|
-
* ```ts
|
|
4054
|
-
* // Object form (value is a flat value here)
|
|
4055
|
-
* bitfab.registerMockOverride({
|
|
4056
|
-
* match: (node) => node.traceFunctionKey === "classify-intent",
|
|
4057
|
-
* value: { label: "refund" },
|
|
4058
|
-
* })
|
|
4059
|
-
* // Ordered form (equivalent); value may also be a function of the context
|
|
4060
|
-
* bitfab.registerMockOverride(
|
|
4061
|
-
* (node) => node.traceFunctionKey === "classify-intent",
|
|
4062
|
-
* ({ inputs }) => ({ label: "refund" }),
|
|
4063
|
-
* )
|
|
4064
|
-
* // Keyed form: the resolver only sees spans for this trace function key.
|
|
4065
|
-
* bitfab.registerMockOverride("classify-intent", ({ inputs }) => ({
|
|
4066
|
-
* label: String(inputs[0]),
|
|
4067
|
-
* }))
|
|
4068
|
-
* // Or one resolver for every child span:
|
|
4069
|
-
* bitfab.registerMockOverride(({ node }) =>
|
|
4070
|
-
* node.traceFunctionKey === "classify-intent"
|
|
4071
|
-
* ? { label: "refund" }
|
|
4072
|
-
* : NO_MOCK_OVERRIDE,
|
|
4073
|
-
* )
|
|
4074
|
-
* ```
|
|
4075
|
-
*/
|
|
4076
|
-
registerMockOverride(override: MockOverride): void;
|
|
4077
|
-
registerMockOverride(resolver: MockOverrideResolver): void;
|
|
4078
|
-
registerMockOverride(match: NodeMatcher, value: MockValue): void;
|
|
4079
|
-
registerMockOverride(traceFunctionKey: string, override: MockOverride | MockOverrideResolver): void;
|
|
4080
|
-
/** Remove all overrides registered via {@link registerMockOverride}. */
|
|
4081
|
-
clearMockOverrides(): void;
|
|
4082
|
-
seedTrace(traceFunctionKey: string, options: SeedCaseOptions): string;
|
|
4083
|
-
seedTrace<TArgs extends unknown[], TReturn>(traceFunctionKey: string, fn: (...args: TArgs) => TReturn, options?: SeedRunOptions<TArgs>): Promise<string>;
|
|
4084
|
-
private seedTraceFromCase;
|
|
4085
|
-
private seedTraceByRunning;
|
|
4086
|
-
reseedTrace(traceFunctionKey: string, fn: (...args: any[]) => unknown, options: ReseedOptions): Promise<ReseedResult>;
|
|
4087
|
-
replay<TReturn>(traceFunctionKey: string, fn: (...args: any[]) => TReturn | Promise<TReturn>, options?: ReplayOptions): Promise<ReplayResult<TReturn>>;
|
|
4088
|
-
}
|
|
4089
|
-
/**
|
|
4090
|
-
* Represents a Bitfab function that can wrap user functions for tracing.
|
|
4091
|
-
*
|
|
4092
|
-
* This provides a fluent API for binding a traceFunctionKey once and
|
|
4093
|
-
* then wrapping multiple functions with that key.
|
|
4094
|
-
*
|
|
4095
|
-
* Example usage:
|
|
4096
|
-
* ```typescript
|
|
4097
|
-
* const client = new Bitfab({ apiKey: 'your-api-key' });
|
|
4098
|
-
*
|
|
4099
|
-
* const orderFunc = client.getFunction('order-processing');
|
|
4100
|
-
* const tracedProcessOrder = orderFunc.withSpan(processOrder);
|
|
4101
|
-
* const tracedValidateOrder = orderFunc.withSpan(validateOrder);
|
|
4102
|
-
* ```
|
|
4103
|
-
*/
|
|
4104
|
-
declare class BitfabFunction {
|
|
4105
|
-
private readonly client;
|
|
4106
|
-
private readonly traceFunctionKey;
|
|
4107
|
-
constructor(client: Bitfab, traceFunctionKey: string);
|
|
4108
|
-
/**
|
|
4109
|
-
* Wrap a function to automatically create a span for its inputs and outputs.
|
|
4110
|
-
*
|
|
4111
|
-
* The wrapped function behaves identically to the original, but sends
|
|
4112
|
-
* span data to Bitfab in the background after each call.
|
|
4113
|
-
*
|
|
4114
|
-
* Example usage:
|
|
4115
|
-
* ```typescript
|
|
4116
|
-
* const orderFunc = client.getFunction('order-processing');
|
|
4117
|
-
*
|
|
4118
|
-
* // Basic usage (defaults to "custom" span type)
|
|
4119
|
-
* const tracedProcessOrder = orderFunc.withSpan(processOrder);
|
|
4120
|
-
*
|
|
4121
|
-
* // With explicit span type
|
|
4122
|
-
* const tracedProcessOrder = orderFunc.withSpan({ type: 'function' }, processOrder);
|
|
4123
|
-
* ```
|
|
4124
|
-
*
|
|
4125
|
-
* @param optionsOrFn - Either SpanOptions or the function to wrap
|
|
4126
|
-
* @param maybeFn - The function to wrap if options were provided
|
|
4127
|
-
* @returns A wrapped function with the same signature that creates spans
|
|
4128
|
-
*/
|
|
4129
|
-
withSpan<TArgs extends unknown[], TReturn>(optionsOrFn: SpanOptions | ((...args: TArgs) => TReturn), maybeFn?: (...args: TArgs) => TReturn): (...args: TArgs) => TReturn;
|
|
4130
|
-
/**
|
|
4131
|
-
* Create a standard ECMAScript method decorator bound to this function key.
|
|
4132
|
-
*
|
|
4133
|
-
* @example
|
|
4134
|
-
* ```typescript
|
|
4135
|
-
* const orders = client.getFunction("order-processing");
|
|
4136
|
-
*
|
|
4137
|
-
* class OrderService {
|
|
4138
|
-
* @orders.span({ type: "agent" })
|
|
4139
|
-
* async process(orderId: string) {
|
|
4140
|
-
* return { orderId };
|
|
4141
|
-
* }
|
|
4142
|
-
* }
|
|
4143
|
-
* ```
|
|
4144
|
-
*
|
|
4145
|
-
* @param options - Span configuration applied to the decorated method
|
|
4146
|
-
* @returns A standard ECMAScript method decorator
|
|
4147
|
-
*/
|
|
4148
|
-
span(options?: SpanOptions): SpanMethodDecorator;
|
|
4149
|
-
/**
|
|
4150
|
-
* Wrap a function as an automatically expanded trace root bound to this
|
|
4151
|
-
* function's key.
|
|
4152
|
-
*
|
|
4153
|
-
* Equivalent to `client.withTrace(key, options, fn)` but reuses the key
|
|
4154
|
-
* bound on this handle.
|
|
4155
|
-
*
|
|
4156
|
-
* @param optionsOrFn - Trace options or the workflow entrypoint to wrap
|
|
4157
|
-
* @param maybeFn - Workflow entrypoint when options are provided
|
|
4158
|
-
* @returns A wrapped function with the same signature
|
|
4159
|
-
* @experimental Automatic child-call instrumentation is experimental.
|
|
4160
|
-
*/
|
|
4161
|
-
withTrace<This, TArgs extends unknown[], TReturn>(fn: (this: This, ...args: TArgs) => TReturn): (this: This, ...args: TArgs) => TReturn;
|
|
4162
|
-
withTrace<This, TArgs extends unknown[], TReturn>(options: TraceOptions, fn: (this: This, ...args: TArgs) => TReturn): (this: This, ...args: TArgs) => TReturn;
|
|
4163
|
-
/**
|
|
4164
|
-
* Create a trace method decorator bound to this function's key.
|
|
4165
|
-
*
|
|
4166
|
-
* Equivalent to `client.trace(key, options)` but reuses the key bound on
|
|
4167
|
-
* this handle.
|
|
4168
|
-
*
|
|
4169
|
-
* @param options - Trace configuration applied to the decorated method
|
|
4170
|
-
* @returns A method decorator configured for this key
|
|
4171
|
-
* @experimental Automatic child-call instrumentation is experimental.
|
|
4172
|
-
*/
|
|
4173
|
-
trace(options?: TraceOptions): TraceMethodDecorator;
|
|
4174
|
-
/**
|
|
4175
|
-
* Get a Vercel AI SDK language-model middleware bound to this function's key.
|
|
4176
|
-
*
|
|
4177
|
-
* Equivalent to `client.getVercelAiMiddleware(key)` but reuses the key bound
|
|
4178
|
-
* on this handle, so an outer `withSpan` root and the middleware-traced model
|
|
4179
|
-
* calls share one key without repeating the string. With a matching key, the
|
|
4180
|
-
* outer span is the replayable root and the model-call spans nest beneath it.
|
|
4181
|
-
*
|
|
4182
|
-
* Nesting is captured when the model is called, so keep the
|
|
4183
|
-
* `generateText` / `streamText` call inside this handle's `withSpan`; the
|
|
4184
|
-
* middleware object itself can be created anywhere.
|
|
4185
|
-
*
|
|
4186
|
-
* ```typescript
|
|
4187
|
-
* const chatTurn = client.getFunction("chat-turn");
|
|
4188
|
-
* const runChatTurn = chatTurn.withSpan(
|
|
4189
|
-
* { type: "agent", finalize: finalizers.aiSdk },
|
|
4190
|
-
* (messages) => streamText({ model, messages }),
|
|
4191
|
-
* );
|
|
4192
|
-
* const model = wrapLanguageModel({
|
|
4193
|
-
* model: openai("gpt-4o"),
|
|
4194
|
-
* middleware: chatTurn.getVercelAiMiddleware(),
|
|
4195
|
-
* });
|
|
4196
|
-
* ```
|
|
4197
|
-
*
|
|
4198
|
-
* @returns A Vercel AI SDK middleware configured for this client and key
|
|
4199
|
-
*/
|
|
4200
|
-
getVercelAiMiddleware(): BitfabLanguageModelMiddleware;
|
|
4201
|
-
/**
|
|
4202
|
-
* Get a Claude Agent SDK handler bound to this function's key.
|
|
4203
|
-
*
|
|
4204
|
-
* Equivalent to `client.getClaudeAgentHandler(key)` but reuses the key bound
|
|
4205
|
-
* on this handle, so an outer `withSpan` root and the handler share one key
|
|
4206
|
-
* without repeating the string. With a matching key, the outer span is the
|
|
4207
|
-
* replayable root and every handler span nests beneath it.
|
|
4208
|
-
*
|
|
4209
|
-
* Use the handler inside this handle's `withSpan` body so its spans capture
|
|
4210
|
-
* the enclosing root; framework calls made with no active span record their
|
|
4211
|
-
* own root instead.
|
|
4212
|
-
*
|
|
4213
|
-
* ```typescript
|
|
4214
|
-
* const pipeline = client.getFunction("my-agent");
|
|
4215
|
-
* const tracedRun = pipeline.withSpan({ type: "agent" }, async (prompt) => {
|
|
4216
|
-
* const handler = pipeline.getClaudeAgentHandler();
|
|
4217
|
-
* const options = handler.instrumentOptions({ model: "claude-sonnet-4-6" });
|
|
4218
|
-
* for await (const msg of handler.wrapQuery(query({ prompt, options }))) { ... }
|
|
4219
|
-
* });
|
|
4220
|
-
* ```
|
|
4221
|
-
*
|
|
4222
|
-
* @returns A Claude Agent SDK handler configured for this client and key
|
|
4223
|
-
*/
|
|
4224
|
-
getClaudeAgentHandler(): BitfabClaudeAgentHandler;
|
|
4225
|
-
/**
|
|
4226
|
-
* Get a LangGraph/LangChain callback handler bound to this function's key.
|
|
4227
|
-
*
|
|
4228
|
-
* Equivalent to `client.getLangGraphCallbackHandler(key)` but reuses the key
|
|
4229
|
-
* bound on this handle, so an outer `withSpan` root and the handler share one
|
|
4230
|
-
* key without repeating the string. With a matching key, the outer span is
|
|
4231
|
-
* the replayable root and the LangGraph spans nest beneath it.
|
|
4232
|
-
*
|
|
4233
|
-
* Use the handler inside this handle's `withSpan` body so its spans capture
|
|
4234
|
-
* the enclosing root; framework calls made with no active span record their
|
|
4235
|
-
* own root instead.
|
|
4236
|
-
*
|
|
4237
|
-
* ```typescript
|
|
4238
|
-
* const pipeline = client.getFunction("my-pipeline");
|
|
4239
|
-
* const tracedRun = pipeline.withSpan({ type: "agent" }, async (query) => {
|
|
4240
|
-
* const handler = pipeline.getLangGraphCallbackHandler();
|
|
4241
|
-
* return agent.invoke({ messages: [...] }, { callbacks: [handler] });
|
|
4242
|
-
* });
|
|
4243
|
-
* ```
|
|
4244
|
-
*
|
|
4245
|
-
* @returns A LangGraph/LangChain callback handler for this client and key
|
|
4246
|
-
*/
|
|
4247
|
-
getLangGraphCallbackHandler(): BitfabLangGraphCallbackHandler;
|
|
4248
|
-
/**
|
|
4249
|
-
* Alias of {@link getLangGraphCallbackHandler} - LangChain and LangGraph
|
|
4250
|
-
* share one callback system, so the same bound handler serves both.
|
|
4251
|
-
*
|
|
4252
|
-
* @returns A LangChain callback handler for this client and key
|
|
4253
|
-
*/
|
|
4254
|
-
getLangChainCallbackHandler(): BitfabLangGraphCallbackHandler;
|
|
4255
|
-
/**
|
|
4256
|
-
* Get the first-class LangGraph tool replay integration bound to this key.
|
|
4257
|
-
*
|
|
4258
|
-
* @experimental This API may change before it is stable.
|
|
4259
|
-
*/
|
|
4260
|
-
getLangGraphIntegration(options?: LangGraphIntegrationOptions): BitfabLangGraphIntegration;
|
|
4261
|
-
/**
|
|
4262
|
-
* Get the first-class Rivet actor-context integration bound to this key.
|
|
4263
|
-
* Use its `bind()` or `withContext()` outside this handle's `withTrace()`
|
|
4264
|
-
* root, then call `getContext()` inside the root.
|
|
4265
|
-
*
|
|
4266
|
-
* @param options - Explicit per-context-path replay mocking policy
|
|
4267
|
-
* @experimental This API may change before it is stable.
|
|
4268
|
-
*/
|
|
4269
|
-
getRivetIntegration<TContext extends object = RivetContextLike>(options?: RivetIntegrationOptions): BitfabRivetIntegration<TContext>;
|
|
4270
|
-
/**
|
|
4271
|
-
* Wrap a BAML client method to automatically capture prompt and LLM metadata.
|
|
4272
|
-
* Delegates to the parent client's wrapBAML method.
|
|
4273
|
-
*
|
|
4274
|
-
* Unlike the other methods on this handle, `wrapBAML` does NOT use the bound
|
|
4275
|
-
* key: it opens no span of its own. It enriches the *current* span (via
|
|
4276
|
-
* `getCurrentSpan().setPrompt()` / `addContext()`), so call it inside a
|
|
4277
|
-
* function wrapped by this handle's `withSpan` - the bound key keys that
|
|
4278
|
-
* wrapper, and the BAML prompt/metadata attach to it.
|
|
4279
|
-
*
|
|
4280
|
-
* @param methodOrClient - Either a BAML method (uses constructor bamlClient) or the BAML client instance
|
|
4281
|
-
* @param maybeMethodOrOptions - The BAML method when the first argument is a client, or WrapBAMLOptions when the first argument is the method
|
|
4282
|
-
* @param maybeOptions - WrapBAMLOptions when using the two-argument (client, method) form
|
|
4283
|
-
* @returns An async function with the same signature that instruments the BAML call
|
|
4284
|
-
*/
|
|
4285
|
-
wrapBAML<TArgs extends unknown[], TReturn>(methodOrClient: unknown, maybeMethodOrOptions?: ((...args: TArgs) => Promise<TReturn>) | WrapBAMLOptions, maybeOptions?: WrapBAMLOptions): WrappedBamlFn<TArgs, TReturn>;
|
|
4286
|
-
}
|
|
4287
|
-
|
|
4288
|
-
interface CommitRef {
|
|
4289
|
-
sha: string;
|
|
4290
|
-
branch: string | null;
|
|
4291
|
-
dirty: boolean | null;
|
|
4292
|
-
remote: string | null;
|
|
4293
|
-
rootSha: string | null;
|
|
4294
|
-
}
|
|
4295
|
-
|
|
4296
|
-
/**
|
|
4297
|
-
* Auto-generated package metadata.
|
|
4298
|
-
* This file is generated by scripts/generate-version.ts during build.
|
|
4299
|
-
* DO NOT EDIT MANUALLY.
|
|
4300
|
-
*/
|
|
4301
|
-
/**
|
|
4302
|
-
* SDK version from package.json (injected at build time)
|
|
4303
|
-
*/
|
|
4304
|
-
declare const __version__ = "0.65.4";
|
|
4305
|
-
|
|
4306
|
-
/**
|
|
4307
|
-
* Constants for the Bitfab SDK.
|
|
4308
|
-
*/
|
|
4309
|
-
/**
|
|
4310
|
-
* Default service URL for Bitfab API.
|
|
4311
|
-
*/
|
|
4312
|
-
declare const DEFAULT_SERVICE_URL = "https://bitfab.ai";
|
|
4313
|
-
|
|
4314
|
-
/**
|
|
4315
|
-
* Prebuilt `finalize` helpers for `withSpan({ finalize }, fn)`.
|
|
4316
|
-
*
|
|
4317
|
-
* A streaming function returns a live stream object that the caller consumes
|
|
4318
|
-
* directly (SSE, a UI message stream). `withSpan` hands that object back
|
|
4319
|
-
* unchanged; a `finalize` function tells it what serializable view to record
|
|
4320
|
-
* as the span output instead of the raw, non-serializable stream.
|
|
4321
|
-
*/
|
|
4322
|
-
/**
|
|
4323
|
-
* Drain a Vercel AI SDK streaming result into a serializable, replayable
|
|
4324
|
-
* span output: `{ text, usage, finishReason, toolCalls, toolResults }`.
|
|
4325
|
-
*
|
|
4326
|
-
* Pass it straight to `withSpan`:
|
|
4327
|
-
*
|
|
4328
|
-
* ```ts
|
|
4329
|
-
* import { finalizers } from "@bitfab/sdk"
|
|
4330
|
-
*
|
|
4331
|
-
* const traced = bitfab.withSpan(
|
|
4332
|
-
* "chat-turn",
|
|
4333
|
-
* { type: "agent", finalize: finalizers.aiSdk },
|
|
4334
|
-
* () => streamText({ model, messages }),
|
|
4335
|
-
* )
|
|
4336
|
-
* const result = traced() // caller still gets the live StreamTextResult
|
|
4337
|
-
* return result.toUIMessageStreamResponse()
|
|
4338
|
-
* ```
|
|
4339
|
-
*
|
|
4340
|
-
* Never throws: any field that is absent or rejects is recorded as
|
|
4341
|
-
* `undefined` so finalize never drops the span.
|
|
4342
|
-
*/
|
|
4343
|
-
declare function aiSdk(result: unknown): Promise<Record<string, unknown>>;
|
|
4344
|
-
/**
|
|
4345
|
-
* Collect a `ReadableStream`'s chunks into an array for the span output,
|
|
4346
|
-
* via a `tee()` so the caller's branch is untouched. The caller MUST use
|
|
4347
|
-
* the returned stream, not the original, since a stream can only be read
|
|
4348
|
-
* once:
|
|
4349
|
-
*
|
|
4350
|
-
* ```ts
|
|
4351
|
-
* let live: ReadableStream
|
|
4352
|
-
* const traced = bitfab.withSpan(
|
|
4353
|
-
* "render",
|
|
4354
|
-
* { finalize: (r) => finalizers.readableStream(r, (s) => { live = s }) },
|
|
4355
|
-
* () => makeReadableStream(),
|
|
4356
|
-
* )
|
|
4357
|
-
* traced()
|
|
4358
|
-
* return new Response(live!)
|
|
4359
|
-
* ```
|
|
4360
|
-
*
|
|
4361
|
-
* Prefer `aiSdk` for the Vercel AI SDK, whose result tees internally and
|
|
4362
|
-
* needs no caller rewiring.
|
|
4363
|
-
*/
|
|
4364
|
-
declare function readableStream(stream: ReadableStream, onLive: (live: ReadableStream) => void): Promise<{
|
|
4365
|
-
chunks: unknown[];
|
|
4366
|
-
}>;
|
|
4367
|
-
declare const finalizers: {
|
|
4368
|
-
aiSdk: typeof aiSdk;
|
|
4369
|
-
readableStream: typeof readableStream;
|
|
4370
|
-
};
|
|
4371
|
-
|
|
4372
|
-
type ReplayFunction = (...args: any[]) => unknown | Promise<unknown>;
|
|
4373
|
-
type ReplayRegistryOptions = Omit<ReplayOptions, "onItemStart" | "onItemFinish" | "onProgress" | "onExperimentStart">;
|
|
4374
|
-
interface ReplayRegistryContext {
|
|
4375
|
-
/** Values supplied through `--params` and repeated `--param name=value`. */
|
|
4376
|
-
params: Readonly<Record<string, unknown>>;
|
|
4377
|
-
}
|
|
4378
|
-
type ReplayOptionsFactory = (context: ReplayRegistryContext) => ReplayRegistryOptions | Promise<ReplayRegistryOptions>;
|
|
4379
|
-
interface ReplayRegistration {
|
|
4380
|
-
/** Client instance used by the production traced function. */
|
|
4381
|
-
client: Bitfab;
|
|
4382
|
-
/** The exact traced function production calls. */
|
|
4383
|
-
fn: ReplayFunction;
|
|
4384
|
-
/**
|
|
4385
|
-
* Required for handler-instrumented or otherwise plain callables. Omit for a
|
|
4386
|
-
* `withSpan`-wrapped function and the registry reads the wrapper's key.
|
|
4387
|
-
*/
|
|
4388
|
-
traceFunctionKey?: string;
|
|
4389
|
-
/**
|
|
4390
|
-
* Per-function replay behavior and defaults. Put executable configuration
|
|
4391
|
-
* such as `mockOverride` and `adaptInputs` here; command-line values override
|
|
4392
|
-
* overlapping scalar defaults such as `mock` and `maxConcurrency`.
|
|
4393
|
-
*/
|
|
4394
|
-
options?: ReplayRegistryOptions;
|
|
4395
|
-
/** Build executable replay behavior from caller-supplied CLI parameters. */
|
|
4396
|
-
optionsFactory?: ReplayOptionsFactory;
|
|
4397
|
-
}
|
|
4398
|
-
type ReplayRegistry = Record<string, ReplayRegistration>;
|
|
4399
|
-
/**
|
|
4400
|
-
* Define the project-owned list of replayable functions.
|
|
4401
|
-
*
|
|
4402
|
-
* The registry is deliberately data-only: the SDK owns the replay CLI, so an
|
|
4403
|
-
* SDK upgrade can add replay features without regenerating the project file.
|
|
4404
|
-
*/
|
|
4405
|
-
declare function defineReplayRegistry<TRegistry extends ReplayRegistry>(registry: TRegistry): TRegistry;
|
|
4406
|
-
/** One case to seed, in the shape `--seed` reads from JSON or JSONL. */
|
|
4407
|
-
interface SeedCase {
|
|
4408
|
-
/** Arguments spread into the registered function at replay. */
|
|
4409
|
-
input: unknown[];
|
|
4410
|
-
/** The output this case should produce. */
|
|
4411
|
-
expected?: unknown;
|
|
4412
|
-
/** Recorded on the trace, for tracing a seeded case back to its source row. */
|
|
4413
|
-
metadata?: Record<string, unknown>;
|
|
4414
|
-
sessionId?: string;
|
|
4415
|
-
}
|
|
4416
|
-
interface SeedResult {
|
|
4417
|
-
pipeline: string;
|
|
4418
|
-
traceFunctionKey: string;
|
|
4419
|
-
traceIds: string[];
|
|
4420
|
-
}
|
|
4421
|
-
interface ReseedRegistryResult {
|
|
4422
|
-
pipeline: string;
|
|
4423
|
-
traceFunctionKey: string;
|
|
4424
|
-
reseeded: {
|
|
4425
|
-
traceId: string;
|
|
4426
|
-
previousRunTraceId: string;
|
|
4427
|
-
}[];
|
|
4428
|
-
}
|
|
4429
|
-
declare function reseedFromRegistry(registry: ReplayRegistry, pipeline: string, traceIds: readonly string[]): Promise<ReseedRegistryResult>;
|
|
4430
|
-
/**
|
|
4431
|
-
* Seed cases through an already-registered pipeline.
|
|
4432
|
-
*
|
|
4433
|
-
* The registration is the whole point: it already holds the client, the exact
|
|
4434
|
-
* function production calls, and the trace function key replay selects by, so
|
|
4435
|
-
* a seeded case is guaranteed to line up with the replay that will read it.
|
|
4436
|
-
* Passing the registered function to `seedTrace` also means a case that cannot
|
|
4437
|
-
* supply its required arguments is rejected here rather than at replay.
|
|
4438
|
-
*/
|
|
4439
|
-
declare function seedFromRegistry(registry: ReplayRegistry, pipeline: string, cases: readonly SeedCase[], options?: {
|
|
4440
|
-
run?: boolean;
|
|
4441
|
-
}): Promise<SeedResult>;
|
|
4442
|
-
|
|
4443
|
-
export { type ActiveSpanContext, type AdaptContext, type AdaptInputsFn, type AddDatasetGradersResult, type AddDatasetTracesResult, type AllowedEnvVars, type ApprovalFields, type ApprovalState, type Approver, type ArchiveAssertionsParams, type ArchiveLabelTarget, AssertionCategoriesClient, type AssertionCategory, type AssertionCategorySummary, type AssertionEvidenceParameter, type AssertionGeneration, AssertionGenerationError, type AssertionGenerationStatus, AssertionGenerationTimeoutError, type AssertionLabelEvidence, type AssertionVerdict, type Assignee, BITFAB_PROGRESS_PREFIX, type BamlExecutionResult, Bitfab, BitfabClaudeAgentHandler, type BitfabConfig, BitfabError, BitfabFunction, BitfabLangGraphCallbackHandler as BitfabLangChainCallbackHandler, BitfabLangGraphCallbackHandler, BitfabLangGraphIntegration, type BitfabLanguageModelMiddleware, BitfabOpenAIAgentHandler, BitfabOpenAITracingProcessor, BitfabRivetIntegration, BitfabVercelAiHandler, type CaptureSurface, type CaptureWhen, type CapturedSpan, CategoriesClient, type Category, type CategorySummary, type CodeChangeFile, type CommitRef, type CurrentSpan, type CurrentTrace, DEFAULT_SERVICE_URL, type Dataset, type DatasetGraderRef, type DatasetTraceIds, DatasetsClient, type DbBranchOptions, DbBranchReplayError, type DbBranchTimings, type DbSnapshotConfig, type DbSnapshotProvider, type DbSnapshotRef, type DetachedTrace, type Experiment, type ExperimentCategoryTally, type ExperimentCodeChangeTotals, type ExperimentJitter, type ExperimentPage, type ExperimentRollup, type ExperimentRollupResult, type ExperimentRunBy, type ExperimentStatus, type ExperimentTally, type ExperimentTotals, ExperimentsClient, type GenerateAssertionsOptions, type GetGraderLabelsParams, type GraderLabel, type GraderLabelOutcome, type GraderLabelSource, type GraderLabelUpdate, type GraderRerun, GraderRerunError, type GraderRerunProgress, type GraderRerunResult, type GraderRerunStatus, GraderRerunTimeoutError, GradersClient, HttpClient, type HumanLabelOutcome, type HumanLabelUpdate, type Justification, type JustificationSpan, type LabelAction, type LabelConfidence, type LabelOutcome, type LabelSource, type LabelStatus, type LabelTarget, type LabelUpdate, LabelsClient, type LangGraphIntegrationOptions, type ListDatasetsParams, type ListExperimentsParams, type ListedSpan, MixedTracingError, type MockOverride, type MockOverrideCtx, type MockOverrideInput, type MockOverrideResolver, type MockStrategy, type MockValue, NO_MOCK_OVERRIDE, type NodeMatcher, type NodeMethodDecorator, type NodeOptions, type OrganizationMember, OrganizationMembersClient, type ProviderDefinition, type RemoveDatasetGradersResult, type RemoveDatasetTracesResult, ReplayBranch, ReplayError, type ReplayExperiment, type ReplayInterrupt, type ReplayInterruptSignal, type ReplayItem, type ReplayItemFinishProgress, type ReplayItemStartProgress, type ReplayNodeIdentity, type ReplayOptions, type ReplayOptionsFactory, type ReplayProgress, type ReplayProgressItem, type ReplayRegistration, type ReplayRegistry, type ReplayRegistryContext, type ReplayRegistryOptions, type ReplayResult, type RerunGradersOptions, type RerunGradersResult, type ReseedOptions, type ReseedRegistryResult, type ReseedResult, type RivetContextLike, type RivetEffectType, type RivetIntegrationOptions, SUPPORTED_PROVIDERS, type SaveAssertion, type SaveAssertionCategoryParams, type SaveAssertionsAllParams, type SaveAssertionsParams, type SaveCategoryParams, type SaveDatasetParams, type SaveDatasetResult, type SaveScenarioActualOutcome, type SaveScenarioData, type SaveScenarioHistoryEntry, type SaveScenarioParams, type Scenario, type ScenarioAction, type ScenarioActualOutcome, type ScenarioData, type ScenarioDataAppend, type ScenarioDataOrigin, type ScenarioHistoryEntry, type ScenarioSource, type ScenarioTrigger, type SeedCase, type SeedCaseOptions, type SeedResult, type SeedRunOptions, type SelectiveReplayDecision, type SelectiveReplayOptions, type SelectiveReplayReport, type SpanListField, type SpanListParams, type SpanListResult, type SpanLookup, type SpanMethodDecorator, type SpanMethodDecoratorContext, type SpanNodeMeta, type SpanOccurrence, type SpanOptions, type SpanType, SpansClient, type TokenUsage, type TraceAssertion, type TraceAssertionSource, type TraceAssertionsResult, type TraceAssertionsUpdate, type TraceIngestionType, type TraceLabels, type TraceResponse, type TraceSearchFilters, type TraceSearchParams, type TraceSearchResult, type TraceTarget, type TraceTargetOccurrence, TracesClient, type TracingProcessor, type UpdateDatasetParams, type VercelCallParams, type VercelGenerateResult, type VercelStreamResult, type Verdict, type WrapBAMLOptions, type WrappedBamlFn, __version__, defineReplayRegistry, finalizers, flushTraces, getCurrentReplayBranch, getCurrentSpan, getCurrentTrace, reportReplayProgress, reseedFromRegistry, seedFromRegistry, serializeReplayResult };
|
|
1
|
+
import { $ as AdaptInputsFn, $n as RerunGradersResult, $t as SaveAssertionsAllParams, A as SpanMethodDecoratorContext, An as ExperimentRollupResult, Ar as MockValue, At as ArchiveLabelTarget, B as TraceResponse, Bn as DatasetGraderRef, Br as ReplayNodeIdentity, Bt as LabelUpdate, C as NodeMethodDecorator, Cn as TracesClient, Cr as TokenUsage, Ct as BitfabLangGraphCallbackHandler, D as SeedCaseOptions, Dn as ExperimentJitter, Dr as MockOverrideCtx, Dt as GraderLabelSource, E as ReseedResult, En as ExperimentCodeChangeTotals, Er as MockOverride, Et as GraderLabelOutcome, F as getCurrentSpan, Fn as ExperimentsClient, Fr as MixedTracingError, Ft as LabelConfidence, G as SpanListResult, Gn as GraderRerunProgress, Gr as BamlExecutionResult, Gt as AssertionEvidenceParameter, H as ListedSpan, Hn as DatasetsClient, Hr as SelectiveReplayOptions, Ht as TraceLabels, I as getCurrentTrace, In as ListExperimentsParams, Ir as DbSnapshotConfig, It as LabelOutcome, J as RivetContextLike, Jn as GraderRerunTimeoutError, Jr as ApprovalState, Jt as AssertionGenerationStatus, K as SpansClient, Kn as GraderRerunResult, Kr as ProviderDefinition, Kt as AssertionGeneration, L as CaptureSurface, Ln as AddDatasetGradersResult, Lr as DbSnapshotProvider, Lt as LabelSource, M as WrapBAMLOptions, Mn as ExperimentStatus, Mr as NodeMatcher, Mt as HumanLabelOutcome, N as WrappedBamlFn, Nn as ExperimentTally, Nr as SpanNodeMeta, Nt as HumanLabelUpdate, O as SeedRunOptions, On as ExperimentPage, Or as MockOverrideInput, Ot as GraderLabelUpdate, P as getCurrentReplayBranch, Pn as ExperimentTotals, Pr as BitfabError, Pt as LabelAction, Q as AdaptContext, Qn as RerunGradersOptions, Qr as JustificationSpan, Qt as SaveAssertion, R as ActiveSpanContext, Rn as AddDatasetTracesResult, Rr as DbSnapshotRef, Rt as LabelStatus, S as DetachedTrace, Sn as TraceTargetOccurrence, Sr as SpanType, St as LangGraphIntegrationOptions, T as ReseedOptions, Tn as ExperimentCategoryTally, Tr as DbBranchTimings, Tt as GraderLabel, U as SpanListField, Un as GraderRerun, Ur as SelectiveReplayReport, Ut as Verdict, V as TracingProcessor, Vn as DatasetTraceIds, Vr as SelectiveReplayDecision, Vt as LabelsClient, W as SpanListParams, Wn as GraderRerunError, Wr as AllowedEnvVars, Wt as ArchiveAssertionsParams, X as RivetIntegrationOptions, Xn as RemoveDatasetGradersResult, Xr as Assignee, Xt as AssertionLabelEvidence, Y as RivetEffectType, Yn as ListDatasetsParams, Yr as Approver, Yt as AssertionGenerationTimeoutError, Z as ReplayBranch, Zn as RemoveDatasetTracesResult, Zr as Justification, Zt as GenerateAssertionsOptions, _ as BitfabConfig, _n as TraceAssertionsUpdate, _r as CapturedSpan, _t as ReplayInterruptSignal, a as ReplayRegistryOptions, an as Scenario, ar as VercelCallParams, at as ReplayExperiment, b as CurrentSpan, bn as TraceSearchResult, br as SpanLookup, bt as BitfabOpenAIAgentHandler, c as SeedResult, cn as ScenarioData, cr as BitfabClaudeAgentHandler, ct as ReplayItemStartProgress, d as seedFromRegistry, dn as ScenarioHistoryEntry, dr as AssertionCategorySummary, dt as ReplayProgressItem, en as SaveAssertionsParams, er as SaveDatasetParams, et as BITFAB_PROGRESS_PREFIX, f as finalizers, fn as ScenarioSource, fr as CategoriesClient, ft as ReplayResult, g as Bitfab, gn as TraceAssertionsResult, gr as SaveCategoryParams, gt as ReplayInterrupt, h as CommitRef, hn as TraceAssertionSource, hr as SaveAssertionCategoryParams, ht as serializeReplayResult, i as ReplayRegistryContext, in as SaveScenarioParams, ir as BitfabVercelAiHandler, it as ReplayError, j as SpanOptions, jn as ExperimentRunBy, jr as NO_MOCK_OVERRIDE, jt as AssertionVerdict, k as SpanMethodDecorator, kn as ExperimentRollup, kr as MockOverrideResolver, kt as GradersClient, l as defineReplayRegistry, ln as ScenarioDataAppend, lr as AssertionCategoriesClient, lt as ReplayOptions, m as __version__, mn as TraceAssertion, mr as CategorySummary, mt as reportReplayProgress, n as ReplayRegistration, nn as SaveScenarioData, nr as UpdateDatasetParams, nt as DbBranchReplayError, o as ReseedRegistryResult, on as ScenarioAction, or as VercelGenerateResult, ot as ReplayItem, p as DEFAULT_SERVICE_URL, pn as ScenarioTrigger, pr as Category, pt as TraceIngestionType, q as BitfabRivetIntegration, qn as GraderRerunStatus, qr as ApprovalFields, qt as AssertionGenerationError, r as ReplayRegistry, rn as SaveScenarioHistoryEntry, rr as BitfabLanguageModelMiddleware, rt as MockStrategy, s as SeedCase, sn as ScenarioActualOutcome, sr as VercelStreamResult, st as ReplayItemFinishProgress, t as ReplayOptionsFactory, tn as SaveScenarioActualOutcome, tr as SaveDatasetResult, tt as DbBranchOptions, u as reseedFromRegistry, un as ScenarioDataOrigin, ur as AssertionCategory, ut as ReplayProgress, v as BitfabFunction, vn as TraceSearchFilters, vr as CodeChangeFile, vt as OrganizationMember, w as NodeOptions, wn as Experiment, wr as flushTraces, wt as GetGraderLabelsParams, x as CurrentTrace, xn as TraceTarget, xr as SpanOccurrence, xt as BitfabLangGraphIntegration, y as CaptureWhen, yn as TraceSearchParams, yr as HttpClient, yt as OrganizationMembersClient, z as BitfabOpenAITracingProcessor, zn as Dataset, zr as SUPPORTED_PROVIDERS, zt as LabelTarget } from "./index-BFkAwvuc.cjs";
|
|
2
|
+
export { type ActiveSpanContext, type AdaptContext, type AdaptInputsFn, type AddDatasetGradersResult, type AddDatasetTracesResult, type AllowedEnvVars, type ApprovalFields, type ApprovalState, type Approver, type ArchiveAssertionsParams, type ArchiveLabelTarget, AssertionCategoriesClient, type AssertionCategory, type AssertionCategorySummary, type AssertionEvidenceParameter, type AssertionGeneration, AssertionGenerationError, type AssertionGenerationStatus, AssertionGenerationTimeoutError, type AssertionLabelEvidence, type AssertionVerdict, type Assignee, BITFAB_PROGRESS_PREFIX, type BamlExecutionResult, Bitfab, BitfabClaudeAgentHandler, type BitfabConfig, BitfabError, BitfabFunction, BitfabLangGraphCallbackHandler as BitfabLangChainCallbackHandler, BitfabLangGraphCallbackHandler, BitfabLangGraphIntegration, type BitfabLanguageModelMiddleware, BitfabOpenAIAgentHandler, BitfabOpenAITracingProcessor, BitfabRivetIntegration, BitfabVercelAiHandler, type CaptureSurface, type CaptureWhen, type CapturedSpan, CategoriesClient, type Category, type CategorySummary, type CodeChangeFile, type CommitRef, type CurrentSpan, type CurrentTrace, DEFAULT_SERVICE_URL, type Dataset, type DatasetGraderRef, type DatasetTraceIds, DatasetsClient, type DbBranchOptions, DbBranchReplayError, type DbBranchTimings, type DbSnapshotConfig, type DbSnapshotProvider, type DbSnapshotRef, type DetachedTrace, type Experiment, type ExperimentCategoryTally, type ExperimentCodeChangeTotals, type ExperimentJitter, type ExperimentPage, type ExperimentRollup, type ExperimentRollupResult, type ExperimentRunBy, type ExperimentStatus, type ExperimentTally, type ExperimentTotals, ExperimentsClient, type GenerateAssertionsOptions, type GetGraderLabelsParams, type GraderLabel, type GraderLabelOutcome, type GraderLabelSource, type GraderLabelUpdate, type GraderRerun, GraderRerunError, type GraderRerunProgress, type GraderRerunResult, type GraderRerunStatus, GraderRerunTimeoutError, GradersClient, HttpClient, type HumanLabelOutcome, type HumanLabelUpdate, type Justification, type JustificationSpan, type LabelAction, type LabelConfidence, type LabelOutcome, type LabelSource, type LabelStatus, type LabelTarget, type LabelUpdate, LabelsClient, type LangGraphIntegrationOptions, type ListDatasetsParams, type ListExperimentsParams, type ListedSpan, MixedTracingError, type MockOverride, type MockOverrideCtx, type MockOverrideInput, type MockOverrideResolver, type MockStrategy, type MockValue, NO_MOCK_OVERRIDE, type NodeMatcher, type NodeMethodDecorator, type NodeOptions, type OrganizationMember, OrganizationMembersClient, type ProviderDefinition, type RemoveDatasetGradersResult, type RemoveDatasetTracesResult, type ReplayBranch, ReplayError, type ReplayExperiment, type ReplayInterrupt, type ReplayInterruptSignal, type ReplayItem, type ReplayItemFinishProgress, type ReplayItemStartProgress, type ReplayNodeIdentity, type ReplayOptions, type ReplayOptionsFactory, type ReplayProgress, type ReplayProgressItem, type ReplayRegistration, type ReplayRegistry, type ReplayRegistryContext, type ReplayRegistryOptions, type ReplayResult, type RerunGradersOptions, type RerunGradersResult, type ReseedOptions, type ReseedRegistryResult, type ReseedResult, type RivetContextLike, type RivetEffectType, type RivetIntegrationOptions, SUPPORTED_PROVIDERS, type SaveAssertion, type SaveAssertionCategoryParams, type SaveAssertionsAllParams, type SaveAssertionsParams, type SaveCategoryParams, type SaveDatasetParams, type SaveDatasetResult, type SaveScenarioActualOutcome, type SaveScenarioData, type SaveScenarioHistoryEntry, type SaveScenarioParams, type Scenario, type ScenarioAction, type ScenarioActualOutcome, type ScenarioData, type ScenarioDataAppend, type ScenarioDataOrigin, type ScenarioHistoryEntry, type ScenarioSource, type ScenarioTrigger, type SeedCase, type SeedCaseOptions, type SeedResult, type SeedRunOptions, type SelectiveReplayDecision, type SelectiveReplayOptions, type SelectiveReplayReport, type SpanListField, type SpanListParams, type SpanListResult, type SpanLookup, type SpanMethodDecorator, type SpanMethodDecoratorContext, type SpanNodeMeta, type SpanOccurrence, type SpanOptions, type SpanType, SpansClient, type TokenUsage, type TraceAssertion, type TraceAssertionSource, type TraceAssertionsResult, type TraceAssertionsUpdate, type TraceIngestionType, type TraceLabels, type TraceResponse, type TraceSearchFilters, type TraceSearchParams, type TraceSearchResult, type TraceTarget, type TraceTargetOccurrence, TracesClient, type TracingProcessor, type UpdateDatasetParams, type VercelCallParams, type VercelGenerateResult, type VercelStreamResult, type Verdict, type WrapBAMLOptions, type WrappedBamlFn, __version__, defineReplayRegistry, finalizers, flushTraces, getCurrentReplayBranch, getCurrentSpan, getCurrentTrace, reportReplayProgress, reseedFromRegistry, seedFromRegistry, serializeReplayResult };
|