@assemble-dev/evals 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +93 -0
- package/dist/index.cjs +1203 -0
- package/dist/index.d.cts +816 -0
- package/dist/index.d.ts +816 -0
- package/dist/index.js +1138 -0
- package/package.json +49 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,816 @@
|
|
|
1
|
+
import { JsonValue } from '@assemble-dev/shared-types/json';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { WorkflowDefinition, PlatformFetchImplementation, TraceSink } from '@assemble-dev/sdk';
|
|
4
|
+
import { ModelAdapter } from '@assemble-dev/providers';
|
|
5
|
+
import { RunState, RunApproval, TokenUsage, ToolCall } from '@assemble-dev/shared-types/runs';
|
|
6
|
+
import { ValidationSchema } from '@assemble-dev/shared-types/validation';
|
|
7
|
+
import * as _assemble_dev_shared_types from '@assemble-dev/shared-types';
|
|
8
|
+
|
|
9
|
+
declare const ExpectedToolCallSchema: z.ZodObject<{
|
|
10
|
+
toolName: z.ZodString;
|
|
11
|
+
count: z.ZodOptional<z.ZodNumber>;
|
|
12
|
+
}, z.core.$strip>;
|
|
13
|
+
type ExpectedToolCall = z.infer<typeof ExpectedToolCallSchema>;
|
|
14
|
+
declare const ExpectedApprovalDecisionSchema: z.ZodObject<{
|
|
15
|
+
toolName: z.ZodOptional<z.ZodString>;
|
|
16
|
+
status: z.ZodEnum<{
|
|
17
|
+
approved: "approved";
|
|
18
|
+
rejected: "rejected";
|
|
19
|
+
}>;
|
|
20
|
+
}, z.core.$strip>;
|
|
21
|
+
type ExpectedApprovalDecision = z.infer<typeof ExpectedApprovalDecisionSchema>;
|
|
22
|
+
type EvalCaseExpectation<TOutput extends JsonValue> = {
|
|
23
|
+
output?: TOutput;
|
|
24
|
+
outputFields?: Record<string, JsonValue>;
|
|
25
|
+
outputPredicate?(this: void, output: TOutput): boolean | string;
|
|
26
|
+
toolCalls?: ExpectedToolCall[];
|
|
27
|
+
approvalDecisions?: ExpectedApprovalDecision[];
|
|
28
|
+
};
|
|
29
|
+
type EvalCase<TInput extends JsonValue, TOutput extends JsonValue> = {
|
|
30
|
+
name: string;
|
|
31
|
+
input: TInput;
|
|
32
|
+
expected?: EvalCaseExpectation<TOutput>;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
declare const SerializedEvalCaseExpectationSchema: z.ZodObject<{
|
|
36
|
+
output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
37
|
+
outputFields: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
38
|
+
toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
39
|
+
toolName: z.ZodString;
|
|
40
|
+
count: z.ZodOptional<z.ZodNumber>;
|
|
41
|
+
}, z.core.$strip>>>;
|
|
42
|
+
approvalDecisions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
43
|
+
toolName: z.ZodOptional<z.ZodString>;
|
|
44
|
+
status: z.ZodEnum<{
|
|
45
|
+
approved: "approved";
|
|
46
|
+
rejected: "rejected";
|
|
47
|
+
}>;
|
|
48
|
+
}, z.core.$strip>>>;
|
|
49
|
+
}, z.core.$strict>;
|
|
50
|
+
declare const SerializedEvalCaseSchema: z.ZodObject<{
|
|
51
|
+
name: z.ZodString;
|
|
52
|
+
input: z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>;
|
|
53
|
+
expected: z.ZodOptional<z.ZodObject<{
|
|
54
|
+
output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
55
|
+
outputFields: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
56
|
+
toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
57
|
+
toolName: z.ZodString;
|
|
58
|
+
count: z.ZodOptional<z.ZodNumber>;
|
|
59
|
+
}, z.core.$strip>>>;
|
|
60
|
+
approvalDecisions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
61
|
+
toolName: z.ZodOptional<z.ZodString>;
|
|
62
|
+
status: z.ZodEnum<{
|
|
63
|
+
approved: "approved";
|
|
64
|
+
rejected: "rejected";
|
|
65
|
+
}>;
|
|
66
|
+
}, z.core.$strip>>>;
|
|
67
|
+
}, z.core.$strict>>;
|
|
68
|
+
}, z.core.$strict>;
|
|
69
|
+
type SerializedEvalCase = z.infer<typeof SerializedEvalCaseSchema>;
|
|
70
|
+
type LoadEvalCasesFromJsonlFileInput = {
|
|
71
|
+
filePath: string;
|
|
72
|
+
};
|
|
73
|
+
declare function loadEvalCasesFromJsonlFile(input: LoadEvalCasesFromJsonlFileInput): SerializedEvalCase[];
|
|
74
|
+
type EvalCasesSource<TInput extends JsonValue, TOutput extends JsonValue> = readonly EvalCase<TInput, TOutput>[] | {
|
|
75
|
+
jsonlFilePath: string;
|
|
76
|
+
};
|
|
77
|
+
declare function loadEvalCases<TInput extends JsonValue, TOutput extends JsonValue>(source: EvalCasesSource<TInput, TOutput>): readonly EvalCase<TInput, TOutput>[] | readonly SerializedEvalCase[];
|
|
78
|
+
|
|
79
|
+
type EvalSuiteConfig<TInput extends JsonValue, TOutput extends JsonValue> = {
|
|
80
|
+
name: string;
|
|
81
|
+
workflow: WorkflowDefinition<TInput, TOutput>;
|
|
82
|
+
cases: readonly EvalCase<TInput, TOutput>[] | readonly SerializedEvalCase[];
|
|
83
|
+
};
|
|
84
|
+
type EvalSuiteDefinition<TInput extends JsonValue, TOutput extends JsonValue> = {
|
|
85
|
+
readonly name: string;
|
|
86
|
+
readonly workflow: WorkflowDefinition<TInput, TOutput>;
|
|
87
|
+
readonly cases: readonly EvalCase<TInput, TOutput>[];
|
|
88
|
+
};
|
|
89
|
+
declare function evalSuite<TInput extends JsonValue, TOutput extends JsonValue>(config: EvalSuiteConfig<TInput, TOutput>): EvalSuiteDefinition<TInput, TOutput>;
|
|
90
|
+
|
|
91
|
+
type MetricResult = {
|
|
92
|
+
name: string;
|
|
93
|
+
passed: boolean;
|
|
94
|
+
score: number;
|
|
95
|
+
reason: string;
|
|
96
|
+
skipped: boolean;
|
|
97
|
+
};
|
|
98
|
+
type MetricExpectationOutcome = {
|
|
99
|
+
satisfied: boolean;
|
|
100
|
+
reason: string;
|
|
101
|
+
};
|
|
102
|
+
declare function createPassedMetricResult(name: string, reason: string): MetricResult;
|
|
103
|
+
declare function createFailedMetricResult(name: string, score: number, reason: string): MetricResult;
|
|
104
|
+
declare function createSkippedMetricResult(name: string, reason: string): MetricResult;
|
|
105
|
+
declare function createScoredMetricResult(name: string, outcomes: readonly MetricExpectationOutcome[]): MetricResult;
|
|
106
|
+
|
|
107
|
+
declare const defaultJudgeMetricName = "llm-judge";
|
|
108
|
+
declare const defaultJudgePassThreshold = 0.7;
|
|
109
|
+
declare const JudgeVerdictSchema: z.ZodObject<{
|
|
110
|
+
score: z.ZodNumber;
|
|
111
|
+
reasoning: z.ZodString;
|
|
112
|
+
}, z.core.$strip>;
|
|
113
|
+
type JudgeVerdict = z.infer<typeof JudgeVerdictSchema>;
|
|
114
|
+
type DefineJudgeMetricConfig = {
|
|
115
|
+
name?: string;
|
|
116
|
+
model: ModelAdapter;
|
|
117
|
+
rubric: string;
|
|
118
|
+
passThreshold?: number;
|
|
119
|
+
};
|
|
120
|
+
type JudgeMetricDefinition = {
|
|
121
|
+
name: string;
|
|
122
|
+
model: ModelAdapter;
|
|
123
|
+
rubric: string;
|
|
124
|
+
passThreshold: number;
|
|
125
|
+
};
|
|
126
|
+
declare function defineJudgeMetric(config: DefineJudgeMetricConfig): JudgeMetricDefinition;
|
|
127
|
+
type EvaluateJudgeMetricInput<TInput extends JsonValue, TOutput extends JsonValue> = {
|
|
128
|
+
definition: JudgeMetricDefinition;
|
|
129
|
+
evalCase: EvalCase<TInput, TOutput>;
|
|
130
|
+
runState: RunState;
|
|
131
|
+
};
|
|
132
|
+
declare function evaluateJudgeMetric<TInput extends JsonValue, TOutput extends JsonValue>(input: EvaluateJudgeMetricInput<TInput, TOutput>): Promise<MetricResult>;
|
|
133
|
+
|
|
134
|
+
declare const approvalCorrectnessMetricName = "approval-correctness";
|
|
135
|
+
type ApprovalCorrectnessMetricInput = {
|
|
136
|
+
expectedApprovalDecisions: readonly ExpectedApprovalDecision[] | undefined;
|
|
137
|
+
approvals: readonly RunApproval[];
|
|
138
|
+
};
|
|
139
|
+
declare function evaluateApprovalCorrectnessMetric(input: ApprovalCorrectnessMetricInput): MetricResult;
|
|
140
|
+
|
|
141
|
+
declare const runStatusMetricName = "run-status";
|
|
142
|
+
type RunPerformance = {
|
|
143
|
+
totalLatencyMs: number | null;
|
|
144
|
+
tokenUsage: TokenUsage | null;
|
|
145
|
+
};
|
|
146
|
+
type CaseMetricsResult = {
|
|
147
|
+
caseName: string;
|
|
148
|
+
metrics: MetricResult[];
|
|
149
|
+
passed: boolean;
|
|
150
|
+
performance: RunPerformance;
|
|
151
|
+
};
|
|
152
|
+
type EvaluateCaseMetricsInput<TInput extends JsonValue, TOutput extends JsonValue> = {
|
|
153
|
+
runState: RunState;
|
|
154
|
+
evalCase: EvalCase<TInput, TOutput>;
|
|
155
|
+
workflow: WorkflowDefinition<TInput, TOutput>;
|
|
156
|
+
};
|
|
157
|
+
declare function collectRunPerformance(runState: RunState): RunPerformance;
|
|
158
|
+
declare function evaluateCaseMetrics<TInput extends JsonValue, TOutput extends JsonValue>(input: EvaluateCaseMetricsInput<TInput, TOutput>): CaseMetricsResult;
|
|
159
|
+
|
|
160
|
+
declare function isJsonObjectValue(value: JsonValue | null): value is {
|
|
161
|
+
[key: string]: JsonValue;
|
|
162
|
+
};
|
|
163
|
+
declare function areJsonValuesDeeplyEqual(left: JsonValue | null, right: JsonValue | null): boolean;
|
|
164
|
+
|
|
165
|
+
declare const schemaValidityMetricName = "schema-validity";
|
|
166
|
+
type SchemaValidityMetricInput = {
|
|
167
|
+
outputSchema: ValidationSchema<JsonValue> | null;
|
|
168
|
+
finalOutput: JsonValue | null;
|
|
169
|
+
};
|
|
170
|
+
declare function evaluateSchemaValidityMetric(input: SchemaValidityMetricInput): MetricResult;
|
|
171
|
+
|
|
172
|
+
type MetricBreakdown = {
|
|
173
|
+
evaluated: number;
|
|
174
|
+
passed: number;
|
|
175
|
+
failed: number;
|
|
176
|
+
skipped: number;
|
|
177
|
+
};
|
|
178
|
+
type SuitePerformanceSummary = {
|
|
179
|
+
totalLatencyMs: number | null;
|
|
180
|
+
averageLatencyMs: number | null;
|
|
181
|
+
tokenUsage: TokenUsage | null;
|
|
182
|
+
};
|
|
183
|
+
type SuiteMetricsSummary = {
|
|
184
|
+
totalCases: number;
|
|
185
|
+
passedCases: number;
|
|
186
|
+
failedCases: number;
|
|
187
|
+
passRate: number;
|
|
188
|
+
metricBreakdown: Record<string, MetricBreakdown>;
|
|
189
|
+
performance: SuitePerformanceSummary;
|
|
190
|
+
};
|
|
191
|
+
type EvalSuiteStatus = "passed" | "failed";
|
|
192
|
+
declare function determineEvalSuiteStatus(summary: SuiteMetricsSummary): EvalSuiteStatus;
|
|
193
|
+
declare function aggregateSuiteMetrics(caseResults: readonly CaseMetricsResult[]): SuiteMetricsSummary;
|
|
194
|
+
|
|
195
|
+
declare const defaultEvalResultsDirectory: string;
|
|
196
|
+
declare const EvalMetricResultSchema: z.ZodObject<{
|
|
197
|
+
name: z.ZodString;
|
|
198
|
+
passed: z.ZodBoolean;
|
|
199
|
+
score: z.ZodNumber;
|
|
200
|
+
reason: z.ZodString;
|
|
201
|
+
skipped: z.ZodBoolean;
|
|
202
|
+
}, z.core.$strip>;
|
|
203
|
+
declare const EvalCasePerformanceSchema: z.ZodObject<{
|
|
204
|
+
totalLatencyMs: z.ZodNullable<z.ZodNumber>;
|
|
205
|
+
tokenUsage: z.ZodNullable<z.ZodObject<{
|
|
206
|
+
inputTokens: z.ZodNumber;
|
|
207
|
+
outputTokens: z.ZodNumber;
|
|
208
|
+
totalTokens: z.ZodNumber;
|
|
209
|
+
cacheCreationInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
210
|
+
cacheReadInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
211
|
+
}, z.core.$strip>>;
|
|
212
|
+
}, z.core.$strip>;
|
|
213
|
+
declare const EvalCaseResultSchema: z.ZodObject<{
|
|
214
|
+
caseName: z.ZodString;
|
|
215
|
+
runId: z.ZodString;
|
|
216
|
+
passed: z.ZodBoolean;
|
|
217
|
+
metrics: z.ZodArray<z.ZodObject<{
|
|
218
|
+
name: z.ZodString;
|
|
219
|
+
passed: z.ZodBoolean;
|
|
220
|
+
score: z.ZodNumber;
|
|
221
|
+
reason: z.ZodString;
|
|
222
|
+
skipped: z.ZodBoolean;
|
|
223
|
+
}, z.core.$strip>>;
|
|
224
|
+
performance: z.ZodObject<{
|
|
225
|
+
totalLatencyMs: z.ZodNullable<z.ZodNumber>;
|
|
226
|
+
tokenUsage: z.ZodNullable<z.ZodObject<{
|
|
227
|
+
inputTokens: z.ZodNumber;
|
|
228
|
+
outputTokens: z.ZodNumber;
|
|
229
|
+
totalTokens: z.ZodNumber;
|
|
230
|
+
cacheCreationInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
231
|
+
cacheReadInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
232
|
+
}, z.core.$strip>>;
|
|
233
|
+
}, z.core.$strip>;
|
|
234
|
+
}, z.core.$strip>;
|
|
235
|
+
declare const EvalMetricBreakdownSchema: z.ZodObject<{
|
|
236
|
+
evaluated: z.ZodNumber;
|
|
237
|
+
passed: z.ZodNumber;
|
|
238
|
+
failed: z.ZodNumber;
|
|
239
|
+
skipped: z.ZodNumber;
|
|
240
|
+
}, z.core.$strip>;
|
|
241
|
+
declare const EvalSuitePerformanceSummarySchema: z.ZodObject<{
|
|
242
|
+
totalLatencyMs: z.ZodNullable<z.ZodNumber>;
|
|
243
|
+
averageLatencyMs: z.ZodNullable<z.ZodNumber>;
|
|
244
|
+
tokenUsage: z.ZodNullable<z.ZodObject<{
|
|
245
|
+
inputTokens: z.ZodNumber;
|
|
246
|
+
outputTokens: z.ZodNumber;
|
|
247
|
+
totalTokens: z.ZodNumber;
|
|
248
|
+
cacheCreationInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
249
|
+
cacheReadInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
250
|
+
}, z.core.$strip>>;
|
|
251
|
+
}, z.core.$strip>;
|
|
252
|
+
declare const EvalSuiteSummarySchema: z.ZodObject<{
|
|
253
|
+
totalCases: z.ZodNumber;
|
|
254
|
+
passedCases: z.ZodNumber;
|
|
255
|
+
failedCases: z.ZodNumber;
|
|
256
|
+
passRate: z.ZodNumber;
|
|
257
|
+
metricBreakdown: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
258
|
+
evaluated: z.ZodNumber;
|
|
259
|
+
passed: z.ZodNumber;
|
|
260
|
+
failed: z.ZodNumber;
|
|
261
|
+
skipped: z.ZodNumber;
|
|
262
|
+
}, z.core.$strip>>;
|
|
263
|
+
performance: z.ZodObject<{
|
|
264
|
+
totalLatencyMs: z.ZodNullable<z.ZodNumber>;
|
|
265
|
+
averageLatencyMs: z.ZodNullable<z.ZodNumber>;
|
|
266
|
+
tokenUsage: z.ZodNullable<z.ZodObject<{
|
|
267
|
+
inputTokens: z.ZodNumber;
|
|
268
|
+
outputTokens: z.ZodNumber;
|
|
269
|
+
totalTokens: z.ZodNumber;
|
|
270
|
+
cacheCreationInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
271
|
+
cacheReadInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
272
|
+
}, z.core.$strip>>;
|
|
273
|
+
}, z.core.$strip>;
|
|
274
|
+
}, z.core.$strip>;
|
|
275
|
+
declare const EvalRunFileSchema: z.ZodObject<{
|
|
276
|
+
evalRunId: z.ZodString;
|
|
277
|
+
suiteName: z.ZodString;
|
|
278
|
+
workflowName: z.ZodString;
|
|
279
|
+
startedAt: z.ZodISODateTime;
|
|
280
|
+
completedAt: z.ZodISODateTime;
|
|
281
|
+
summary: z.ZodObject<{
|
|
282
|
+
totalCases: z.ZodNumber;
|
|
283
|
+
passedCases: z.ZodNumber;
|
|
284
|
+
failedCases: z.ZodNumber;
|
|
285
|
+
passRate: z.ZodNumber;
|
|
286
|
+
metricBreakdown: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
287
|
+
evaluated: z.ZodNumber;
|
|
288
|
+
passed: z.ZodNumber;
|
|
289
|
+
failed: z.ZodNumber;
|
|
290
|
+
skipped: z.ZodNumber;
|
|
291
|
+
}, z.core.$strip>>;
|
|
292
|
+
performance: z.ZodObject<{
|
|
293
|
+
totalLatencyMs: z.ZodNullable<z.ZodNumber>;
|
|
294
|
+
averageLatencyMs: z.ZodNullable<z.ZodNumber>;
|
|
295
|
+
tokenUsage: z.ZodNullable<z.ZodObject<{
|
|
296
|
+
inputTokens: z.ZodNumber;
|
|
297
|
+
outputTokens: z.ZodNumber;
|
|
298
|
+
totalTokens: z.ZodNumber;
|
|
299
|
+
cacheCreationInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
300
|
+
cacheReadInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
301
|
+
}, z.core.$strip>>;
|
|
302
|
+
}, z.core.$strip>;
|
|
303
|
+
}, z.core.$strip>;
|
|
304
|
+
caseResults: z.ZodArray<z.ZodObject<{
|
|
305
|
+
caseName: z.ZodString;
|
|
306
|
+
runId: z.ZodString;
|
|
307
|
+
passed: z.ZodBoolean;
|
|
308
|
+
metrics: z.ZodArray<z.ZodObject<{
|
|
309
|
+
name: z.ZodString;
|
|
310
|
+
passed: z.ZodBoolean;
|
|
311
|
+
score: z.ZodNumber;
|
|
312
|
+
reason: z.ZodString;
|
|
313
|
+
skipped: z.ZodBoolean;
|
|
314
|
+
}, z.core.$strip>>;
|
|
315
|
+
performance: z.ZodObject<{
|
|
316
|
+
totalLatencyMs: z.ZodNullable<z.ZodNumber>;
|
|
317
|
+
tokenUsage: z.ZodNullable<z.ZodObject<{
|
|
318
|
+
inputTokens: z.ZodNumber;
|
|
319
|
+
outputTokens: z.ZodNumber;
|
|
320
|
+
totalTokens: z.ZodNumber;
|
|
321
|
+
cacheCreationInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
322
|
+
cacheReadInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
323
|
+
}, z.core.$strip>>;
|
|
324
|
+
}, z.core.$strip>;
|
|
325
|
+
}, z.core.$strip>>;
|
|
326
|
+
events: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
327
|
+
id: z.ZodString;
|
|
328
|
+
runId: z.ZodString;
|
|
329
|
+
workflowName: z.ZodString;
|
|
330
|
+
timestamp: z.ZodISODateTime;
|
|
331
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
332
|
+
redacted: z.ZodBoolean;
|
|
333
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
334
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
335
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
336
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
337
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
338
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
339
|
+
}, z.core.$strip>>;
|
|
340
|
+
type: z.ZodLiteral<"run.started">;
|
|
341
|
+
input: z.ZodType<_assemble_dev_shared_types.JsonValue, unknown, z.core.$ZodTypeInternals<_assemble_dev_shared_types.JsonValue, unknown>>;
|
|
342
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
343
|
+
id: z.ZodString;
|
|
344
|
+
runId: z.ZodString;
|
|
345
|
+
workflowName: z.ZodString;
|
|
346
|
+
timestamp: z.ZodISODateTime;
|
|
347
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
348
|
+
redacted: z.ZodBoolean;
|
|
349
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
350
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
351
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
352
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
353
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
354
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
355
|
+
}, z.core.$strip>>;
|
|
356
|
+
type: z.ZodLiteral<"run.completed">;
|
|
357
|
+
output: z.ZodType<_assemble_dev_shared_types.JsonValue, unknown, z.core.$ZodTypeInternals<_assemble_dev_shared_types.JsonValue, unknown>>;
|
|
358
|
+
latencyMs: z.ZodNumber;
|
|
359
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
360
|
+
id: z.ZodString;
|
|
361
|
+
runId: z.ZodString;
|
|
362
|
+
workflowName: z.ZodString;
|
|
363
|
+
timestamp: z.ZodISODateTime;
|
|
364
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
365
|
+
redacted: z.ZodBoolean;
|
|
366
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
367
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
368
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
369
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
370
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
371
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
372
|
+
}, z.core.$strip>>;
|
|
373
|
+
type: z.ZodLiteral<"run.failed">;
|
|
374
|
+
error: z.ZodObject<{
|
|
375
|
+
message: z.ZodString;
|
|
376
|
+
code: z.ZodOptional<z.ZodString>;
|
|
377
|
+
stack: z.ZodOptional<z.ZodString>;
|
|
378
|
+
}, z.core.$strip>;
|
|
379
|
+
latencyMs: z.ZodNumber;
|
|
380
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
381
|
+
id: z.ZodString;
|
|
382
|
+
runId: z.ZodString;
|
|
383
|
+
workflowName: z.ZodString;
|
|
384
|
+
timestamp: z.ZodISODateTime;
|
|
385
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
386
|
+
redacted: z.ZodBoolean;
|
|
387
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
388
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
389
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
390
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
391
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
392
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
393
|
+
}, z.core.$strip>>;
|
|
394
|
+
type: z.ZodLiteral<"agent.started">;
|
|
395
|
+
agentName: z.ZodString;
|
|
396
|
+
input: z.ZodType<_assemble_dev_shared_types.JsonValue, unknown, z.core.$ZodTypeInternals<_assemble_dev_shared_types.JsonValue, unknown>>;
|
|
397
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
398
|
+
id: z.ZodString;
|
|
399
|
+
runId: z.ZodString;
|
|
400
|
+
workflowName: z.ZodString;
|
|
401
|
+
timestamp: z.ZodISODateTime;
|
|
402
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
403
|
+
redacted: z.ZodBoolean;
|
|
404
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
405
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
406
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
407
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
408
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
409
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
410
|
+
}, z.core.$strip>>;
|
|
411
|
+
type: z.ZodLiteral<"agent.completed">;
|
|
412
|
+
agentName: z.ZodString;
|
|
413
|
+
output: z.ZodType<_assemble_dev_shared_types.JsonValue, unknown, z.core.$ZodTypeInternals<_assemble_dev_shared_types.JsonValue, unknown>>;
|
|
414
|
+
latencyMs: z.ZodNumber;
|
|
415
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
416
|
+
id: z.ZodString;
|
|
417
|
+
runId: z.ZodString;
|
|
418
|
+
workflowName: z.ZodString;
|
|
419
|
+
timestamp: z.ZodISODateTime;
|
|
420
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
421
|
+
redacted: z.ZodBoolean;
|
|
422
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
423
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
424
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
425
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
426
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
427
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
428
|
+
}, z.core.$strip>>;
|
|
429
|
+
type: z.ZodLiteral<"agent.failed">;
|
|
430
|
+
agentName: z.ZodString;
|
|
431
|
+
error: z.ZodObject<{
|
|
432
|
+
message: z.ZodString;
|
|
433
|
+
code: z.ZodOptional<z.ZodString>;
|
|
434
|
+
stack: z.ZodOptional<z.ZodString>;
|
|
435
|
+
}, z.core.$strip>;
|
|
436
|
+
latencyMs: z.ZodNumber;
|
|
437
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
438
|
+
id: z.ZodString;
|
|
439
|
+
runId: z.ZodString;
|
|
440
|
+
workflowName: z.ZodString;
|
|
441
|
+
timestamp: z.ZodISODateTime;
|
|
442
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
443
|
+
redacted: z.ZodBoolean;
|
|
444
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
445
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
446
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
447
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
448
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
449
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
450
|
+
}, z.core.$strip>>;
|
|
451
|
+
type: z.ZodLiteral<"tool.requested">;
|
|
452
|
+
toolName: z.ZodString;
|
|
453
|
+
agentName: z.ZodString;
|
|
454
|
+
input: z.ZodType<_assemble_dev_shared_types.JsonValue, unknown, z.core.$ZodTypeInternals<_assemble_dev_shared_types.JsonValue, unknown>>;
|
|
455
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
456
|
+
id: z.ZodString;
|
|
457
|
+
runId: z.ZodString;
|
|
458
|
+
workflowName: z.ZodString;
|
|
459
|
+
timestamp: z.ZodISODateTime;
|
|
460
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
461
|
+
redacted: z.ZodBoolean;
|
|
462
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
463
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
464
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
465
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
466
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
467
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
468
|
+
}, z.core.$strip>>;
|
|
469
|
+
type: z.ZodLiteral<"tool.approved">;
|
|
470
|
+
toolName: z.ZodString;
|
|
471
|
+
agentName: z.ZodString;
|
|
472
|
+
approvalId: z.ZodNullable<z.ZodString>;
|
|
473
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
474
|
+
id: z.ZodString;
|
|
475
|
+
runId: z.ZodString;
|
|
476
|
+
workflowName: z.ZodString;
|
|
477
|
+
timestamp: z.ZodISODateTime;
|
|
478
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
479
|
+
redacted: z.ZodBoolean;
|
|
480
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
481
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
482
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
483
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
484
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
485
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
486
|
+
}, z.core.$strip>>;
|
|
487
|
+
type: z.ZodLiteral<"tool.blocked">;
|
|
488
|
+
toolName: z.ZodString;
|
|
489
|
+
agentName: z.ZodString;
|
|
490
|
+
policyName: z.ZodString;
|
|
491
|
+
reason: z.ZodString;
|
|
492
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
493
|
+
id: z.ZodString;
|
|
494
|
+
runId: z.ZodString;
|
|
495
|
+
workflowName: z.ZodString;
|
|
496
|
+
timestamp: z.ZodISODateTime;
|
|
497
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
498
|
+
redacted: z.ZodBoolean;
|
|
499
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
500
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
501
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
502
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
503
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
504
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
505
|
+
}, z.core.$strip>>;
|
|
506
|
+
type: z.ZodLiteral<"tool.completed">;
|
|
507
|
+
toolName: z.ZodString;
|
|
508
|
+
agentName: z.ZodString;
|
|
509
|
+
output: z.ZodType<_assemble_dev_shared_types.JsonValue, unknown, z.core.$ZodTypeInternals<_assemble_dev_shared_types.JsonValue, unknown>>;
|
|
510
|
+
latencyMs: z.ZodNumber;
|
|
511
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
512
|
+
id: z.ZodString;
|
|
513
|
+
runId: z.ZodString;
|
|
514
|
+
workflowName: z.ZodString;
|
|
515
|
+
timestamp: z.ZodISODateTime;
|
|
516
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
517
|
+
redacted: z.ZodBoolean;
|
|
518
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
519
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
520
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
521
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
522
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
523
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
524
|
+
}, z.core.$strip>>;
|
|
525
|
+
type: z.ZodLiteral<"tool.failed">;
|
|
526
|
+
toolName: z.ZodString;
|
|
527
|
+
agentName: z.ZodString;
|
|
528
|
+
error: z.ZodObject<{
|
|
529
|
+
message: z.ZodString;
|
|
530
|
+
code: z.ZodOptional<z.ZodString>;
|
|
531
|
+
stack: z.ZodOptional<z.ZodString>;
|
|
532
|
+
}, z.core.$strip>;
|
|
533
|
+
latencyMs: z.ZodNumber;
|
|
534
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
535
|
+
id: z.ZodString;
|
|
536
|
+
runId: z.ZodString;
|
|
537
|
+
workflowName: z.ZodString;
|
|
538
|
+
timestamp: z.ZodISODateTime;
|
|
539
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
540
|
+
redacted: z.ZodBoolean;
|
|
541
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
542
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
543
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
544
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
545
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
546
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
547
|
+
}, z.core.$strip>>;
|
|
548
|
+
type: z.ZodLiteral<"policy.evaluated">;
|
|
549
|
+
policyName: z.ZodString;
|
|
550
|
+
action: z.ZodEnum<{
|
|
551
|
+
allow: "allow";
|
|
552
|
+
block: "block";
|
|
553
|
+
requireApproval: "requireApproval";
|
|
554
|
+
logOnly: "logOnly";
|
|
555
|
+
redact: "redact";
|
|
556
|
+
}>;
|
|
557
|
+
agentName: z.ZodOptional<z.ZodString>;
|
|
558
|
+
toolName: z.ZodOptional<z.ZodString>;
|
|
559
|
+
reason: z.ZodString;
|
|
560
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
561
|
+
id: z.ZodString;
|
|
562
|
+
runId: z.ZodString;
|
|
563
|
+
workflowName: z.ZodString;
|
|
564
|
+
timestamp: z.ZodISODateTime;
|
|
565
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
566
|
+
redacted: z.ZodBoolean;
|
|
567
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
568
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
569
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
570
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
571
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
572
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
573
|
+
}, z.core.$strip>>;
|
|
574
|
+
type: z.ZodLiteral<"route.selected">;
|
|
575
|
+
supervisorName: z.ZodString;
|
|
576
|
+
selectedAgentName: z.ZodString;
|
|
577
|
+
candidateAgentNames: z.ZodArray<z.ZodString>;
|
|
578
|
+
reason: z.ZodString;
|
|
579
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
580
|
+
id: z.ZodString;
|
|
581
|
+
runId: z.ZodString;
|
|
582
|
+
workflowName: z.ZodString;
|
|
583
|
+
timestamp: z.ZodISODateTime;
|
|
584
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
585
|
+
redacted: z.ZodBoolean;
|
|
586
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
587
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
588
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
589
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
590
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
591
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
592
|
+
}, z.core.$strip>>;
|
|
593
|
+
type: z.ZodLiteral<"approval.required">;
|
|
594
|
+
approvalId: z.ZodString;
|
|
595
|
+
toolName: z.ZodOptional<z.ZodString>;
|
|
596
|
+
agentName: z.ZodOptional<z.ZodString>;
|
|
597
|
+
reason: z.ZodString;
|
|
598
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
599
|
+
id: z.ZodString;
|
|
600
|
+
runId: z.ZodString;
|
|
601
|
+
workflowName: z.ZodString;
|
|
602
|
+
timestamp: z.ZodISODateTime;
|
|
603
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
604
|
+
redacted: z.ZodBoolean;
|
|
605
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
606
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
607
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
608
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
609
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
610
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
611
|
+
}, z.core.$strip>>;
|
|
612
|
+
type: z.ZodLiteral<"approval.approved">;
|
|
613
|
+
approvalId: z.ZodString;
|
|
614
|
+
decidedBy: z.ZodNullable<z.ZodString>;
|
|
615
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
616
|
+
id: z.ZodString;
|
|
617
|
+
runId: z.ZodString;
|
|
618
|
+
workflowName: z.ZodString;
|
|
619
|
+
timestamp: z.ZodISODateTime;
|
|
620
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
621
|
+
redacted: z.ZodBoolean;
|
|
622
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
623
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
624
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
625
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
626
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
627
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
628
|
+
}, z.core.$strip>>;
|
|
629
|
+
type: z.ZodLiteral<"approval.rejected">;
|
|
630
|
+
approvalId: z.ZodString;
|
|
631
|
+
decidedBy: z.ZodNullable<z.ZodString>;
|
|
632
|
+
reason: z.ZodString;
|
|
633
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
634
|
+
id: z.ZodString;
|
|
635
|
+
runId: z.ZodString;
|
|
636
|
+
workflowName: z.ZodString;
|
|
637
|
+
timestamp: z.ZodISODateTime;
|
|
638
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
639
|
+
redacted: z.ZodBoolean;
|
|
640
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
641
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
642
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
643
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
644
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
645
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
646
|
+
}, z.core.$strip>>;
|
|
647
|
+
type: z.ZodLiteral<"eval.started">;
|
|
648
|
+
evalRunId: z.ZodString;
|
|
649
|
+
suiteName: z.ZodString;
|
|
650
|
+
caseCount: z.ZodNumber;
|
|
651
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
652
|
+
id: z.ZodString;
|
|
653
|
+
runId: z.ZodString;
|
|
654
|
+
workflowName: z.ZodString;
|
|
655
|
+
timestamp: z.ZodISODateTime;
|
|
656
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
657
|
+
redacted: z.ZodBoolean;
|
|
658
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
659
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
660
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
661
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
662
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
663
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
664
|
+
}, z.core.$strip>>;
|
|
665
|
+
type: z.ZodLiteral<"eval.completed">;
|
|
666
|
+
evalRunId: z.ZodString;
|
|
667
|
+
passedCaseCount: z.ZodNumber;
|
|
668
|
+
failedCaseCount: z.ZodNumber;
|
|
669
|
+
latencyMs: z.ZodNumber;
|
|
670
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
671
|
+
id: z.ZodString;
|
|
672
|
+
runId: z.ZodString;
|
|
673
|
+
workflowName: z.ZodString;
|
|
674
|
+
timestamp: z.ZodISODateTime;
|
|
675
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
676
|
+
redacted: z.ZodBoolean;
|
|
677
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
678
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
679
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
680
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
681
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
682
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
683
|
+
}, z.core.$strip>>;
|
|
684
|
+
type: z.ZodLiteral<"replay.started">;
|
|
685
|
+
sourceRunId: z.ZodString;
|
|
686
|
+
fromEventId: z.ZodNullable<z.ZodString>;
|
|
687
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
688
|
+
id: z.ZodString;
|
|
689
|
+
runId: z.ZodString;
|
|
690
|
+
workflowName: z.ZodString;
|
|
691
|
+
timestamp: z.ZodISODateTime;
|
|
692
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
693
|
+
redacted: z.ZodBoolean;
|
|
694
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
695
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
696
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
697
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
698
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
699
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
700
|
+
}, z.core.$strip>>;
|
|
701
|
+
type: z.ZodLiteral<"replay.completed">;
|
|
702
|
+
sourceRunId: z.ZodString;
|
|
703
|
+
latencyMs: z.ZodNumber;
|
|
704
|
+
}, z.core.$strip>], "type">>;
|
|
705
|
+
}, z.core.$strip>;
|
|
706
|
+
type EvalCaseRunResult = z.infer<typeof EvalCaseResultSchema>;
|
|
707
|
+
type EvalRunFile = z.infer<typeof EvalRunFileSchema>;
|
|
708
|
+
declare function resolveEvalRunFilePath(evalRunId: string, directory?: string): string;
|
|
709
|
+
type SaveEvalRunFileInput = {
|
|
710
|
+
evalRunFile: EvalRunFile;
|
|
711
|
+
directory?: string;
|
|
712
|
+
};
|
|
713
|
+
declare function saveEvalRunFile(input: SaveEvalRunFileInput): string;
|
|
714
|
+
type LoadEvalRunFileInput = {
|
|
715
|
+
evalRunId: string;
|
|
716
|
+
directory?: string;
|
|
717
|
+
};
|
|
718
|
+
declare function loadEvalRunFile(input: LoadEvalRunFileInput): EvalRunFile;
|
|
719
|
+
|
|
720
|
+
type EvalToolMock = (input: JsonValue) => JsonValue;
|
|
721
|
+
type EvalPlatformOptions = {
|
|
722
|
+
apiKey?: string;
|
|
723
|
+
baseUrl?: string;
|
|
724
|
+
fetchImplementation?: PlatformFetchImplementation;
|
|
725
|
+
environmentVariables?: Record<string, string | undefined>;
|
|
726
|
+
homeDirectory?: string;
|
|
727
|
+
};
|
|
728
|
+
type RunEvalSuiteOptions = {
|
|
729
|
+
toolMocks?: Record<string, EvalToolMock>;
|
|
730
|
+
judges?: JudgeMetricDefinition[];
|
|
731
|
+
sinks?: TraceSink[];
|
|
732
|
+
resultsDirectory?: string;
|
|
733
|
+
platform?: EvalPlatformOptions;
|
|
734
|
+
now?: () => Date;
|
|
735
|
+
evalRunId?: string;
|
|
736
|
+
writeLine?: (line: string) => void;
|
|
737
|
+
};
|
|
738
|
+
type RunEvalSuiteInput<TInput extends JsonValue, TOutput extends JsonValue> = {
|
|
739
|
+
suite: EvalSuiteDefinition<TInput, TOutput>;
|
|
740
|
+
options?: RunEvalSuiteOptions;
|
|
741
|
+
};
|
|
742
|
+
type EvalUploadStatus = "uploaded" | "skipped-no-key" | "failed";
|
|
743
|
+
type RunEvalSuiteResult = {
|
|
744
|
+
evalRunId: string;
|
|
745
|
+
suiteName: string;
|
|
746
|
+
summary: SuiteMetricsSummary;
|
|
747
|
+
caseResults: EvalCaseRunResult[];
|
|
748
|
+
resultsFilePath: string;
|
|
749
|
+
uploadStatus: EvalUploadStatus;
|
|
750
|
+
};
|
|
751
|
+
declare function runEvalSuite<TInput extends JsonValue, TOutput extends JsonValue>(input: RunEvalSuiteInput<TInput, TOutput>): Promise<RunEvalSuiteResult>;
|
|
752
|
+
declare function resolveEvalRunsUploadUrl(baseUrl: string): string;
|
|
753
|
+
|
|
754
|
+
declare const EvalWebhookCaseSchema: z.ZodObject<{
|
|
755
|
+
caseName: z.ZodString;
|
|
756
|
+
passed: z.ZodBoolean;
|
|
757
|
+
metrics: z.ZodArray<z.ZodObject<{
|
|
758
|
+
name: z.ZodString;
|
|
759
|
+
passed: z.ZodBoolean;
|
|
760
|
+
score: z.ZodNumber;
|
|
761
|
+
reason: z.ZodString;
|
|
762
|
+
skipped: z.ZodBoolean;
|
|
763
|
+
}, z.core.$strip>>;
|
|
764
|
+
}, z.core.$strip>;
|
|
765
|
+
declare const EvalWebhookPayloadSchema: z.ZodObject<{
|
|
766
|
+
suiteName: z.ZodString;
|
|
767
|
+
status: z.ZodEnum<{
|
|
768
|
+
failed: "failed";
|
|
769
|
+
passed: "passed";
|
|
770
|
+
}>;
|
|
771
|
+
totalCases: z.ZodNumber;
|
|
772
|
+
passedCases: z.ZodNumber;
|
|
773
|
+
failedCases: z.ZodNumber;
|
|
774
|
+
cases: z.ZodArray<z.ZodObject<{
|
|
775
|
+
caseName: z.ZodString;
|
|
776
|
+
passed: z.ZodBoolean;
|
|
777
|
+
metrics: z.ZodArray<z.ZodObject<{
|
|
778
|
+
name: z.ZodString;
|
|
779
|
+
passed: z.ZodBoolean;
|
|
780
|
+
score: z.ZodNumber;
|
|
781
|
+
reason: z.ZodString;
|
|
782
|
+
skipped: z.ZodBoolean;
|
|
783
|
+
}, z.core.$strip>>;
|
|
784
|
+
}, z.core.$strip>>;
|
|
785
|
+
}, z.core.$strip>;
|
|
786
|
+
type EvalWebhookCase = z.infer<typeof EvalWebhookCaseSchema>;
|
|
787
|
+
type EvalWebhookPayload = z.infer<typeof EvalWebhookPayloadSchema>;
|
|
788
|
+
declare function buildEvalWebhookPayload(runResult: RunEvalSuiteResult): EvalWebhookPayload;
|
|
789
|
+
type PostEvalResultsWebhookInput = {
|
|
790
|
+
webhookUrl: string;
|
|
791
|
+
payload: EvalWebhookPayload;
|
|
792
|
+
fetchImplementation?: PlatformFetchImplementation;
|
|
793
|
+
};
|
|
794
|
+
type PostEvalResultsWebhookResult = {
|
|
795
|
+
outcome: "delivered";
|
|
796
|
+
} | {
|
|
797
|
+
outcome: "failed";
|
|
798
|
+
message: string;
|
|
799
|
+
};
|
|
800
|
+
declare function postEvalResultsToWebhook(input: PostEvalResultsWebhookInput): Promise<PostEvalResultsWebhookResult>;
|
|
801
|
+
|
|
802
|
+
declare const taskSuccessMetricName = "task-success";
|
|
803
|
+
type TaskSuccessMetricInput = {
|
|
804
|
+
expectation: EvalCaseExpectation<JsonValue> | undefined;
|
|
805
|
+
finalOutput: JsonValue | null;
|
|
806
|
+
};
|
|
807
|
+
declare function evaluateTaskSuccessMetric(input: TaskSuccessMetricInput): MetricResult;
|
|
808
|
+
|
|
809
|
+
declare const toolAccuracyMetricName = "tool-accuracy";
|
|
810
|
+
type ToolAccuracyMetricInput = {
|
|
811
|
+
expectedToolCalls: readonly ExpectedToolCall[] | undefined;
|
|
812
|
+
toolCalls: readonly ToolCall[];
|
|
813
|
+
};
|
|
814
|
+
declare function evaluateToolAccuracyMetric(input: ToolAccuracyMetricInput): MetricResult;
|
|
815
|
+
|
|
816
|
+
export { type ApprovalCorrectnessMetricInput, type CaseMetricsResult, type DefineJudgeMetricConfig, type EvalCase, type EvalCaseExpectation, EvalCasePerformanceSchema, EvalCaseResultSchema, type EvalCaseRunResult, type EvalCasesSource, EvalMetricBreakdownSchema, EvalMetricResultSchema, type EvalPlatformOptions, type EvalRunFile, EvalRunFileSchema, type EvalSuiteConfig, type EvalSuiteDefinition, EvalSuitePerformanceSummarySchema, type EvalSuiteStatus, EvalSuiteSummarySchema, type EvalToolMock, type EvalUploadStatus, type EvalWebhookCase, EvalWebhookCaseSchema, type EvalWebhookPayload, EvalWebhookPayloadSchema, type EvaluateCaseMetricsInput, type EvaluateJudgeMetricInput, type ExpectedApprovalDecision, ExpectedApprovalDecisionSchema, type ExpectedToolCall, ExpectedToolCallSchema, type JudgeMetricDefinition, type JudgeVerdict, JudgeVerdictSchema, type LoadEvalCasesFromJsonlFileInput, type LoadEvalRunFileInput, type MetricBreakdown, type MetricExpectationOutcome, type MetricResult, type PostEvalResultsWebhookInput, type PostEvalResultsWebhookResult, type RunEvalSuiteInput, type RunEvalSuiteOptions, type RunEvalSuiteResult, type RunPerformance, type SaveEvalRunFileInput, type SchemaValidityMetricInput, type SerializedEvalCase, SerializedEvalCaseExpectationSchema, SerializedEvalCaseSchema, type SuiteMetricsSummary, type SuitePerformanceSummary, type TaskSuccessMetricInput, type ToolAccuracyMetricInput, aggregateSuiteMetrics, approvalCorrectnessMetricName, areJsonValuesDeeplyEqual, buildEvalWebhookPayload, collectRunPerformance, createFailedMetricResult, createPassedMetricResult, createScoredMetricResult, createSkippedMetricResult, defaultEvalResultsDirectory, defaultJudgeMetricName, defaultJudgePassThreshold, defineJudgeMetric, determineEvalSuiteStatus, evalSuite, evaluateApprovalCorrectnessMetric, evaluateCaseMetrics, evaluateJudgeMetric, evaluateSchemaValidityMetric, evaluateTaskSuccessMetric, evaluateToolAccuracyMetric, isJsonObjectValue, loadEvalCases, loadEvalCasesFromJsonlFile, loadEvalRunFile, postEvalResultsToWebhook, resolveEvalRunFilePath, resolveEvalRunsUploadUrl, runEvalSuite, runStatusMetricName, saveEvalRunFile, schemaValidityMetricName, taskSuccessMetricName, toolAccuracyMetricName };
|