@kortyx/agent 0.27.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +27 -0
- package/dist/browser.d.ts +15 -2
- package/dist/chat-message-D4EsFcr1.d.ts +9 -0
- package/dist/evals.d.ts +1016 -0
- package/dist/evals.js +551 -0
- package/dist/evals.js.map +1 -0
- package/dist/index.d.ts +55 -201
- package/dist/index.js +2030 -141
- package/dist/index.js.map +1 -1
- package/dist/studio-judge-BFF0TsT2.d.ts +2279 -0
- package/package.json +9 -5
- package/dist/browser-EVWVR9BO.d.ts +0 -22
package/dist/evals.js
ADDED
|
@@ -0,0 +1,551 @@
|
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
3
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
4
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
5
|
+
var __export = (target, all) => {
|
|
6
|
+
for (var name in all)
|
|
7
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
8
|
+
};
|
|
9
|
+
var __copyProps = (to, from, except, desc) => {
|
|
10
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
11
|
+
for (let key of __getOwnPropNames(from))
|
|
12
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
13
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
14
|
+
}
|
|
15
|
+
return to;
|
|
16
|
+
};
|
|
17
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
18
|
+
|
|
19
|
+
// src/evals.ts
|
|
20
|
+
var evals_exports = {};
|
|
21
|
+
__export(evals_exports, {
|
|
22
|
+
EvalCaseResultSchema: () => EvalCaseResultSchema,
|
|
23
|
+
EvalConfigurationError: () => EvalConfigurationError,
|
|
24
|
+
EvalCostAmountSchema: () => EvalCostAmountSchema,
|
|
25
|
+
EvalCostsSchema: () => EvalCostsSchema,
|
|
26
|
+
EvalJudgeIdentitySchema: () => EvalJudgeIdentitySchema,
|
|
27
|
+
EvalJudgeUsageSchema: () => EvalJudgeUsageSchema,
|
|
28
|
+
EvalManifestSchema: () => EvalManifestSchema,
|
|
29
|
+
EvalObservationSchema: () => EvalObservationSchema,
|
|
30
|
+
EvalProgressSchema: () => EvalProgressSchema,
|
|
31
|
+
EvalRemoteRunRequestSchema: () => EvalRemoteRunRequestSchema,
|
|
32
|
+
EvalResumeResponseSchema: () => EvalResumeResponseSchema,
|
|
33
|
+
EvalRunResultSchema: () => EvalRunResultSchema,
|
|
34
|
+
EvalStepResultSchema: () => EvalStepResultSchema,
|
|
35
|
+
EvalSuiteSchema: () => EvalSuiteSchema,
|
|
36
|
+
EvalVerdictSchema: () => EvalVerdictSchema,
|
|
37
|
+
EvalWireEventSchema: () => EvalWireEventSchema,
|
|
38
|
+
StudioEvalDetailSchema: () => StudioEvalDetailSchema,
|
|
39
|
+
StudioEvalHistorySchema: () => StudioEvalHistorySchema,
|
|
40
|
+
StudioEvalJudgeRequestSchema: () => StudioEvalJudgeRequestSchema,
|
|
41
|
+
StudioEvalJudgeResponseSchema: () => StudioEvalJudgeResponseSchema,
|
|
42
|
+
StudioEvalRunSummarySchema: () => StudioEvalRunSummarySchema,
|
|
43
|
+
StudioEvalStartRequestSchema: () => StudioEvalStartRequestSchema,
|
|
44
|
+
StudioEvalTargetsResponseSchema: () => StudioEvalTargetsResponseSchema,
|
|
45
|
+
createStudioEvalJudge: () => createStudioEvalJudge,
|
|
46
|
+
defineSuite: () => defineSuite,
|
|
47
|
+
parseEvalSuite: () => parseEvalSuite
|
|
48
|
+
});
|
|
49
|
+
module.exports = __toCommonJS(evals_exports);
|
|
50
|
+
|
|
51
|
+
// src/evals/contracts.ts
|
|
52
|
+
var import_zod = require("zod");
|
|
53
|
+
var id = import_zod.z.string().trim().min(1).max(256);
|
|
54
|
+
var handler = import_zod.z.object({ using: id, params: import_zod.z.json().optional() }).strict();
|
|
55
|
+
var EvalResumeResponseSchema = import_zod.z.discriminatedUnion("type", [
|
|
56
|
+
import_zod.z.object({ type: import_zod.z.literal("text"), text: import_zod.z.string() }).strict(),
|
|
57
|
+
import_zod.z.object({ type: import_zod.z.literal("select"), ids: import_zod.z.array(import_zod.z.string()) }).strict(),
|
|
58
|
+
import_zod.z.object({ type: import_zod.z.literal("value"), value: import_zod.z.json() }).strict(),
|
|
59
|
+
import_zod.z.object({ type: import_zod.z.literal("cancel") }).strict()
|
|
60
|
+
]);
|
|
61
|
+
var expectation = import_zod.z.object({
|
|
62
|
+
type: import_zod.z.enum(["answer", "interrupt"]),
|
|
63
|
+
schemaId: id.optional(),
|
|
64
|
+
schemaVersion: id.optional(),
|
|
65
|
+
outputs: import_zod.z.array(import_zod.z.object({ schemaId: id, schemaVersion: id.optional() }).strict()).optional(),
|
|
66
|
+
criteria: import_zod.z.array(
|
|
67
|
+
import_zod.z.union([
|
|
68
|
+
import_zod.z.string().trim().min(1),
|
|
69
|
+
import_zod.z.object({ id, text: import_zod.z.string().trim().min(1) }).strict()
|
|
70
|
+
])
|
|
71
|
+
).optional(),
|
|
72
|
+
reference: import_zod.z.json().optional()
|
|
73
|
+
}).strict().superRefine((value, ctx) => {
|
|
74
|
+
if (value.type === "answer" && (value.schemaId || value.schemaVersion))
|
|
75
|
+
ctx.addIssue({
|
|
76
|
+
code: "custom",
|
|
77
|
+
message: "Interrupt schemas require an interrupt expectation."
|
|
78
|
+
});
|
|
79
|
+
const ids = value.criteria?.map(
|
|
80
|
+
(item, index) => typeof item === "string" ? String(index) : item.id
|
|
81
|
+
) ?? [];
|
|
82
|
+
if (new Set(ids).size !== ids.length)
|
|
83
|
+
ctx.addIssue({
|
|
84
|
+
code: "custom",
|
|
85
|
+
message: "Criterion IDs must be unique within a step."
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
var EvalSuiteSchema = import_zod.z.object({
|
|
89
|
+
id,
|
|
90
|
+
name: import_zod.z.string().optional(),
|
|
91
|
+
cases: import_zod.z.array(
|
|
92
|
+
import_zod.z.object({
|
|
93
|
+
id,
|
|
94
|
+
name: import_zod.z.string().optional(),
|
|
95
|
+
params: import_zod.z.json().optional(),
|
|
96
|
+
workflowId: id.optional(),
|
|
97
|
+
steps: import_zod.z.array(
|
|
98
|
+
import_zod.z.union([
|
|
99
|
+
import_zod.z.object({ message: import_zod.z.string().min(1), expect: expectation }).strict(),
|
|
100
|
+
import_zod.z.object({
|
|
101
|
+
resume: import_zod.z.union([EvalResumeResponseSchema, handler]),
|
|
102
|
+
expect: expectation
|
|
103
|
+
}).strict()
|
|
104
|
+
])
|
|
105
|
+
).min(1)
|
|
106
|
+
}).strict()
|
|
107
|
+
).min(1)
|
|
108
|
+
}).strict();
|
|
109
|
+
var EvalVerdictSchema = import_zod.z.object({
|
|
110
|
+
passed: import_zod.z.boolean(),
|
|
111
|
+
reason: import_zod.z.string().trim().min(1),
|
|
112
|
+
evidence: import_zod.z.array(import_zod.z.string().min(1))
|
|
113
|
+
}).strict();
|
|
114
|
+
var EvalObservationSchema = import_zod.z.object({
|
|
115
|
+
type: import_zod.z.enum(["answer", "interrupt", "error", "cancelled"]),
|
|
116
|
+
text: import_zod.z.string(),
|
|
117
|
+
structured: import_zod.z.array(import_zod.z.json()),
|
|
118
|
+
events: import_zod.z.array(import_zod.z.json()).optional(),
|
|
119
|
+
runId: import_zod.z.string().optional(),
|
|
120
|
+
checkpointId: import_zod.z.string().optional(),
|
|
121
|
+
interrupt: import_zod.z.object({
|
|
122
|
+
requestId: import_zod.z.string(),
|
|
123
|
+
kind: import_zod.z.enum(["text", "choice", "multi-choice", "custom"]),
|
|
124
|
+
question: import_zod.z.string().optional(),
|
|
125
|
+
schemaId: import_zod.z.string().optional(),
|
|
126
|
+
schemaVersion: import_zod.z.string().optional(),
|
|
127
|
+
options: import_zod.z.array(
|
|
128
|
+
import_zod.z.object({
|
|
129
|
+
id: import_zod.z.string(),
|
|
130
|
+
label: import_zod.z.string(),
|
|
131
|
+
description: import_zod.z.string().optional()
|
|
132
|
+
}).strict()
|
|
133
|
+
),
|
|
134
|
+
request: import_zod.z.json().optional()
|
|
135
|
+
}).strict().optional()
|
|
136
|
+
}).strict().superRefine((value, ctx) => {
|
|
137
|
+
if (value.type === "interrupt" !== Boolean(value.interrupt))
|
|
138
|
+
ctx.addIssue({
|
|
139
|
+
code: "custom",
|
|
140
|
+
message: "Interrupt observations require exactly one interrupt."
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
var status = import_zod.z.enum(["passed", "failed", "error", "cancelled", "ungraded"]);
|
|
144
|
+
var issue = import_zod.z.object({
|
|
145
|
+
phase: import_zod.z.enum([
|
|
146
|
+
"params",
|
|
147
|
+
"setup",
|
|
148
|
+
"execute",
|
|
149
|
+
"responder",
|
|
150
|
+
"reference",
|
|
151
|
+
"grading",
|
|
152
|
+
"cleanup",
|
|
153
|
+
"reporting"
|
|
154
|
+
]),
|
|
155
|
+
code: import_zod.z.string(),
|
|
156
|
+
message: import_zod.z.string()
|
|
157
|
+
}).strict();
|
|
158
|
+
var EvalJudgeUsageSchema = import_zod.z.object({
|
|
159
|
+
provider: import_zod.z.string().min(1),
|
|
160
|
+
model: import_zod.z.string().min(1),
|
|
161
|
+
occurredAt: import_zod.z.iso.datetime(),
|
|
162
|
+
usage: import_zod.z.object({
|
|
163
|
+
input: import_zod.z.number().finite().nonnegative().optional(),
|
|
164
|
+
output: import_zod.z.number().finite().nonnegative().optional(),
|
|
165
|
+
total: import_zod.z.number().finite().nonnegative().optional(),
|
|
166
|
+
reasoning: import_zod.z.number().finite().nonnegative().optional(),
|
|
167
|
+
cacheRead: import_zod.z.number().finite().nonnegative().optional(),
|
|
168
|
+
cacheWrite: import_zod.z.number().finite().nonnegative().optional(),
|
|
169
|
+
cacheWrite1h: import_zod.z.number().finite().nonnegative().optional(),
|
|
170
|
+
outputIncludesReasoning: import_zod.z.boolean().optional(),
|
|
171
|
+
inputIncludesCacheRead: import_zod.z.boolean().optional(),
|
|
172
|
+
inputIncludesCacheWrite: import_zod.z.boolean().optional()
|
|
173
|
+
}).strict().optional(),
|
|
174
|
+
pricingContext: import_zod.z.object({
|
|
175
|
+
serviceTier: import_zod.z.string().min(1).max(100).optional(),
|
|
176
|
+
inferenceGeo: import_zod.z.string().min(1).max(100).optional(),
|
|
177
|
+
speed: import_zod.z.string().min(1).max(100).optional()
|
|
178
|
+
}).strict().optional(),
|
|
179
|
+
pricing: import_zod.z.object({
|
|
180
|
+
source: import_zod.z.literal("provider"),
|
|
181
|
+
currency: import_zod.z.literal("USD"),
|
|
182
|
+
actualCostMicros: import_zod.z.number().int().nonnegative().safe()
|
|
183
|
+
}).strict().optional()
|
|
184
|
+
}).strict();
|
|
185
|
+
var EvalStepResultSchema = import_zod.z.object({
|
|
186
|
+
index: import_zod.z.number().int().nonnegative(),
|
|
187
|
+
input: import_zod.z.union([
|
|
188
|
+
import_zod.z.object({ message: import_zod.z.string() }).strict(),
|
|
189
|
+
import_zod.z.object({ resume: EvalResumeResponseSchema }).strict()
|
|
190
|
+
]),
|
|
191
|
+
expectation,
|
|
192
|
+
observation: EvalObservationSchema,
|
|
193
|
+
reference: import_zod.z.json().optional(),
|
|
194
|
+
status,
|
|
195
|
+
reason: import_zod.z.string().optional(),
|
|
196
|
+
judgeCalls: import_zod.z.number().int().nonnegative().optional(),
|
|
197
|
+
judgeUsage: import_zod.z.array(EvalJudgeUsageSchema).optional(),
|
|
198
|
+
criteria: import_zod.z.array(
|
|
199
|
+
EvalVerdictSchema.extend({ id: import_zod.z.string(), text: import_zod.z.string() })
|
|
200
|
+
)
|
|
201
|
+
}).strict();
|
|
202
|
+
var EvalCaseResultSchema = import_zod.z.object({
|
|
203
|
+
caseId: id,
|
|
204
|
+
repetition: import_zod.z.number().int().positive(),
|
|
205
|
+
sessionId: import_zod.z.string(),
|
|
206
|
+
status,
|
|
207
|
+
durationMs: import_zod.z.number().nonnegative(),
|
|
208
|
+
steps: import_zod.z.array(EvalStepResultSchema),
|
|
209
|
+
errors: import_zod.z.array(issue)
|
|
210
|
+
}).strict();
|
|
211
|
+
var EvalJudgeIdentitySchema = import_zod.z.object({
|
|
212
|
+
id,
|
|
213
|
+
version: id,
|
|
214
|
+
location: import_zod.z.enum(["app", "studio"]).optional()
|
|
215
|
+
}).strict();
|
|
216
|
+
var judge = EvalJudgeIdentitySchema;
|
|
217
|
+
var StudioEvalJudgeRequestSchema = import_zod.z.object({
|
|
218
|
+
environment: id,
|
|
219
|
+
judge: EvalJudgeIdentitySchema,
|
|
220
|
+
criterion: import_zod.z.object({ id, text: import_zod.z.string().trim().min(1).max(16384) }).strict(),
|
|
221
|
+
input: EvalStepResultSchema.shape.input,
|
|
222
|
+
observation: EvalObservationSchema,
|
|
223
|
+
reference: import_zod.z.json().optional(),
|
|
224
|
+
conversation: import_zod.z.array(EvalStepResultSchema).max(100)
|
|
225
|
+
}).strict();
|
|
226
|
+
var StudioEvalJudgeResponseSchema = import_zod.z.object({
|
|
227
|
+
judge: EvalJudgeIdentitySchema,
|
|
228
|
+
verdict: EvalVerdictSchema,
|
|
229
|
+
usage: import_zod.z.array(EvalJudgeUsageSchema).optional()
|
|
230
|
+
}).strict();
|
|
231
|
+
var EvalRunResultSchema = import_zod.z.object({
|
|
232
|
+
id: import_zod.z.uuid(),
|
|
233
|
+
suiteId: id,
|
|
234
|
+
suiteRevision: import_zod.z.string().regex(/^[a-f0-9]{64}$/),
|
|
235
|
+
suite: EvalSuiteSchema,
|
|
236
|
+
judge: judge.optional(),
|
|
237
|
+
startedAt: import_zod.z.iso.datetime(),
|
|
238
|
+
durationMs: import_zod.z.number().nonnegative(),
|
|
239
|
+
status,
|
|
240
|
+
counts: import_zod.z.object({
|
|
241
|
+
passed: import_zod.z.number().int().nonnegative(),
|
|
242
|
+
failed: import_zod.z.number().int().nonnegative(),
|
|
243
|
+
error: import_zod.z.number().int().nonnegative(),
|
|
244
|
+
cancelled: import_zod.z.number().int().nonnegative(),
|
|
245
|
+
ungraded: import_zod.z.number().int().nonnegative().optional()
|
|
246
|
+
}).strict(),
|
|
247
|
+
cases: import_zod.z.array(EvalCaseResultSchema),
|
|
248
|
+
errors: import_zod.z.array(issue)
|
|
249
|
+
}).strict();
|
|
250
|
+
var EvalManifestSchema = import_zod.z.object({
|
|
251
|
+
schemaVersion: import_zod.z.literal(1),
|
|
252
|
+
studioJudging: import_zod.z.literal(true).optional(),
|
|
253
|
+
suites: import_zod.z.array(EvalSuiteSchema),
|
|
254
|
+
responders: import_zod.z.array(
|
|
255
|
+
import_zod.z.object({
|
|
256
|
+
name: id,
|
|
257
|
+
schemaId: id.optional(),
|
|
258
|
+
schemaVersion: id.optional()
|
|
259
|
+
}).strict()
|
|
260
|
+
),
|
|
261
|
+
references: import_zod.z.array(id),
|
|
262
|
+
paramsSchema: import_zod.z.record(import_zod.z.string(), import_zod.z.unknown()).optional(),
|
|
263
|
+
judge: judge.optional()
|
|
264
|
+
}).strict();
|
|
265
|
+
var EvalProgressSchema = import_zod.z.discriminatedUnion("type", [
|
|
266
|
+
import_zod.z.object({
|
|
267
|
+
type: import_zod.z.literal("run-started"),
|
|
268
|
+
caseIds: import_zod.z.array(id).min(1),
|
|
269
|
+
repetitions: import_zod.z.number().int().positive(),
|
|
270
|
+
concurrency: import_zod.z.number().int().positive()
|
|
271
|
+
}).strict(),
|
|
272
|
+
import_zod.z.object({
|
|
273
|
+
type: import_zod.z.literal("case-progress"),
|
|
274
|
+
caseId: id,
|
|
275
|
+
repetition: import_zod.z.number().int().positive(),
|
|
276
|
+
phase: issue.shape.phase,
|
|
277
|
+
stepIndex: import_zod.z.number().int().nonnegative().optional()
|
|
278
|
+
}).strict(),
|
|
279
|
+
import_zod.z.object({
|
|
280
|
+
type: import_zod.z.literal("case-started"),
|
|
281
|
+
caseId: id,
|
|
282
|
+
repetition: import_zod.z.number().int().positive(),
|
|
283
|
+
sessionId: import_zod.z.string()
|
|
284
|
+
}).strict(),
|
|
285
|
+
import_zod.z.object({
|
|
286
|
+
type: import_zod.z.literal("step-completed"),
|
|
287
|
+
caseId: id,
|
|
288
|
+
repetition: import_zod.z.number().int().positive(),
|
|
289
|
+
step: EvalStepResultSchema
|
|
290
|
+
}).strict(),
|
|
291
|
+
import_zod.z.object({ type: import_zod.z.literal("case-completed"), result: EvalCaseResultSchema }).strict()
|
|
292
|
+
]);
|
|
293
|
+
var EvalRemoteRunRequestSchema = import_zod.z.object({
|
|
294
|
+
suiteId: id,
|
|
295
|
+
suiteRevision: import_zod.z.string().regex(/^[a-f0-9]{64}$/),
|
|
296
|
+
grading: import_zod.z.enum(["app", "studio"]).optional(),
|
|
297
|
+
judge: EvalJudgeIdentitySchema.optional(),
|
|
298
|
+
caseIds: import_zod.z.array(id).min(1).max(100).optional(),
|
|
299
|
+
repetitions: import_zod.z.number().int().min(1).max(20).default(1),
|
|
300
|
+
concurrency: import_zod.z.number().int().min(1).max(4).default(1)
|
|
301
|
+
}).strict();
|
|
302
|
+
var StudioEvalStartRequestSchema = EvalRemoteRunRequestSchema.omit({
|
|
303
|
+
grading: true,
|
|
304
|
+
judge: true
|
|
305
|
+
}).extend({
|
|
306
|
+
targetId: import_zod.z.string().min(1).max(128),
|
|
307
|
+
judge: import_zod.z.enum(["studio", "app"]).default("studio")
|
|
308
|
+
});
|
|
309
|
+
var EvalWireEventSchema = import_zod.z.discriminatedUnion("type", [
|
|
310
|
+
import_zod.z.object({ type: import_zod.z.literal("progress"), event: EvalProgressSchema }).strict(),
|
|
311
|
+
import_zod.z.object({ type: import_zod.z.literal("result"), result: EvalRunResultSchema }).strict(),
|
|
312
|
+
import_zod.z.object({ type: import_zod.z.literal("error"), message: import_zod.z.string() }).strict()
|
|
313
|
+
]);
|
|
314
|
+
var EvalConfigurationError = class extends Error {
|
|
315
|
+
name = "EvalConfigurationError";
|
|
316
|
+
};
|
|
317
|
+
function parseEvalSuite(value) {
|
|
318
|
+
const result = EvalSuiteSchema.safeParse(value);
|
|
319
|
+
if (!result.success)
|
|
320
|
+
throw new EvalConfigurationError(
|
|
321
|
+
"Invalid suite. Expected serializable cases, conversation steps, and expectations."
|
|
322
|
+
);
|
|
323
|
+
const ids = result.data.cases.map((item) => item.id);
|
|
324
|
+
if (new Set(ids).size !== ids.length)
|
|
325
|
+
throw new EvalConfigurationError("Case IDs must be unique within a suite.");
|
|
326
|
+
for (const item of result.data.cases) {
|
|
327
|
+
let waiting = false;
|
|
328
|
+
for (const step of item.steps) {
|
|
329
|
+
if ("resume" in step !== waiting)
|
|
330
|
+
throw new EvalConfigurationError(
|
|
331
|
+
"A resume step must follow an expected interrupt; a message must follow an answer."
|
|
332
|
+
);
|
|
333
|
+
waiting = step.expect.type === "interrupt";
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
return result.data;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
// src/evals/define-suite.ts
|
|
340
|
+
function defineSuite(suite) {
|
|
341
|
+
return suite;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// src/evals/studio-contracts.ts
|
|
345
|
+
var import_zod2 = require("zod");
|
|
346
|
+
var StudioEvalTargetsResponseSchema = import_zod2.z.object({
|
|
347
|
+
canRun: import_zod2.z.boolean(),
|
|
348
|
+
studioJudge: EvalJudgeIdentitySchema.nullable().optional(),
|
|
349
|
+
targets: import_zod2.z.array(
|
|
350
|
+
import_zod2.z.object({
|
|
351
|
+
id: import_zod2.z.string(),
|
|
352
|
+
name: import_zod2.z.string(),
|
|
353
|
+
environment: import_zod2.z.string(),
|
|
354
|
+
error: import_zod2.z.string().nullable(),
|
|
355
|
+
manifest: EvalManifestSchema.nullable(),
|
|
356
|
+
revisions: import_zod2.z.record(import_zod2.z.string(), import_zod2.z.string())
|
|
357
|
+
})
|
|
358
|
+
)
|
|
359
|
+
});
|
|
360
|
+
var EvalCostAmountSchema = import_zod2.z.object({
|
|
361
|
+
amount: import_zod2.z.number().finite().nonnegative().nullable(),
|
|
362
|
+
currency: import_zod2.z.string().nullable(),
|
|
363
|
+
status: import_zod2.z.enum(["complete", "partial", "unavailable"]),
|
|
364
|
+
calls: import_zod2.z.number().int().nonnegative(),
|
|
365
|
+
unpricedCalls: import_zod2.z.number().int().nonnegative(),
|
|
366
|
+
estimated: import_zod2.z.boolean()
|
|
367
|
+
});
|
|
368
|
+
var EvalCostsSchema = import_zod2.z.object({
|
|
369
|
+
workflow: EvalCostAmountSchema,
|
|
370
|
+
judge: EvalCostAmountSchema,
|
|
371
|
+
total: EvalCostAmountSchema
|
|
372
|
+
});
|
|
373
|
+
var StudioEvalRunSummarySchema = import_zod2.z.object({
|
|
374
|
+
id: import_zod2.z.uuid(),
|
|
375
|
+
targetId: import_zod2.z.string(),
|
|
376
|
+
targetName: import_zod2.z.string(),
|
|
377
|
+
environment: import_zod2.z.string(),
|
|
378
|
+
suiteId: import_zod2.z.string(),
|
|
379
|
+
suiteRevision: import_zod2.z.string(),
|
|
380
|
+
status: import_zod2.z.enum([
|
|
381
|
+
"queued",
|
|
382
|
+
"running",
|
|
383
|
+
"passed",
|
|
384
|
+
"failed",
|
|
385
|
+
"error",
|
|
386
|
+
"cancelled"
|
|
387
|
+
]),
|
|
388
|
+
createdAt: import_zod2.z.string(),
|
|
389
|
+
startedAt: import_zod2.z.string().nullable(),
|
|
390
|
+
endedAt: import_zod2.z.string().nullable(),
|
|
391
|
+
error: import_zod2.z.string().nullable(),
|
|
392
|
+
cancelRequestedAt: import_zod2.z.string().nullable(),
|
|
393
|
+
costs: EvalCostsSchema.optional(),
|
|
394
|
+
counts: import_zod2.z.object({
|
|
395
|
+
passed: import_zod2.z.number(),
|
|
396
|
+
failed: import_zod2.z.number(),
|
|
397
|
+
error: import_zod2.z.number(),
|
|
398
|
+
cancelled: import_zod2.z.number()
|
|
399
|
+
}).nullable().optional(),
|
|
400
|
+
suiteName: import_zod2.z.string().nullable().optional()
|
|
401
|
+
});
|
|
402
|
+
var StudioEvalHistorySchema = import_zod2.z.object({
|
|
403
|
+
runs: import_zod2.z.array(StudioEvalRunSummarySchema)
|
|
404
|
+
});
|
|
405
|
+
var StudioEvalDetailSchema = import_zod2.z.object({
|
|
406
|
+
run: StudioEvalRunSummarySchema.extend({
|
|
407
|
+
caseCosts: import_zod2.z.record(import_zod2.z.string(), EvalCostsSchema).optional(),
|
|
408
|
+
request: import_zod2.z.object({
|
|
409
|
+
caseIds: import_zod2.z.array(import_zod2.z.string()).optional(),
|
|
410
|
+
grading: import_zod2.z.enum(["app", "studio"]).optional(),
|
|
411
|
+
judge: EvalJudgeIdentitySchema.optional(),
|
|
412
|
+
repetitions: import_zod2.z.number().int().positive(),
|
|
413
|
+
concurrency: import_zod2.z.number().optional()
|
|
414
|
+
}).optional(),
|
|
415
|
+
suite: EvalSuiteSchema,
|
|
416
|
+
result: EvalRunResultSchema.nullable(),
|
|
417
|
+
events: import_zod2.z.array(import_zod2.z.object({ id: import_zod2.z.number(), event: EvalProgressSchema }))
|
|
418
|
+
})
|
|
419
|
+
});
|
|
420
|
+
|
|
421
|
+
// src/evals/studio-judge.ts
|
|
422
|
+
async function readJson(response, signal) {
|
|
423
|
+
if (!response.ok || !response.body) {
|
|
424
|
+
await response.body?.cancel().catch(() => {
|
|
425
|
+
});
|
|
426
|
+
throw new Error("Studio judge is unavailable or rejected the request.");
|
|
427
|
+
}
|
|
428
|
+
const reader = response.body.getReader();
|
|
429
|
+
const abort = () => {
|
|
430
|
+
void reader.cancel().catch(() => {
|
|
431
|
+
});
|
|
432
|
+
};
|
|
433
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
434
|
+
const decoder = new TextDecoder();
|
|
435
|
+
let text = "";
|
|
436
|
+
let bytes = 0;
|
|
437
|
+
try {
|
|
438
|
+
for (; ; ) {
|
|
439
|
+
signal.throwIfAborted();
|
|
440
|
+
const part = await reader.read();
|
|
441
|
+
if (part.done) break;
|
|
442
|
+
bytes += part.value.byteLength;
|
|
443
|
+
if (bytes > 1e6)
|
|
444
|
+
throw new Error("Studio judge response is too large.");
|
|
445
|
+
text += decoder.decode(part.value, { stream: true });
|
|
446
|
+
}
|
|
447
|
+
signal.throwIfAborted();
|
|
448
|
+
return JSON.parse(text + decoder.decode());
|
|
449
|
+
} finally {
|
|
450
|
+
signal.removeEventListener("abort", abort);
|
|
451
|
+
await reader.cancel().catch(() => {
|
|
452
|
+
});
|
|
453
|
+
reader.releaseLock();
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
async function createStudioEvalJudge(options) {
|
|
457
|
+
const base = new URL(options.url);
|
|
458
|
+
if (base.username || base.password || base.search || base.hash || base.pathname !== "/" && base.pathname !== "" || base.protocol !== "https:" && !(options.allowInsecureHttp && base.protocol === "http:") || !options.apiKey.trim() || !options.environment.trim())
|
|
459
|
+
throw new EvalConfigurationError(
|
|
460
|
+
"Studio judging requires an API origin, key and environment, with HTTPS or explicit local HTTP opt-in."
|
|
461
|
+
);
|
|
462
|
+
const endpoint = new URL("/v1/studio/evals/judge", base);
|
|
463
|
+
const discovery = new URL(endpoint);
|
|
464
|
+
discovery.searchParams.set("environment", options.environment);
|
|
465
|
+
const signal = options.signal ? AbortSignal.any([options.signal, AbortSignal.timeout(15e3)]) : AbortSignal.timeout(15e3);
|
|
466
|
+
const identity = EvalJudgeIdentitySchema.parse(
|
|
467
|
+
await readJson(
|
|
468
|
+
await fetch(discovery, {
|
|
469
|
+
headers: { authorization: `Bearer ${options.apiKey}` },
|
|
470
|
+
redirect: "error",
|
|
471
|
+
signal
|
|
472
|
+
}),
|
|
473
|
+
signal
|
|
474
|
+
)
|
|
475
|
+
);
|
|
476
|
+
if (identity.location !== "studio")
|
|
477
|
+
throw new EvalConfigurationError(
|
|
478
|
+
"Endpoint did not provide a Studio judge."
|
|
479
|
+
);
|
|
480
|
+
return {
|
|
481
|
+
...identity,
|
|
482
|
+
location: "studio",
|
|
483
|
+
async grade(input) {
|
|
484
|
+
const request = StudioEvalJudgeRequestSchema.parse({
|
|
485
|
+
environment: options.environment,
|
|
486
|
+
judge: identity,
|
|
487
|
+
criterion: input.criterion,
|
|
488
|
+
input: input.input,
|
|
489
|
+
observation: input.observation,
|
|
490
|
+
reference: input.reference,
|
|
491
|
+
conversation: input.conversation
|
|
492
|
+
});
|
|
493
|
+
const signal2 = AbortSignal.any([
|
|
494
|
+
input.signal,
|
|
495
|
+
AbortSignal.timeout(6e4)
|
|
496
|
+
]);
|
|
497
|
+
signal2.throwIfAborted();
|
|
498
|
+
const response = StudioEvalJudgeResponseSchema.parse(
|
|
499
|
+
await readJson(
|
|
500
|
+
await fetch(endpoint, {
|
|
501
|
+
method: "POST",
|
|
502
|
+
headers: {
|
|
503
|
+
authorization: `Bearer ${options.apiKey}`,
|
|
504
|
+
"content-type": "application/json"
|
|
505
|
+
},
|
|
506
|
+
body: JSON.stringify(request),
|
|
507
|
+
redirect: "error",
|
|
508
|
+
signal: signal2
|
|
509
|
+
}),
|
|
510
|
+
signal2
|
|
511
|
+
)
|
|
512
|
+
);
|
|
513
|
+
if (response.judge.id !== identity.id || response.judge.version !== identity.version || response.judge.location !== "studio")
|
|
514
|
+
throw new Error(
|
|
515
|
+
"Studio judge identity changed. Refresh before running."
|
|
516
|
+
);
|
|
517
|
+
for (const usage of response.usage ?? []) input.onUsage?.(usage);
|
|
518
|
+
return response.verdict;
|
|
519
|
+
}
|
|
520
|
+
};
|
|
521
|
+
}
|
|
522
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
523
|
+
0 && (module.exports = {
|
|
524
|
+
EvalCaseResultSchema,
|
|
525
|
+
EvalConfigurationError,
|
|
526
|
+
EvalCostAmountSchema,
|
|
527
|
+
EvalCostsSchema,
|
|
528
|
+
EvalJudgeIdentitySchema,
|
|
529
|
+
EvalJudgeUsageSchema,
|
|
530
|
+
EvalManifestSchema,
|
|
531
|
+
EvalObservationSchema,
|
|
532
|
+
EvalProgressSchema,
|
|
533
|
+
EvalRemoteRunRequestSchema,
|
|
534
|
+
EvalResumeResponseSchema,
|
|
535
|
+
EvalRunResultSchema,
|
|
536
|
+
EvalStepResultSchema,
|
|
537
|
+
EvalSuiteSchema,
|
|
538
|
+
EvalVerdictSchema,
|
|
539
|
+
EvalWireEventSchema,
|
|
540
|
+
StudioEvalDetailSchema,
|
|
541
|
+
StudioEvalHistorySchema,
|
|
542
|
+
StudioEvalJudgeRequestSchema,
|
|
543
|
+
StudioEvalJudgeResponseSchema,
|
|
544
|
+
StudioEvalRunSummarySchema,
|
|
545
|
+
StudioEvalStartRequestSchema,
|
|
546
|
+
StudioEvalTargetsResponseSchema,
|
|
547
|
+
createStudioEvalJudge,
|
|
548
|
+
defineSuite,
|
|
549
|
+
parseEvalSuite
|
|
550
|
+
});
|
|
551
|
+
//# sourceMappingURL=evals.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/evals.ts","../src/evals/contracts.ts","../src/evals/define-suite.ts","../src/evals/studio-contracts.ts","../src/evals/studio-judge.ts"],"sourcesContent":["export * from \"./evals/contracts\";\nexport { defineSuite } from \"./evals/define-suite\";\nexport * from \"./evals/studio-contracts\";\nexport { createStudioEvalJudge } from \"./evals/studio-judge\";\nexport type * from \"./evals/types\";\n","import { z } from \"zod\";\nimport type { EvalSuite } from \"./types\";\n\nconst id = z.string().trim().min(1).max(256);\nconst handler = z.object({ using: id, params: z.json().optional() }).strict();\nexport const EvalResumeResponseSchema = z.discriminatedUnion(\"type\", [\n z.object({ type: z.literal(\"text\"), text: z.string() }).strict(),\n z.object({ type: z.literal(\"select\"), ids: z.array(z.string()) }).strict(),\n z.object({ type: z.literal(\"value\"), value: z.json() }).strict(),\n z.object({ type: z.literal(\"cancel\") }).strict(),\n]);\nconst expectation = z\n .object({\n type: z.enum([\"answer\", \"interrupt\"]),\n schemaId: id.optional(),\n schemaVersion: id.optional(),\n outputs: z\n .array(z.object({ schemaId: id, schemaVersion: id.optional() }).strict())\n .optional(),\n criteria: z\n .array(\n z.union([\n z.string().trim().min(1),\n z.object({ id, text: z.string().trim().min(1) }).strict(),\n ]),\n )\n .optional(),\n reference: z.json().optional(),\n })\n .strict()\n .superRefine((value, ctx) => {\n if (value.type === \"answer\" && (value.schemaId || value.schemaVersion))\n ctx.addIssue({\n code: \"custom\",\n message: \"Interrupt schemas require an interrupt expectation.\",\n });\n const ids =\n value.criteria?.map((item, index) =>\n typeof item === \"string\" ? String(index) : item.id,\n ) ?? [];\n if (new Set(ids).size !== ids.length)\n ctx.addIssue({\n code: \"custom\",\n message: \"Criterion IDs must be unique within a step.\",\n });\n });\nexport const EvalSuiteSchema = z\n .object({\n id,\n name: z.string().optional(),\n cases: z\n .array(\n z\n .object({\n id,\n name: z.string().optional(),\n params: z.json().optional(),\n workflowId: id.optional(),\n steps: z\n .array(\n z.union([\n z\n .object({ message: z.string().min(1), expect: expectation })\n .strict(),\n z\n .object({\n resume: z.union([EvalResumeResponseSchema, handler]),\n expect: expectation,\n })\n .strict(),\n ]),\n )\n .min(1),\n })\n .strict(),\n )\n .min(1),\n })\n .strict();\nexport const EvalVerdictSchema = z\n .object({\n passed: z.boolean(),\n reason: z.string().trim().min(1),\n evidence: z.array(z.string().min(1)),\n })\n .strict();\nexport const EvalObservationSchema = z\n .object({\n type: z.enum([\"answer\", \"interrupt\", \"error\", \"cancelled\"]),\n text: z.string(),\n structured: z.array(z.json()),\n events: z.array(z.json()).optional(),\n runId: z.string().optional(),\n checkpointId: z.string().optional(),\n interrupt: z\n .object({\n requestId: z.string(),\n kind: z.enum([\"text\", \"choice\", \"multi-choice\", \"custom\"]),\n question: z.string().optional(),\n schemaId: z.string().optional(),\n schemaVersion: z.string().optional(),\n options: z.array(\n z\n .object({\n id: z.string(),\n label: z.string(),\n description: z.string().optional(),\n })\n .strict(),\n ),\n request: z.json().optional(),\n })\n .strict()\n .optional(),\n })\n .strict()\n .superRefine((value, ctx) => {\n if ((value.type === \"interrupt\") !== Boolean(value.interrupt))\n ctx.addIssue({\n code: \"custom\",\n message: \"Interrupt observations require exactly one interrupt.\",\n });\n });\n\nconst status = z.enum([\"passed\", \"failed\", \"error\", \"cancelled\", \"ungraded\"]);\nconst issue = z\n .object({\n phase: z.enum([\n \"params\",\n \"setup\",\n \"execute\",\n \"responder\",\n \"reference\",\n \"grading\",\n \"cleanup\",\n \"reporting\",\n ]),\n code: z.string(),\n message: z.string(),\n })\n .strict();\n/** Billing evidence only: no prompts, provider raw responses, or credentials. */\nexport const EvalJudgeUsageSchema = z\n .object({\n provider: z.string().min(1),\n model: z.string().min(1),\n occurredAt: z.iso.datetime(),\n usage: z\n .object({\n input: z.number().finite().nonnegative().optional(),\n output: z.number().finite().nonnegative().optional(),\n total: z.number().finite().nonnegative().optional(),\n reasoning: z.number().finite().nonnegative().optional(),\n cacheRead: z.number().finite().nonnegative().optional(),\n cacheWrite: z.number().finite().nonnegative().optional(),\n cacheWrite1h: z.number().finite().nonnegative().optional(),\n outputIncludesReasoning: z.boolean().optional(),\n inputIncludesCacheRead: z.boolean().optional(),\n inputIncludesCacheWrite: z.boolean().optional(),\n })\n .strict()\n .optional(),\n pricingContext: z\n .object({\n serviceTier: z.string().min(1).max(100).optional(),\n inferenceGeo: z.string().min(1).max(100).optional(),\n speed: z.string().min(1).max(100).optional(),\n })\n .strict()\n .optional(),\n pricing: z\n .object({\n source: z.literal(\"provider\"),\n currency: z.literal(\"USD\"),\n actualCostMicros: z.number().int().nonnegative().safe(),\n })\n .strict()\n .optional(),\n })\n .strict();\nexport const EvalStepResultSchema = z\n .object({\n index: z.number().int().nonnegative(),\n input: z.union([\n z.object({ message: z.string() }).strict(),\n z.object({ resume: EvalResumeResponseSchema }).strict(),\n ]),\n expectation,\n observation: EvalObservationSchema,\n reference: z.json().optional(),\n status,\n reason: z.string().optional(),\n judgeCalls: z.number().int().nonnegative().optional(),\n judgeUsage: z.array(EvalJudgeUsageSchema).optional(),\n criteria: z.array(\n EvalVerdictSchema.extend({ id: z.string(), text: z.string() }),\n ),\n })\n .strict();\nexport const EvalCaseResultSchema = z\n .object({\n caseId: id,\n repetition: z.number().int().positive(),\n sessionId: z.string(),\n status,\n durationMs: z.number().nonnegative(),\n steps: z.array(EvalStepResultSchema),\n errors: z.array(issue),\n })\n .strict();\nexport const EvalJudgeIdentitySchema = z\n .object({\n id,\n version: id,\n location: z.enum([\"app\", \"studio\"]).optional(),\n })\n .strict();\nconst judge = EvalJudgeIdentitySchema;\nexport const StudioEvalJudgeRequestSchema = z\n .object({\n environment: id,\n judge: EvalJudgeIdentitySchema,\n criterion: z\n .object({ id, text: z.string().trim().min(1).max(16_384) })\n .strict(),\n input: EvalStepResultSchema.shape.input,\n observation: EvalObservationSchema,\n reference: z.json().optional(),\n conversation: z.array(EvalStepResultSchema).max(100),\n })\n .strict();\nexport const StudioEvalJudgeResponseSchema = z\n .object({\n judge: EvalJudgeIdentitySchema,\n verdict: EvalVerdictSchema,\n usage: z.array(EvalJudgeUsageSchema).optional(),\n })\n .strict();\nexport const EvalRunResultSchema = z\n .object({\n id: z.uuid(),\n suiteId: id,\n suiteRevision: z.string().regex(/^[a-f0-9]{64}$/),\n suite: EvalSuiteSchema,\n judge: judge.optional(),\n startedAt: z.iso.datetime(),\n durationMs: z.number().nonnegative(),\n status,\n counts: z\n .object({\n passed: z.number().int().nonnegative(),\n failed: z.number().int().nonnegative(),\n error: z.number().int().nonnegative(),\n cancelled: z.number().int().nonnegative(),\n ungraded: z.number().int().nonnegative().optional(),\n })\n .strict(),\n cases: z.array(EvalCaseResultSchema),\n errors: z.array(issue),\n })\n .strict();\nexport const EvalManifestSchema = z\n .object({\n schemaVersion: z.literal(1),\n studioJudging: z.literal(true).optional(),\n suites: z.array(EvalSuiteSchema),\n responders: z.array(\n z\n .object({\n name: id,\n schemaId: id.optional(),\n schemaVersion: id.optional(),\n })\n .strict(),\n ),\n references: z.array(id),\n paramsSchema: z.record(z.string(), z.unknown()).optional(),\n judge: judge.optional(),\n })\n .strict();\nexport const EvalProgressSchema = z.discriminatedUnion(\"type\", [\n z\n .object({\n type: z.literal(\"run-started\"),\n caseIds: z.array(id).min(1),\n repetitions: z.number().int().positive(),\n concurrency: z.number().int().positive(),\n })\n .strict(),\n z\n .object({\n type: z.literal(\"case-progress\"),\n caseId: id,\n repetition: z.number().int().positive(),\n phase: issue.shape.phase,\n stepIndex: z.number().int().nonnegative().optional(),\n })\n .strict(),\n z\n .object({\n type: z.literal(\"case-started\"),\n caseId: id,\n repetition: z.number().int().positive(),\n sessionId: z.string(),\n })\n .strict(),\n z\n .object({\n type: z.literal(\"step-completed\"),\n caseId: id,\n repetition: z.number().int().positive(),\n step: EvalStepResultSchema,\n })\n .strict(),\n z\n .object({ type: z.literal(\"case-completed\"), result: EvalCaseResultSchema })\n .strict(),\n]);\nexport const EvalRemoteRunRequestSchema = z\n .object({\n suiteId: id,\n suiteRevision: z.string().regex(/^[a-f0-9]{64}$/),\n grading: z.enum([\"app\", \"studio\"]).optional(),\n judge: EvalJudgeIdentitySchema.optional(),\n caseIds: z.array(id).min(1).max(100).optional(),\n repetitions: z.number().int().min(1).max(20).default(1),\n concurrency: z.number().int().min(1).max(4).default(1),\n })\n .strict();\nexport const StudioEvalStartRequestSchema = EvalRemoteRunRequestSchema.omit({\n grading: true,\n judge: true,\n}).extend({\n targetId: z.string().min(1).max(128),\n judge: z.enum([\"studio\", \"app\"]).default(\"studio\"),\n});\nexport const EvalWireEventSchema = z.discriminatedUnion(\"type\", [\n z.object({ type: z.literal(\"progress\"), event: EvalProgressSchema }).strict(),\n z.object({ type: z.literal(\"result\"), result: EvalRunResultSchema }).strict(),\n z.object({ type: z.literal(\"error\"), message: z.string() }).strict(),\n]);\nexport type EvalManifest = z.infer<typeof EvalManifestSchema>;\nexport type EvalRemoteRunRequest = z.infer<typeof EvalRemoteRunRequestSchema>;\nexport type EvalWireEvent = z.infer<typeof EvalWireEventSchema>;\n\nexport class EvalConfigurationError extends Error {\n override name = \"EvalConfigurationError\";\n}\n\nexport function parseEvalSuite(value: unknown): EvalSuite {\n const result = EvalSuiteSchema.safeParse(value);\n if (!result.success)\n throw new EvalConfigurationError(\n \"Invalid suite. Expected serializable cases, conversation steps, and expectations.\",\n );\n const ids = result.data.cases.map((item) => item.id);\n if (new Set(ids).size !== ids.length)\n throw new EvalConfigurationError(\"Case IDs must be unique within a suite.\");\n for (const item of result.data.cases) {\n let waiting = false;\n for (const step of item.steps) {\n if (\"resume\" in step !== waiting)\n throw new EvalConfigurationError(\n \"A resume step must follow an expected interrupt; a message must follow an answer.\",\n );\n waiting = step.expect.type === \"interrupt\";\n }\n }\n return result.data as EvalSuite;\n}\n","import type { EvalCase, EvalJson, EvalSuite } from \"./types\";\n\ntype DefinedSuite<Params> = Omit<EvalSuite<Params>, \"cases\"> & {\n cases: readonly (Omit<EvalCase<Params>, \"params\"> &\n (undefined extends Params\n ? { params?: Exclude<Params, undefined> }\n : { params: Params }))[];\n};\n\n/** Author a suite with typed case parameters. Runtime validation belongs to createEvals. */\nexport function defineSuite<Params = EvalJson | undefined>(\n suite: DefinedSuite<Params>,\n): DefinedSuite<Params> {\n return suite;\n}\n","import { z } from \"zod\";\nimport {\n EvalJudgeIdentitySchema,\n EvalManifestSchema,\n EvalProgressSchema,\n EvalRunResultSchema,\n EvalSuiteSchema,\n} from \"./contracts\";\nexport const StudioEvalTargetsResponseSchema = z.object({\n canRun: z.boolean(),\n studioJudge: EvalJudgeIdentitySchema.nullable().optional(),\n targets: z.array(\n z.object({\n id: z.string(),\n name: z.string(),\n environment: z.string(),\n error: z.string().nullable(),\n manifest: EvalManifestSchema.nullable(),\n revisions: z.record(z.string(), z.string()),\n }),\n ),\n});\nexport const EvalCostAmountSchema = z.object({\n amount: z.number().finite().nonnegative().nullable(),\n currency: z.string().nullable(),\n status: z.enum([\"complete\", \"partial\", \"unavailable\"]),\n calls: z.number().int().nonnegative(),\n unpricedCalls: z.number().int().nonnegative(),\n estimated: z.boolean(),\n});\nexport const EvalCostsSchema = z.object({\n workflow: EvalCostAmountSchema,\n judge: EvalCostAmountSchema,\n total: EvalCostAmountSchema,\n});\nexport type EvalCosts = z.infer<typeof EvalCostsSchema>;\nexport type EvalCostAmount = z.infer<typeof EvalCostAmountSchema>;\nexport const StudioEvalRunSummarySchema = z.object({\n id: z.uuid(),\n targetId: z.string(),\n targetName: z.string(),\n environment: z.string(),\n suiteId: z.string(),\n suiteRevision: z.string(),\n status: z.enum([\n \"queued\",\n \"running\",\n \"passed\",\n \"failed\",\n \"error\",\n \"cancelled\",\n ]),\n createdAt: z.string(),\n startedAt: z.string().nullable(),\n endedAt: z.string().nullable(),\n error: z.string().nullable(),\n cancelRequestedAt: z.string().nullable(),\n costs: EvalCostsSchema.optional(),\n counts: z\n .object({\n passed: z.number(),\n failed: z.number(),\n error: z.number(),\n cancelled: z.number(),\n })\n .nullable()\n .optional(),\n suiteName: z.string().nullable().optional(),\n});\nexport const StudioEvalHistorySchema = z.object({\n runs: z.array(StudioEvalRunSummarySchema),\n});\nexport const StudioEvalDetailSchema = z.object({\n run: StudioEvalRunSummarySchema.extend({\n caseCosts: z.record(z.string(), EvalCostsSchema).optional(),\n request: z\n .object({\n caseIds: z.array(z.string()).optional(),\n grading: z.enum([\"app\", \"studio\"]).optional(),\n judge: EvalJudgeIdentitySchema.optional(),\n repetitions: z.number().int().positive(),\n concurrency: z.number().optional(),\n })\n .optional(),\n suite: EvalSuiteSchema,\n result: EvalRunResultSchema.nullable(),\n events: z.array(z.object({ id: z.number(), event: EvalProgressSchema })),\n }),\n});\nexport type StudioEvalTargets = z.infer<typeof StudioEvalTargetsResponseSchema>;\nexport type StudioEvalHistory = z.infer<typeof StudioEvalHistorySchema>;\nexport type StudioEvalDetail = z.infer<typeof StudioEvalDetailSchema>[\"run\"];\n\nexport type StudioEvalRunSummary = z.infer<typeof StudioEvalRunSummarySchema>;\n","import {\n EvalConfigurationError,\n EvalJudgeIdentitySchema,\n StudioEvalJudgeRequestSchema,\n StudioEvalJudgeResponseSchema,\n} from \"./contracts\";\nimport type { EvalJudge, StudioEvalJudgeOptions } from \"./types\";\n\nasync function readJson(\n response: Response,\n signal: AbortSignal,\n): Promise<unknown> {\n if (!response.ok || !response.body) {\n await response.body?.cancel().catch(() => {});\n throw new Error(\"Studio judge is unavailable or rejected the request.\");\n }\n const reader = response.body.getReader();\n const abort = () => {\n void reader.cancel().catch(() => {});\n };\n signal.addEventListener(\"abort\", abort, { once: true });\n const decoder = new TextDecoder();\n let text = \"\";\n let bytes = 0;\n try {\n for (;;) {\n signal.throwIfAborted();\n const part = await reader.read();\n if (part.done) break;\n bytes += part.value.byteLength;\n if (bytes > 1_000_000)\n throw new Error(\"Studio judge response is too large.\");\n text += decoder.decode(part.value, { stream: true });\n }\n signal.throwIfAborted();\n return JSON.parse(text + decoder.decode());\n } finally {\n signal.removeEventListener(\"abort\", abort);\n await reader.cancel().catch(() => {});\n reader.releaseLock();\n }\n}\n\n/** Discover and pin the Studio backend's judge; grading never runs in the browser. */\nexport async function createStudioEvalJudge(\n options: StudioEvalJudgeOptions,\n): Promise<EvalJudge> {\n const base = new URL(options.url);\n if (\n base.username ||\n base.password ||\n base.search ||\n base.hash ||\n (base.pathname !== \"/\" && base.pathname !== \"\") ||\n (base.protocol !== \"https:\" &&\n !(options.allowInsecureHttp && base.protocol === \"http:\")) ||\n !options.apiKey.trim() ||\n !options.environment.trim()\n )\n throw new EvalConfigurationError(\n \"Studio judging requires an API origin, key and environment, with HTTPS or explicit local HTTP opt-in.\",\n );\n const endpoint = new URL(\"/v1/studio/evals/judge\", base);\n const discovery = new URL(endpoint);\n discovery.searchParams.set(\"environment\", options.environment);\n const signal = options.signal\n ? AbortSignal.any([options.signal, AbortSignal.timeout(15_000)])\n : AbortSignal.timeout(15_000);\n const identity = EvalJudgeIdentitySchema.parse(\n await readJson(\n await fetch(discovery, {\n headers: { authorization: `Bearer ${options.apiKey}` },\n redirect: \"error\",\n signal,\n }),\n signal,\n ),\n );\n if (identity.location !== \"studio\")\n throw new EvalConfigurationError(\n \"Endpoint did not provide a Studio judge.\",\n );\n return {\n ...identity,\n location: \"studio\",\n async grade(input) {\n const request = StudioEvalJudgeRequestSchema.parse({\n environment: options.environment,\n judge: identity,\n criterion: input.criterion,\n input: input.input,\n observation: input.observation,\n reference: input.reference,\n conversation: input.conversation,\n });\n const signal = AbortSignal.any([\n input.signal,\n AbortSignal.timeout(60_000),\n ]);\n signal.throwIfAborted();\n const response = StudioEvalJudgeResponseSchema.parse(\n await readJson(\n await fetch(endpoint, {\n method: \"POST\",\n headers: {\n authorization: `Bearer ${options.apiKey}`,\n \"content-type\": \"application/json\",\n },\n body: JSON.stringify(request),\n redirect: \"error\",\n signal,\n }),\n signal,\n ),\n );\n if (\n response.judge.id !== identity.id ||\n response.judge.version !== identity.version ||\n response.judge.location !== \"studio\"\n )\n throw new Error(\n \"Studio judge identity changed. Refresh before running.\",\n );\n for (const usage of response.usage ?? []) input.onUsage?.(usage);\n return response.verdict;\n },\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,iBAAkB;AAGlB,IAAM,KAAK,aAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG;AAC3C,IAAM,UAAU,aAAE,OAAO,EAAE,OAAO,IAAI,QAAQ,aAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO;AACrE,IAAM,2BAA2B,aAAE,mBAAmB,QAAQ;AAAA,EACnE,aAAE,OAAO,EAAE,MAAM,aAAE,QAAQ,MAAM,GAAG,MAAM,aAAE,OAAO,EAAE,CAAC,EAAE,OAAO;AAAA,EAC/D,aAAE,OAAO,EAAE,MAAM,aAAE,QAAQ,QAAQ,GAAG,KAAK,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,CAAC,EAAE,OAAO;AAAA,EACzE,aAAE,OAAO,EAAE,MAAM,aAAE,QAAQ,OAAO,GAAG,OAAO,aAAE,KAAK,EAAE,CAAC,EAAE,OAAO;AAAA,EAC/D,aAAE,OAAO,EAAE,MAAM,aAAE,QAAQ,QAAQ,EAAE,CAAC,EAAE,OAAO;AACjD,CAAC;AACD,IAAM,cAAc,aACjB,OAAO;AAAA,EACN,MAAM,aAAE,KAAK,CAAC,UAAU,WAAW,CAAC;AAAA,EACpC,UAAU,GAAG,SAAS;AAAA,EACtB,eAAe,GAAG,SAAS;AAAA,EAC3B,SAAS,aACN,MAAM,aAAE,OAAO,EAAE,UAAU,IAAI,eAAe,GAAG,SAAS,EAAE,CAAC,EAAE,OAAO,CAAC,EACvE,SAAS;AAAA,EACZ,UAAU,aACP;AAAA,IACC,aAAE,MAAM;AAAA,MACN,aAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC;AAAA,MACvB,aAAE,OAAO,EAAE,IAAI,MAAM,aAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,EAAE,OAAO;AAAA,IAC1D,CAAC;AAAA,EACH,EACC,SAAS;AAAA,EACZ,WAAW,aAAE,KAAK,EAAE,SAAS;AAC/B,CAAC,EACA,OAAO,EACP,YAAY,CAAC,OAAO,QAAQ;AAC3B,MAAI,MAAM,SAAS,aAAa,MAAM,YAAY,MAAM;AACtD,QAAI,SAAS;AAAA,MACX,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AACH,QAAM,MACJ,MAAM,UAAU;AAAA,IAAI,CAAC,MAAM,UACzB,OAAO,SAAS,WAAW,OAAO,KAAK,IAAI,KAAK;AAAA,EAClD,KAAK,CAAC;AACR,MAAI,IAAI,IAAI,GAAG,EAAE,SAAS,IAAI;AAC5B,QAAI,SAAS;AAAA,MACX,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AACL,CAAC;AACI,IAAM,kBAAkB,aAC5B,OAAO;AAAA,EACN;AAAA,EACA,MAAM,aAAE,OAAO,EAAE,SAAS;AAAA,EAC1B,OAAO,aACJ;AAAA,IACC,aACG,OAAO;AAAA,MACN;AAAA,MACA,MAAM,aAAE,OAAO,EAAE,SAAS;AAAA,MAC1B,QAAQ,aAAE,KAAK,EAAE,SAAS;AAAA,MAC1B,YAAY,GAAG,SAAS;AAAA,MACxB,OAAO,aACJ;AAAA,QACC,aAAE,MAAM;AAAA,UACN,aACG,OAAO,EAAE,SAAS,aAAE,OAAO,EAAE,IAAI,CAAC,GAAG,QAAQ,YAAY,CAAC,EAC1D,OAAO;AAAA,UACV,aACG,OAAO;AAAA,YACN,QAAQ,aAAE,MAAM,CAAC,0BAA0B,OAAO,CAAC;AAAA,YACnD,QAAQ;AAAA,UACV,CAAC,EACA,OAAO;AAAA,QACZ,CAAC;AAAA,MACH,EACC,IAAI,CAAC;AAAA,IACV,CAAC,EACA,OAAO;AAAA,EACZ,EACC,IAAI,CAAC;AACV,CAAC,EACA,OAAO;AACH,IAAM,oBAAoB,aAC9B,OAAO;AAAA,EACN,QAAQ,aAAE,QAAQ;AAAA,EAClB,QAAQ,aAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC;AAAA,EAC/B,UAAU,aAAE,MAAM,aAAE,OAAO,EAAE,IAAI,CAAC,CAAC;AACrC,CAAC,EACA,OAAO;AACH,IAAM,wBAAwB,aAClC,OAAO;AAAA,EACN,MAAM,aAAE,KAAK,CAAC,UAAU,aAAa,SAAS,WAAW,CAAC;AAAA,EAC1D,MAAM,aAAE,OAAO;AAAA,EACf,YAAY,aAAE,MAAM,aAAE,KAAK,CAAC;AAAA,EAC5B,QAAQ,aAAE,MAAM,aAAE,KAAK,CAAC,EAAE,SAAS;AAAA,EACnC,OAAO,aAAE,OAAO,EAAE,SAAS;AAAA,EAC3B,cAAc,aAAE,OAAO,EAAE,SAAS;AAAA,EAClC,WAAW,aACR,OAAO;AAAA,IACN,WAAW,aAAE,OAAO;AAAA,IACpB,MAAM,aAAE,KAAK,CAAC,QAAQ,UAAU,gBAAgB,QAAQ,CAAC;AAAA,IACzD,UAAU,aAAE,OAAO,EAAE,SAAS;AAAA,IAC9B,UAAU,aAAE,OAAO,EAAE,SAAS;AAAA,IAC9B,eAAe,aAAE,OAAO,EAAE,SAAS;AAAA,IACnC,SAAS,aAAE;AAAA,MACT,aACG,OAAO;AAAA,QACN,IAAI,aAAE,OAAO;AAAA,QACb,OAAO,aAAE,OAAO;AAAA,QAChB,aAAa,aAAE,OAAO,EAAE,SAAS;AAAA,MACnC,CAAC,EACA,OAAO;AAAA,IACZ;AAAA,IACA,SAAS,aAAE,KAAK,EAAE,SAAS;AAAA,EAC7B,CAAC,EACA,OAAO,EACP,SAAS;AACd,CAAC,EACA,OAAO,EACP,YAAY,CAAC,OAAO,QAAQ;AAC3B,MAAK,MAAM,SAAS,gBAAiB,QAAQ,MAAM,SAAS;AAC1D,QAAI,SAAS;AAAA,MACX,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AACL,CAAC;AAEH,IAAM,SAAS,aAAE,KAAK,CAAC,UAAU,UAAU,SAAS,aAAa,UAAU,CAAC;AAC5E,IAAM,QAAQ,aACX,OAAO;AAAA,EACN,OAAO,aAAE,KAAK;AAAA,IACZ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AAAA,EACD,MAAM,aAAE,OAAO;AAAA,EACf,SAAS,aAAE,OAAO;AACpB,CAAC,EACA,OAAO;AAEH,IAAM,uBAAuB,aACjC,OAAO;AAAA,EACN,UAAU,aAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EAC1B,OAAO,aAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EACvB,YAAY,aAAE,IAAI,SAAS;AAAA,EAC3B,OAAO,aACJ,OAAO;AAAA,IACN,OAAO,aAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS;AAAA,IAClD,QAAQ,aAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS;AAAA,IACnD,OAAO,aAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS;AAAA,IAClD,WAAW,aAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS;AAAA,IACtD,WAAW,aAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS;AAAA,IACtD,YAAY,aAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS;AAAA,IACvD,cAAc,aAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS;AAAA,IACzD,yBAAyB,aAAE,QAAQ,EAAE,SAAS;AAAA,IAC9C,wBAAwB,aAAE,QAAQ,EAAE,SAAS;AAAA,IAC7C,yBAAyB,aAAE,QAAQ,EAAE,SAAS;AAAA,EAChD,CAAC,EACA,OAAO,EACP,SAAS;AAAA,EACZ,gBAAgB,aACb,OAAO;AAAA,IACN,aAAa,aAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG,EAAE,SAAS;AAAA,IACjD,cAAc,aAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG,EAAE,SAAS;AAAA,IAClD,OAAO,aAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG,EAAE,SAAS;AAAA,EAC7C,CAAC,EACA,OAAO,EACP,SAAS;AAAA,EACZ,SAAS,aACN,OAAO;AAAA,IACN,QAAQ,aAAE,QAAQ,UAAU;AAAA,IAC5B,UAAU,aAAE,QAAQ,KAAK;AAAA,IACzB,kBAAkB,aAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK;AAAA,EACxD,CAAC,EACA,OAAO,EACP,SAAS;AACd,CAAC,EACA,OAAO;AACH,IAAM,uBAAuB,aACjC,OAAO;AAAA,EACN,OAAO,aAAE,OAAO,EAAE,IAAI,EAAE,YAAY;AAAA,EACpC,OAAO,aAAE,MAAM;AAAA,IACb,aAAE,OAAO,EAAE,SAAS,aAAE,OAAO,EAAE,CAAC,EAAE,OAAO;AAAA,IACzC,aAAE,OAAO,EAAE,QAAQ,yBAAyB,CAAC,EAAE,OAAO;AAAA,EACxD,CAAC;AAAA,EACD;AAAA,EACA,aAAa;AAAA,EACb,WAAW,aAAE,KAAK,EAAE,SAAS;AAAA,EAC7B;AAAA,EACA,QAAQ,aAAE,OAAO,EAAE,SAAS;AAAA,EAC5B,YAAY,aAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS;AAAA,EACpD,YAAY,aAAE,MAAM,oBAAoB,EAAE,SAAS;AAAA,EACnD,UAAU,aAAE;AAAA,IACV,kBAAkB,OAAO,EAAE,IAAI,aAAE,OAAO,GAAG,MAAM,aAAE,OAAO,EAAE,CAAC;AAAA,EAC/D;AACF,CAAC,EACA,OAAO;AACH,IAAM,uBAAuB,aACjC,OAAO;AAAA,EACN,QAAQ;AAAA,EACR,YAAY,aAAE,OAAO,EAAE,IAAI,EAAE,SAAS;AAAA,EACtC,WAAW,aAAE,OAAO;AAAA,EACpB;AAAA,EACA,YAAY,aAAE,OAAO,EAAE,YAAY;AAAA,EACnC,OAAO,aAAE,MAAM,oBAAoB;AAAA,EACnC,QAAQ,aAAE,MAAM,KAAK;AACvB,CAAC,EACA,OAAO;AACH,IAAM,0BAA0B,aACpC,OAAO;AAAA,EACN;AAAA,EACA,SAAS;AAAA,EACT,UAAU,aAAE,KAAK,CAAC,OAAO,QAAQ,CAAC,EAAE,SAAS;AAC/C,CAAC,EACA,OAAO;AACV,IAAM,QAAQ;AACP,IAAM,+BAA+B,aACzC,OAAO;AAAA,EACN,aAAa;AAAA,EACb,OAAO;AAAA,EACP,WAAW,aACR,OAAO,EAAE,IAAI,MAAM,aAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,KAAM,EAAE,CAAC,EACzD,OAAO;AAAA,EACV,OAAO,qBAAqB,MAAM;AAAA,EAClC,aAAa;AAAA,EACb,WAAW,aAAE,KAAK,EAAE,SAAS;AAAA,EAC7B,cAAc,aAAE,MAAM,oBAAoB,EAAE,IAAI,GAAG;AACrD,CAAC,EACA,OAAO;AACH,IAAM,gCAAgC,aAC1C,OAAO;AAAA,EACN,OAAO;AAAA,EACP,SAAS;AAAA,EACT,OAAO,aAAE,MAAM,oBAAoB,EAAE,SAAS;AAChD,CAAC,EACA,OAAO;AACH,IAAM,sBAAsB,aAChC,OAAO;AAAA,EACN,IAAI,aAAE,KAAK;AAAA,EACX,SAAS;AAAA,EACT,eAAe,aAAE,OAAO,EAAE,MAAM,gBAAgB;AAAA,EAChD,OAAO;AAAA,EACP,OAAO,MAAM,SAAS;AAAA,EACtB,WAAW,aAAE,IAAI,SAAS;AAAA,EAC1B,YAAY,aAAE,OAAO,EAAE,YAAY;AAAA,EACnC;AAAA,EACA,QAAQ,aACL,OAAO;AAAA,IACN,QAAQ,aAAE,OAAO,EAAE,IAAI,EAAE,YAAY;AAAA,IACrC,QAAQ,aAAE,OAAO,EAAE,IAAI,EAAE,YAAY;AAAA,IACrC,OAAO,aAAE,OAAO,EAAE,IAAI,EAAE,YAAY;AAAA,IACpC,WAAW,aAAE,OAAO,EAAE,IAAI,EAAE,YAAY;AAAA,IACxC,UAAU,aAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS;AAAA,EACpD,CAAC,EACA,OAAO;AAAA,EACV,OAAO,aAAE,MAAM,oBAAoB;AAAA,EACnC,QAAQ,aAAE,MAAM,KAAK;AACvB,CAAC,EACA,OAAO;AACH,IAAM,qBAAqB,aAC/B,OAAO;AAAA,EACN,eAAe,aAAE,QAAQ,CAAC;AAAA,EAC1B,eAAe,aAAE,QAAQ,IAAI,EAAE,SAAS;AAAA,EACxC,QAAQ,aAAE,MAAM,eAAe;AAAA,EAC/B,YAAY,aAAE;AAAA,IACZ,aACG,OAAO;AAAA,MACN,MAAM;AAAA,MACN,UAAU,GAAG,SAAS;AAAA,MACtB,eAAe,GAAG,SAAS;AAAA,IAC7B,CAAC,EACA,OAAO;AAAA,EACZ;AAAA,EACA,YAAY,aAAE,MAAM,EAAE;AAAA,EACtB,cAAc,aAAE,OAAO,aAAE,OAAO,GAAG,aAAE,QAAQ,CAAC,EAAE,SAAS;AAAA,EACzD,OAAO,MAAM,SAAS;AACxB,CAAC,EACA,OAAO;AACH,IAAM,qBAAqB,aAAE,mBAAmB,QAAQ;AAAA,EAC7D,aACG,OAAO;AAAA,IACN,MAAM,aAAE,QAAQ,aAAa;AAAA,IAC7B,SAAS,aAAE,MAAM,EAAE,EAAE,IAAI,CAAC;AAAA,IAC1B,aAAa,aAAE,OAAO,EAAE,IAAI,EAAE,SAAS;AAAA,IACvC,aAAa,aAAE,OAAO,EAAE,IAAI,EAAE,SAAS;AAAA,EACzC,CAAC,EACA,OAAO;AAAA,EACV,aACG,OAAO;AAAA,IACN,MAAM,aAAE,QAAQ,eAAe;AAAA,IAC/B,QAAQ;AAAA,IACR,YAAY,aAAE,OAAO,EAAE,IAAI,EAAE,SAAS;AAAA,IACtC,OAAO,MAAM,MAAM;AAAA,IACnB,WAAW,aAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS;AAAA,EACrD,CAAC,EACA,OAAO;AAAA,EACV,aACG,OAAO;AAAA,IACN,MAAM,aAAE,QAAQ,cAAc;AAAA,IAC9B,QAAQ;AAAA,IACR,YAAY,aAAE,OAAO,EAAE,IAAI,EAAE,SAAS;AAAA,IACtC,WAAW,aAAE,OAAO;AAAA,EACtB,CAAC,EACA,OAAO;AAAA,EACV,aACG,OAAO;AAAA,IACN,MAAM,aAAE,QAAQ,gBAAgB;AAAA,IAChC,QAAQ;AAAA,IACR,YAAY,aAAE,OAAO,EAAE,IAAI,EAAE,SAAS;AAAA,IACtC,MAAM;AAAA,EACR,CAAC,EACA,OAAO;AAAA,EACV,aACG,OAAO,EAAE,MAAM,aAAE,QAAQ,gBAAgB,GAAG,QAAQ,qBAAqB,CAAC,EAC1E,OAAO;AACZ,CAAC;AACM,IAAM,6BAA6B,aACvC,OAAO;AAAA,EACN,SAAS;AAAA,EACT,eAAe,aAAE,OAAO,EAAE,MAAM,gBAAgB;AAAA,EAChD,SAAS,aAAE,KAAK,CAAC,OAAO,QAAQ,CAAC,EAAE,SAAS;AAAA,EAC5C,OAAO,wBAAwB,SAAS;AAAA,EACxC,SAAS,aAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG,EAAE,SAAS;AAAA,EAC9C,aAAa,aAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,QAAQ,CAAC;AAAA,EACtD,aAAa,aAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,QAAQ,CAAC;AACvD,CAAC,EACA,OAAO;AACH,IAAM,+BAA+B,2BAA2B,KAAK;AAAA,EAC1E,SAAS;AAAA,EACT,OAAO;AACT,CAAC,EAAE,OAAO;AAAA,EACR,UAAU,aAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG;AAAA,EACnC,OAAO,aAAE,KAAK,CAAC,UAAU,KAAK,CAAC,EAAE,QAAQ,QAAQ;AACnD,CAAC;AACM,IAAM,sBAAsB,aAAE,mBAAmB,QAAQ;AAAA,EAC9D,aAAE,OAAO,EAAE,MAAM,aAAE,QAAQ,UAAU,GAAG,OAAO,mBAAmB,CAAC,EAAE,OAAO;AAAA,EAC5E,aAAE,OAAO,EAAE,MAAM,aAAE,QAAQ,QAAQ,GAAG,QAAQ,oBAAoB,CAAC,EAAE,OAAO;AAAA,EAC5E,aAAE,OAAO,EAAE,MAAM,aAAE,QAAQ,OAAO,GAAG,SAAS,aAAE,OAAO,EAAE,CAAC,EAAE,OAAO;AACrE,CAAC;AAKM,IAAM,yBAAN,cAAqC,MAAM;AAAA,EACvC,OAAO;AAClB;AAEO,SAAS,eAAe,OAA2B;AACxD,QAAM,SAAS,gBAAgB,UAAU,KAAK;AAC9C,MAAI,CAAC,OAAO;AACV,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AACF,QAAM,MAAM,OAAO,KAAK,MAAM,IAAI,CAAC,SAAS,KAAK,EAAE;AACnD,MAAI,IAAI,IAAI,GAAG,EAAE,SAAS,IAAI;AAC5B,UAAM,IAAI,uBAAuB,yCAAyC;AAC5E,aAAW,QAAQ,OAAO,KAAK,OAAO;AACpC,QAAI,UAAU;AACd,eAAW,QAAQ,KAAK,OAAO;AAC7B,UAAI,YAAY,SAAS;AACvB,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AACF,gBAAU,KAAK,OAAO,SAAS;AAAA,IACjC;AAAA,EACF;AACA,SAAO,OAAO;AAChB;;;ACvWO,SAAS,YACd,OACsB;AACtB,SAAO;AACT;;;ACdA,IAAAA,cAAkB;AAQX,IAAM,kCAAkC,cAAE,OAAO;AAAA,EACtD,QAAQ,cAAE,QAAQ;AAAA,EAClB,aAAa,wBAAwB,SAAS,EAAE,SAAS;AAAA,EACzD,SAAS,cAAE;AAAA,IACT,cAAE,OAAO;AAAA,MACP,IAAI,cAAE,OAAO;AAAA,MACb,MAAM,cAAE,OAAO;AAAA,MACf,aAAa,cAAE,OAAO;AAAA,MACtB,OAAO,cAAE,OAAO,EAAE,SAAS;AAAA,MAC3B,UAAU,mBAAmB,SAAS;AAAA,MACtC,WAAW,cAAE,OAAO,cAAE,OAAO,GAAG,cAAE,OAAO,CAAC;AAAA,IAC5C,CAAC;AAAA,EACH;AACF,CAAC;AACM,IAAM,uBAAuB,cAAE,OAAO;AAAA,EAC3C,QAAQ,cAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS;AAAA,EACnD,UAAU,cAAE,OAAO,EAAE,SAAS;AAAA,EAC9B,QAAQ,cAAE,KAAK,CAAC,YAAY,WAAW,aAAa,CAAC;AAAA,EACrD,OAAO,cAAE,OAAO,EAAE,IAAI,EAAE,YAAY;AAAA,EACpC,eAAe,cAAE,OAAO,EAAE,IAAI,EAAE,YAAY;AAAA,EAC5C,WAAW,cAAE,QAAQ;AACvB,CAAC;AACM,IAAM,kBAAkB,cAAE,OAAO;AAAA,EACtC,UAAU;AAAA,EACV,OAAO;AAAA,EACP,OAAO;AACT,CAAC;AAGM,IAAM,6BAA6B,cAAE,OAAO;AAAA,EACjD,IAAI,cAAE,KAAK;AAAA,EACX,UAAU,cAAE,OAAO;AAAA,EACnB,YAAY,cAAE,OAAO;AAAA,EACrB,aAAa,cAAE,OAAO;AAAA,EACtB,SAAS,cAAE,OAAO;AAAA,EAClB,eAAe,cAAE,OAAO;AAAA,EACxB,QAAQ,cAAE,KAAK;AAAA,IACb;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AAAA,EACD,WAAW,cAAE,OAAO;AAAA,EACpB,WAAW,cAAE,OAAO,EAAE,SAAS;AAAA,EAC/B,SAAS,cAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,OAAO,cAAE,OAAO,EAAE,SAAS;AAAA,EAC3B,mBAAmB,cAAE,OAAO,EAAE,SAAS;AAAA,EACvC,OAAO,gBAAgB,SAAS;AAAA,EAChC,QAAQ,cACL,OAAO;AAAA,IACN,QAAQ,cAAE,OAAO;AAAA,IACjB,QAAQ,cAAE,OAAO;AAAA,IACjB,OAAO,cAAE,OAAO;AAAA,IAChB,WAAW,cAAE,OAAO;AAAA,EACtB,CAAC,EACA,SAAS,EACT,SAAS;AAAA,EACZ,WAAW,cAAE,OAAO,EAAE,SAAS,EAAE,SAAS;AAC5C,CAAC;AACM,IAAM,0BAA0B,cAAE,OAAO;AAAA,EAC9C,MAAM,cAAE,MAAM,0BAA0B;AAC1C,CAAC;AACM,IAAM,yBAAyB,cAAE,OAAO;AAAA,EAC7C,KAAK,2BAA2B,OAAO;AAAA,IACrC,WAAW,cAAE,OAAO,cAAE,OAAO,GAAG,eAAe,EAAE,SAAS;AAAA,IAC1D,SAAS,cACN,OAAO;AAAA,MACN,SAAS,cAAE,MAAM,cAAE,OAAO,CAAC,EAAE,SAAS;AAAA,MACtC,SAAS,cAAE,KAAK,CAAC,OAAO,QAAQ,CAAC,EAAE,SAAS;AAAA,MAC5C,OAAO,wBAAwB,SAAS;AAAA,MACxC,aAAa,cAAE,OAAO,EAAE,IAAI,EAAE,SAAS;AAAA,MACvC,aAAa,cAAE,OAAO,EAAE,SAAS;AAAA,IACnC,CAAC,EACA,SAAS;AAAA,IACZ,OAAO;AAAA,IACP,QAAQ,oBAAoB,SAAS;AAAA,IACrC,QAAQ,cAAE,MAAM,cAAE,OAAO,EAAE,IAAI,cAAE,OAAO,GAAG,OAAO,mBAAmB,CAAC,CAAC;AAAA,EACzE,CAAC;AACH,CAAC;;;AChFD,eAAe,SACb,UACA,QACkB;AAClB,MAAI,CAAC,SAAS,MAAM,CAAC,SAAS,MAAM;AAClC,UAAM,SAAS,MAAM,OAAO,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAC5C,UAAM,IAAI,MAAM,sDAAsD;AAAA,EACxE;AACA,QAAM,SAAS,SAAS,KAAK,UAAU;AACvC,QAAM,QAAQ,MAAM;AAClB,SAAK,OAAO,OAAO,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACrC;AACA,SAAO,iBAAiB,SAAS,OAAO,EAAE,MAAM,KAAK,CAAC;AACtD,QAAM,UAAU,IAAI,YAAY;AAChC,MAAI,OAAO;AACX,MAAI,QAAQ;AACZ,MAAI;AACF,eAAS;AACP,aAAO,eAAe;AACtB,YAAM,OAAO,MAAM,OAAO,KAAK;AAC/B,UAAI,KAAK,KAAM;AACf,eAAS,KAAK,MAAM;AACpB,UAAI,QAAQ;AACV,cAAM,IAAI,MAAM,qCAAqC;AACvD,cAAQ,QAAQ,OAAO,KAAK,OAAO,EAAE,QAAQ,KAAK,CAAC;AAAA,IACrD;AACA,WAAO,eAAe;AACtB,WAAO,KAAK,MAAM,OAAO,QAAQ,OAAO,CAAC;AAAA,EAC3C,UAAE;AACA,WAAO,oBAAoB,SAAS,KAAK;AACzC,UAAM,OAAO,OAAO,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AACpC,WAAO,YAAY;AAAA,EACrB;AACF;AAGA,eAAsB,sBACpB,SACoB;AACpB,QAAM,OAAO,IAAI,IAAI,QAAQ,GAAG;AAChC,MACE,KAAK,YACL,KAAK,YACL,KAAK,UACL,KAAK,QACJ,KAAK,aAAa,OAAO,KAAK,aAAa,MAC3C,KAAK,aAAa,YACjB,EAAE,QAAQ,qBAAqB,KAAK,aAAa,YACnD,CAAC,QAAQ,OAAO,KAAK,KACrB,CAAC,QAAQ,YAAY,KAAK;AAE1B,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AACF,QAAM,WAAW,IAAI,IAAI,0BAA0B,IAAI;AACvD,QAAM,YAAY,IAAI,IAAI,QAAQ;AAClC,YAAU,aAAa,IAAI,eAAe,QAAQ,WAAW;AAC7D,QAAM,SAAS,QAAQ,SACnB,YAAY,IAAI,CAAC,QAAQ,QAAQ,YAAY,QAAQ,IAAM,CAAC,CAAC,IAC7D,YAAY,QAAQ,IAAM;AAC9B,QAAM,WAAW,wBAAwB;AAAA,IACvC,MAAM;AAAA,MACJ,MAAM,MAAM,WAAW;AAAA,QACrB,SAAS,EAAE,eAAe,UAAU,QAAQ,MAAM,GAAG;AAAA,QACrD,UAAU;AAAA,QACV;AAAA,MACF,CAAC;AAAA,MACD;AAAA,IACF;AAAA,EACF;AACA,MAAI,SAAS,aAAa;AACxB,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AACF,SAAO;AAAA,IACL,GAAG;AAAA,IACH,UAAU;AAAA,IACV,MAAM,MAAM,OAAO;AACjB,YAAM,UAAU,6BAA6B,MAAM;AAAA,QACjD,aAAa,QAAQ;AAAA,QACrB,OAAO;AAAA,QACP,WAAW,MAAM;AAAA,QACjB,OAAO,MAAM;AAAA,QACb,aAAa,MAAM;AAAA,QACnB,WAAW,MAAM;AAAA,QACjB,cAAc,MAAM;AAAA,MACtB,CAAC;AACD,YAAMC,UAAS,YAAY,IAAI;AAAA,QAC7B,MAAM;AAAA,QACN,YAAY,QAAQ,GAAM;AAAA,MAC5B,CAAC;AACD,MAAAA,QAAO,eAAe;AACtB,YAAM,WAAW,8BAA8B;AAAA,QAC7C,MAAM;AAAA,UACJ,MAAM,MAAM,UAAU;AAAA,YACpB,QAAQ;AAAA,YACR,SAAS;AAAA,cACP,eAAe,UAAU,QAAQ,MAAM;AAAA,cACvC,gBAAgB;AAAA,YAClB;AAAA,YACA,MAAM,KAAK,UAAU,OAAO;AAAA,YAC5B,UAAU;AAAA,YACV,QAAAA;AAAA,UACF,CAAC;AAAA,UACDA;AAAA,QACF;AAAA,MACF;AACA,UACE,SAAS,MAAM,OAAO,SAAS,MAC/B,SAAS,MAAM,YAAY,SAAS,WACpC,SAAS,MAAM,aAAa;AAE5B,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AACF,iBAAW,SAAS,SAAS,SAAS,CAAC,EAAG,OAAM,UAAU,KAAK;AAC/D,aAAO,SAAS;AAAA,IAClB;AAAA,EACF;AACF;","names":["import_zod","signal"]}
|