anpord 0.1.11 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +56 -78
  2. package/dist/bin.cjs +211 -204
  3. package/dist/bin.mjs +211 -204
  4. package/dist/cli-runtime.cjs +2 -2
  5. package/dist/cli-runtime.d.cts +1 -1
  6. package/dist/cli-runtime.d.mts +1 -1
  7. package/dist/cli-runtime.mjs +2 -2
  8. package/dist/client-BfdjFtlV.mjs +305 -0
  9. package/dist/{client-C9m0uLCz.d.cts → client-BgRs8JvW.d.mts} +2150 -174
  10. package/dist/{client-TFCLuMv8.d.mts → client-BirFO3jz.d.cts} +2150 -174
  11. package/dist/client-C-P2I00z.cjs +346 -0
  12. package/dist/{compiler-DXlpU_In.mjs → compiler-D96wI7UP.mjs} +108 -75
  13. package/dist/{compiler-Ap8DM-JU.cjs → compiler-uwb1pkdK.cjs} +116 -83
  14. package/dist/config.cjs +1 -1
  15. package/dist/config.d.cts +1 -1
  16. package/dist/config.d.mts +1 -1
  17. package/dist/config.mjs +1 -1
  18. package/dist/{errors-C9bQUcA3.mjs → errors-B0YknR5V.mjs} +3 -19
  19. package/dist/{errors-Boum34f5.cjs → errors-BX1wry8K.cjs} +3 -19
  20. package/dist/{errors-BK3f7Rdr.d.cts → errors-aGrd21jy.d.cts} +0 -3
  21. package/dist/{errors-BK3f7Rdr.d.mts → errors-aGrd21jy.d.mts} +0 -3
  22. package/dist/eval-judges-BkmiVLLq.cjs +59 -0
  23. package/dist/eval-judges-CwTvTq7N.mjs +42 -0
  24. package/dist/eval-judges-DPPUftbh.d.cts +48 -0
  25. package/dist/eval-judges-DPPUftbh.d.mts +48 -0
  26. package/dist/eval-validations-BRdoZrTQ.mjs +113 -0
  27. package/dist/eval-validations-_MlnQ_uv.cjs +148 -0
  28. package/dist/eval.cjs +1 -1
  29. package/dist/eval.d.cts +60 -6
  30. package/dist/eval.d.mts +60 -6
  31. package/dist/eval.mjs +1 -1
  32. package/dist/evals-B6aT3xgH.d.cts +1364 -0
  33. package/dist/evals-B6aT3xgH.d.mts +1364 -0
  34. package/dist/evals-Dcc_x6jW.mjs +582 -0
  35. package/dist/evals-DouMsrsM.cjs +737 -0
  36. package/dist/index.cjs +23 -99
  37. package/dist/index.d.cts +138 -34
  38. package/dist/index.d.mts +138 -34
  39. package/dist/index.mjs +23 -99
  40. package/dist/source.d.cts +1 -1
  41. package/dist/source.d.mts +1 -1
  42. package/dist/{types-oaMF6-E1.d.mts → types-Codjaq-o.d.mts} +146 -45
  43. package/dist/{types-H9L450Iu.d.cts → types-ISsteq95.d.cts} +146 -45
  44. package/dist/validator-runtime.cjs +238 -0
  45. package/dist/validator-runtime.d.cts +9 -0
  46. package/dist/validator-runtime.d.mts +9 -0
  47. package/dist/validator-runtime.mjs +237 -0
  48. package/dist/validators.cjs +10 -0
  49. package/dist/validators.d.cts +8 -0
  50. package/dist/validators.d.mts +8 -0
  51. package/dist/validators.mjs +9 -0
  52. package/package.json +11 -1
  53. package/dist/client-5K3dMI5Q.mjs +0 -849
  54. package/dist/client-Ci0woWLW.cjs +0 -890
  55. package/dist/evals-BEgEmfiO.d.cts +0 -603
  56. package/dist/evals-BEgEmfiO.d.mts +0 -603
  57. package/dist/harness-profile-BMjsN460.mjs +0 -43
  58. package/dist/harness-profile-CDQpPLPd.cjs +0 -78
@@ -0,0 +1,582 @@
1
+ import { n as EvalJudgment, r as EvalHarness$1, t as EvalJudge } from "./eval-judges-CwTvTq7N.mjs";
2
+ import { t as EvalValidations } from "./eval-validations-BRdoZrTQ.mjs";
3
+ import { Schema } from "effect";
4
+ //#region ../schema/src/domain/eval-limits.ts
5
+ const PROMPT_LIMIT = 16384;
6
+ const VERIFY_LIMIT = 8192;
7
+ const NAME_LIMIT = 200;
8
+ const VARIABLE_VALUE_LIMIT = 2048;
9
+ const EvalPrompt = Schema.String.pipe(Schema.maxLength(PROMPT_LIMIT), Schema.annotations({
10
+ description: "What the agent is asked to do.",
11
+ identifier: "EvalPrompt",
12
+ message: () => `A prompt must be at most ${PROMPT_LIMIT} characters, because it is quoted into one sandbox command line.`
13
+ }));
14
+ const EvalVerify = Schema.String.pipe(Schema.maxLength(VERIFY_LIMIT), Schema.annotations({
15
+ description: "The shell command that decides whether a trial passed.",
16
+ identifier: "EvalVerify",
17
+ message: () => `A verifier must be at most ${VERIFY_LIMIT} characters, because it is quoted into one sandbox command line.`
18
+ }));
19
+ const EvalCaseName = Schema.String.pipe(Schema.maxLength(NAME_LIMIT), Schema.annotations({
20
+ description: "What a case is called.",
21
+ identifier: "EvalCaseName",
22
+ message: () => `A case name must be at most ${NAME_LIMIT} characters.`
23
+ }));
24
+ const EvalVariableValue = Schema.String.pipe(Schema.maxLength(VARIABLE_VALUE_LIMIT), Schema.annotations({
25
+ identifier: "EvalVariableValue",
26
+ message: () => `A variable value must be at most ${VARIABLE_VALUE_LIMIT} characters, because it is substituted into the prompt before it is quoted.`
27
+ }));
28
+ //#endregion
29
+ //#region ../schema/src/domain/eval-trigger.ts
30
+ const EvalTrigger = Schema.Struct({
31
+ source: Schema.Literal("dashboard", "api", "cli", "ci", "mcp"),
32
+ url: Schema.optional(Schema.String.pipe(Schema.maxLength(2048), Schema.filter((value) => {
33
+ if (!URL.canParse(value)) return false;
34
+ const url = new URL(value);
35
+ return ["https:", "http:"].includes(url.protocol) && !url.username && !url.password;
36
+ }, { message: () => "Use an HTTP or HTTPS URL without credentials." })))
37
+ }).annotations({
38
+ identifier: "EvalTrigger",
39
+ description: "What started this run. Client-reported context, not authorization."
40
+ });
41
+ //#endregion
42
+ //#region ../schema/src/domain/credentials.ts
43
+ const CredentialScope = Schema.Literal("organization", "personal");
44
+ const CredentialStatus = Schema.Literal("active", "invalid");
45
+ const CredentialField = Schema.Struct({
46
+ hint: Schema.optional(Schema.String),
47
+ label: Schema.String,
48
+ name: Schema.String,
49
+ required: Schema.Boolean,
50
+ secret: Schema.Boolean
51
+ });
52
+ const CredentialAuthMethod = Schema.Struct({
53
+ fields: Schema.Array(CredentialField),
54
+ id: Schema.String,
55
+ kind: Schema.Literal("secret", "device", "env"),
56
+ label: Schema.String
57
+ });
58
+ Schema.Struct({
59
+ authMethods: Schema.Array(CredentialAuthMethod),
60
+ category: Schema.Literal("harness", "sandbox"),
61
+ id: Schema.String,
62
+ label: Schema.String
63
+ });
64
+ Schema.Struct({
65
+ integrationId: Schema.String,
66
+ owners: Schema.Array(Schema.String)
67
+ }).annotations({
68
+ description: "Who else in the organization has an integration connected.",
69
+ identifier: "IntegrationAwareness"
70
+ });
71
+ Schema.Struct({
72
+ authMethodId: Schema.String,
73
+ createdAt: Schema.DateTimeUtc,
74
+ id: Schema.String,
75
+ integrationId: Schema.String,
76
+ isDefault: Schema.Boolean,
77
+ lastUsedAt: Schema.NullOr(Schema.DateTimeUtc),
78
+ name: Schema.String,
79
+ scope: CredentialScope,
80
+ status: CredentialStatus
81
+ });
82
+ const CredentialValues = Schema.Record({
83
+ key: Schema.String,
84
+ value: Schema.String
85
+ });
86
+ Schema.Struct({
87
+ authMethodId: Schema.String,
88
+ connectionId: Schema.String,
89
+ integrationId: Schema.String,
90
+ revision: Schema.Int,
91
+ values: CredentialValues
92
+ });
93
+ Schema.Struct({
94
+ authMethodId: Schema.String,
95
+ integrationId: Schema.String,
96
+ isDefault: Schema.optionalWith(Schema.Boolean, { default: () => false }),
97
+ name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(80)),
98
+ scope: CredentialScope,
99
+ values: CredentialValues
100
+ });
101
+ const CredentialBindings = Schema.Struct({
102
+ harnessConnectionId: Schema.optional(Schema.String),
103
+ sandboxConnectionId: Schema.optional(Schema.String)
104
+ });
105
+ const CredentialSelections = Schema.Record({
106
+ key: Schema.String,
107
+ value: Schema.String
108
+ });
109
+ Schema.Struct({ values: CredentialValues });
110
+ Schema.Struct({
111
+ integrationId: Schema.Literal("codex"),
112
+ name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(80)),
113
+ scope: CredentialScope
114
+ });
115
+ Schema.Struct({
116
+ attemptId: Schema.String,
117
+ code: Schema.String,
118
+ expiresAt: Schema.DateTimeUtc,
119
+ verificationUrl: Schema.String
120
+ });
121
+ Schema.Struct({
122
+ connectionId: Schema.NullOr(Schema.String),
123
+ status: Schema.Literal("pending", "complete", "failed", "expired")
124
+ });
125
+ //#endregion
126
+ //#region ../schema/src/domain/eval-source-files.ts
127
+ const EvalSourceFile = Schema.Struct({
128
+ path: Schema.String.pipe(Schema.maxLength(512), Schema.pattern(/^(?!\.{1,2}(?:\/|$))[^/\\:\0]+(?:\/(?!\.{1,2}(?:\/|$))[^/\\:\0]+)*$/)),
129
+ content: Schema.String.pipe(Schema.maxLength(1e6))
130
+ });
131
+ const EvalSourceFiles = Schema.Array(EvalSourceFile).pipe(Schema.maxItems(100), Schema.filter((files) => new Set(files.map(({ path }) => path)).size === files.length), Schema.filter((files) => files.reduce((size, file) => size + file.content.length, 0) <= 1e6));
132
+ //#endregion
133
+ //#region ../schema/src/domain/harness-profile.ts
134
+ const PROFILE_LIMITS = {
135
+ fileChars: 96e3,
136
+ files: 256,
137
+ totalChars: 2e6
138
+ };
139
+ const ProfileName = Schema.String.pipe(Schema.pattern(/^[a-z0-9][a-z0-9-]{0,63}$/), Schema.annotations({
140
+ description: "Lowercase letters, digits and hyphens, at most 64 characters, starting with a letter or digit.",
141
+ identifier: "ProfileName"
142
+ }));
143
+ const ProfilePath = Schema.String.pipe(Schema.pattern(/^(home|workspace)(?:\/(?!\.\.?(?:\/|$))[^/\0]+)+$/), Schema.maxLength(512), Schema.annotations({
144
+ description: "A relative path under home/ or workspace/, without a .. segment.",
145
+ identifier: "ProfilePath"
146
+ }));
147
+ const EnvName = Schema.String.pipe(Schema.pattern(/^[A-Z_][A-Z0-9_]*$/), Schema.annotations({
148
+ description: "An environment variable name.",
149
+ identifier: "EnvName"
150
+ }));
151
+ const ProfileFiles = Schema.Record({
152
+ key: ProfilePath,
153
+ value: Schema.String.pipe(Schema.maxLength(PROFILE_LIMITS.fileChars))
154
+ }).pipe(Schema.filter((files) => Object.keys(files).length <= PROFILE_LIMITS.files, { message: () => `A profile holds at most ${PROFILE_LIMITS.files} files.` }), Schema.filter((files) => Object.values(files).reduce((sum, content) => sum + content.length, 0) <= PROFILE_LIMITS.totalChars, { message: () => `A profile's files hold at most ${PROFILE_LIMITS.totalChars} characters in total.` }));
155
+ const HarnessProfile = Schema.Struct({
156
+ env: Schema.optional(Schema.Record({
157
+ key: EnvName,
158
+ value: Schema.String
159
+ })),
160
+ files: ProfileFiles,
161
+ install: Schema.optional(Schema.String),
162
+ name: ProfileName,
163
+ run: Schema.optional(Schema.String),
164
+ systemPrompt: Schema.optional(Schema.String)
165
+ }).annotations({
166
+ description: "Configuration layered on a harness: files written under the sandbox home and workspace, a system prompt, environment, an install command run before the harness, and for the command harness the run command.",
167
+ identifier: "HarnessProfile"
168
+ });
169
+ const profileFitsHarness = (task) => task.harness === "command" ? task.profile?.run !== void 0 : task.profile?.run === void 0;
170
+ const PROFILE_HARNESS_RULE = "The command harness needs a profile with a run command; other harnesses take no run. Any harness may take an install command.";
171
+ //#endregion
172
+ //#region ../schema/src/domain/evals.ts
173
+ const EvalHarness = EvalHarness$1;
174
+ const EvalSandbox = Schema.Literal("daytona", "e2b", "upstash", "modal", "cloudflare", "vercel");
175
+ EvalSandbox.literals;
176
+ const EvalTrialStatus = Schema.Literal("queued", "running", "passed", "failed", "void");
177
+ const EvalRunStatus = Schema.Literal("running", "finished", "failed");
178
+ const EvalSource = Schema.Union(Schema.Struct({ kind: Schema.Literal("empty") }), Schema.Struct({
179
+ kind: Schema.Literal("repo"),
180
+ ref: Schema.NullOr(Schema.String),
181
+ url: Schema.String.pipe(Schema.minLength(1), Schema.annotations({ message: () => "Give the repository a URL." }))
182
+ }), Schema.Struct({
183
+ files: Schema.Record({
184
+ key: Schema.String,
185
+ value: Schema.String
186
+ }),
187
+ kind: Schema.Literal("files")
188
+ })).annotations({
189
+ description: "The workspace available to the harness before setup runs.",
190
+ identifier: "EvalSource"
191
+ });
192
+ const EvalCodeValidator = Schema.Struct({
193
+ capture: Schema.optional(Schema.Boolean),
194
+ manifest: Schema.optional(Schema.Array(Schema.Struct({
195
+ index: Schema.NonNegativeInt,
196
+ name: Schema.String.pipe(Schema.maxLength(200))
197
+ })).pipe(Schema.minItems(1), Schema.maxItems(20), Schema.filter((checks) => new Set(checks.map((check) => check.index)).size === checks.length, { message: () => "Validator indices must be unique" }))),
198
+ name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(100)),
199
+ source: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(1e6))
200
+ });
201
+ const EvalValidator = Schema.Union(EvalCodeValidator.pipe(Schema.extend(Schema.Struct({ sourceFiles: Schema.optional(EvalSourceFiles) }))), Schema.Struct({
202
+ kind: Schema.Literal("judged"),
203
+ capture: Schema.optional(Schema.Boolean),
204
+ name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(100)),
205
+ checks: Schema.Array(EvalCodeValidator).pipe(Schema.maxItems(20)),
206
+ judges: Schema.Array(EvalJudge).pipe(Schema.minItems(1), Schema.maxItems(20)),
207
+ sourceFiles: Schema.optional(EvalSourceFiles)
208
+ }).pipe(Schema.filter((validator) => validator.checks.reduce((total, check) => total + (check.manifest?.length ?? 1), 0) <= 20, { message: () => "At most 20 code validators are supported" }))).annotations({
209
+ description: "A bundled TypeScript validator and its exported function name.",
210
+ identifier: "EvalValidator"
211
+ });
212
+ const EvalPrepare = Schema.Struct({
213
+ name: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(100)),
214
+ source: Schema.String.pipe(Schema.minLength(1), Schema.maxLength(1e6))
215
+ }).annotations({
216
+ description: "A bundled TypeScript workspace setup and its exported function name.",
217
+ identifier: "EvalPrepare"
218
+ });
219
+ const EvalPrepareValue = Schema.Record({
220
+ key: Schema.String,
221
+ value: Schema.Unknown
222
+ }).annotations({
223
+ description: "What a workspace setup returned, handed to the validator.",
224
+ identifier: "EvalPrepareValue"
225
+ });
226
+ const EvalVariables = Schema.Record({
227
+ key: Schema.String,
228
+ value: EvalVariableValue
229
+ }).annotations({
230
+ description: "Values for the placeholders the run prompt names, such as {{task}}.",
231
+ identifier: "EvalVariables"
232
+ });
233
+ const CaseCache = Schema.Struct({
234
+ key: Schema.String.pipe(Schema.minLength(1)),
235
+ path: Schema.String.pipe(Schema.minLength(1), Schema.filter((value) => !(value.startsWith("/") || value.split("/").includes("..")), { message: () => "a cache path must stay inside the workspace" }))
236
+ }).annotations({ identifier: "CaseCache" });
237
+ const EvalCase = Schema.Struct({
238
+ cache: Schema.optional(CaseCache),
239
+ name: EvalCaseName,
240
+ prepare: Schema.NullOr(EvalPrepare),
241
+ source: EvalSource,
242
+ variables: Schema.optionalWith(EvalVariables, { default: () => ({}) }),
243
+ validator: Schema.optionalWith(Schema.NullOr(EvalValidator), { default: () => null }),
244
+ verify: Schema.NullOr(EvalVerify)
245
+ });
246
+ const EvalTaskProfile = Schema.Struct({
247
+ name: Schema.String,
248
+ version: Schema.String
249
+ }).annotations({
250
+ description: "The profile a cell's harness ran under, by name and version.",
251
+ identifier: "EvalTaskProfile"
252
+ });
253
+ const EvalTask = Schema.Struct({
254
+ harness: EvalHarness,
255
+ harnessVersion: Schema.String,
256
+ model: Schema.String,
257
+ profile: Schema.optional(Schema.NullOr(EvalTaskProfile)),
258
+ sandbox: EvalSandbox
259
+ }).annotations({
260
+ description: "The harness, installed version, profile, model, and sandbox for a cell.",
261
+ identifier: "EvalTask"
262
+ });
263
+ const EvalTaskRequest = Schema.Struct({
264
+ credentials: Schema.optional(CredentialBindings),
265
+ harness: EvalHarness,
266
+ model: Schema.String,
267
+ profile: Schema.optional(HarnessProfile),
268
+ sandbox: Schema.optional(EvalSandbox)
269
+ }).pipe(Schema.filter(profileFitsHarness, { message: () => PROFILE_HARNESS_RULE }));
270
+ const EvalName = Schema.String.pipe(Schema.minLength(1), Schema.maxLength(100));
271
+ Schema.Struct({
272
+ cases: Schema.Array(EvalCase).pipe(Schema.minItems(1), Schema.maxItems(100)),
273
+ name: Schema.optional(EvalName),
274
+ prompt: EvalPrompt,
275
+ tasks: Schema.Array(EvalTaskRequest).pipe(Schema.minItems(1), Schema.maxItems(20)),
276
+ trials: Schema.Int.pipe(Schema.between(1, 10))
277
+ });
278
+ const OccurredAtMillis = Schema.NullOr(Schema.Number);
279
+ const EvalUsage = Schema.Struct({
280
+ cacheReadTokens: Schema.Int,
281
+ cacheWriteTokens: Schema.Int,
282
+ costUsd: Schema.optional(Schema.NullOr(Schema.Number)),
283
+ inputTokens: Schema.Int,
284
+ outputTokens: Schema.Int,
285
+ totalTokens: Schema.Int
286
+ }).annotations({
287
+ description: "Token usage reported by the harness.",
288
+ identifier: "EvalUsage"
289
+ });
290
+ const EvalJournalEntry = Schema.Union(Schema.Struct({
291
+ _tag: Schema.Literal("command"),
292
+ command: Schema.String,
293
+ exitCode: Schema.NullOr(Schema.Int),
294
+ finishedAtMillis: OccurredAtMillis,
295
+ output: Schema.String,
296
+ outputTruncated: Schema.optional(Schema.Boolean),
297
+ startedAtMillis: OccurredAtMillis
298
+ }), Schema.Struct({
299
+ _tag: Schema.Literal("message"),
300
+ finishedAtMillis: OccurredAtMillis,
301
+ text: Schema.String,
302
+ usage: Schema.optional(Schema.NullOr(EvalUsage))
303
+ }), Schema.Struct({
304
+ _tag: Schema.Literal("toolCall"),
305
+ finishedAtMillis: OccurredAtMillis,
306
+ input: Schema.optional(Schema.String),
307
+ name: Schema.String,
308
+ output: Schema.optional(Schema.String),
309
+ error: Schema.optional(Schema.String),
310
+ outputTruncated: Schema.optional(Schema.Boolean),
311
+ inputTruncated: Schema.optional(Schema.Boolean),
312
+ errorTruncated: Schema.optional(Schema.Boolean),
313
+ startedAtMillis: Schema.optional(OccurredAtMillis),
314
+ status: Schema.NullOr(Schema.String)
315
+ }), Schema.Struct({
316
+ _tag: Schema.Literal("fileChange"),
317
+ finishedAtMillis: OccurredAtMillis,
318
+ paths: Schema.Array(Schema.String)
319
+ })).annotations({
320
+ description: "A normalized event recorded from the harness trajectory.",
321
+ identifier: "EvalJournalEntry"
322
+ });
323
+ const EvalVerifyStep = Schema.Struct({
324
+ command: Schema.String,
325
+ exitCode: Schema.Int
326
+ }).annotations({
327
+ description: "One condition of the verifier, and how it exited. Only the steps that ran are listed: the script stops at the first failure.",
328
+ identifier: "EvalVerifyStep"
329
+ });
330
+ const CostClassification = Schema.Literal("actual", "allocated", "estimate", "included", "managed", "unknown");
331
+ const CostComponentName = Schema.Literal("judge", "harness", "model", "platform", "sandbox");
332
+ const EvalCostComponent = Schema.Struct({
333
+ classification: CostClassification,
334
+ component: CostComponentName,
335
+ detail: Schema.Record({
336
+ key: Schema.String,
337
+ value: Schema.Unknown
338
+ }),
339
+ explanation: Schema.String,
340
+ source: Schema.String,
341
+ usd: Schema.NullOr(Schema.Number)
342
+ }).annotations({
343
+ description: "What one layer of a trial cost, and how far that is known.",
344
+ identifier: "EvalCostComponent"
345
+ });
346
+ const EvalCosts = Schema.Struct({
347
+ allocatedUsd: Schema.Number,
348
+ components: Schema.Array(EvalCostComponent),
349
+ estimatedEquivalentUsd: Schema.Number,
350
+ incomplete: Schema.Boolean,
351
+ knownActualUsd: Schema.Number
352
+ }).annotations({
353
+ description: "Cost by component, kept apart by classification.",
354
+ identifier: "EvalCosts"
355
+ });
356
+ const EvalTrial = Schema.Struct({
357
+ validations: Schema.optional(EvalValidations),
358
+ judgments: Schema.optional(Schema.Array(EvalJudgment)),
359
+ commands: Schema.Int,
360
+ costs: Schema.NullOr(EvalCosts),
361
+ prepared: Schema.NullOr(EvalPrepareValue),
362
+ exitCode: Schema.Int,
363
+ failedCommands: Schema.Int,
364
+ filesChanged: Schema.Array(Schema.String),
365
+ modelMs: Schema.Int,
366
+ ordinal: Schema.Int,
367
+ passed: Schema.Boolean,
368
+ sandboxId: Schema.NullOr(Schema.String),
369
+ sandboxMs: Schema.Int,
370
+ status: EvalTrialStatus,
371
+ timed: Schema.Boolean,
372
+ trajectory: Schema.Array(EvalJournalEntry),
373
+ usage: Schema.NullOr(EvalUsage),
374
+ verifySteps: Schema.Array(EvalVerifyStep),
375
+ voidFields: Schema.Array(Schema.String)
376
+ }).annotations({
377
+ description: "One sandbox attempt for a grid cell.",
378
+ identifier: "EvalTrial"
379
+ });
380
+ const EvalDistribution = Schema.Struct({
381
+ commandMax: Schema.Int,
382
+ commandMedian: Schema.Number,
383
+ commandMin: Schema.Int,
384
+ deterministic: Schema.Boolean,
385
+ failed: Schema.Int,
386
+ passRate: Schema.Number,
387
+ passed: Schema.Int,
388
+ scored: Schema.Int,
389
+ trials: Schema.Int,
390
+ voided: Schema.Int
391
+ }).annotations({
392
+ description: "The scored outcome across all trials in a cell.",
393
+ identifier: "EvalDistribution"
394
+ });
395
+ const EvalVerdict = Schema.Literal("improved", "incomparable", "regressed", "unchanged");
396
+ const EvalComparison = Schema.Struct({
397
+ baselineHarnessVersion: Schema.String,
398
+ baselinePassRate: Schema.Number,
399
+ baselineProfileVersion: Schema.NullOr(Schema.String),
400
+ candidateHarnessVersion: Schema.String,
401
+ candidatePassRate: Schema.Number,
402
+ candidateProfileVersion: Schema.NullOr(Schema.String),
403
+ delta: Schema.Number,
404
+ determinismLost: Schema.Boolean,
405
+ reason: Schema.NullOr(Schema.String),
406
+ verdict: EvalVerdict
407
+ }).annotations({
408
+ description: "The cell result compared with its most recent baseline.",
409
+ identifier: "EvalComparison"
410
+ });
411
+ const EvalSetup = Schema.Struct({
412
+ prompt: Schema.String,
413
+ repoRef: Schema.NullOr(Schema.String),
414
+ repoUrl: Schema.NullOr(Schema.String),
415
+ prepareName: Schema.NullOr(Schema.String),
416
+ validatorName: Schema.NullOr(Schema.String),
417
+ validatorFiles: Schema.optional(EvalSourceFiles),
418
+ verifyCommand: Schema.NullOr(Schema.String),
419
+ workspace: Schema.String
420
+ }).annotations({
421
+ description: "The prompt, workspace, setup, and verifier used by a cell.",
422
+ identifier: "EvalSetup"
423
+ });
424
+ const EvalCell = Schema.Struct({
425
+ caseName: Schema.String,
426
+ cellKey: Schema.NullOr(Schema.String),
427
+ costs: Schema.NullOr(EvalCosts),
428
+ comparison: Schema.NullOr(EvalComparison),
429
+ distribution: Schema.NullOr(EvalDistribution),
430
+ internalId: Schema.NullOr(Schema.String),
431
+ setup: Schema.NullOr(EvalSetup),
432
+ status: EvalRunStatus,
433
+ taskIndex: Schema.Int,
434
+ trials: Schema.Array(EvalTrial)
435
+ }).annotations({
436
+ description: "One case and task combination in an eval grid.",
437
+ identifier: "EvalCell"
438
+ });
439
+ const EvalTimestamp = Schema.DateTimeUtc.annotations({
440
+ description: "An ISO-8601 timestamp in UTC.",
441
+ identifier: "EvalTimestamp",
442
+ jsonSchema: { format: "date-time" }
443
+ });
444
+ const EvalRun = Schema.Struct({
445
+ trigger: Schema.optionalWith(Schema.NullOr(EvalTrigger), { default: () => null }),
446
+ cases: Schema.Array(Schema.String),
447
+ cells: Schema.Array(EvalCell),
448
+ costs: Schema.NullOr(EvalCosts),
449
+ failure: Schema.NullOr(Schema.String),
450
+ finishedAt: Schema.NullOr(EvalTimestamp),
451
+ id: Schema.String,
452
+ name: Schema.NullOr(EvalName),
453
+ startedAt: EvalTimestamp,
454
+ status: EvalRunStatus,
455
+ tasks: Schema.Array(EvalTask)
456
+ }).annotations({
457
+ description: "A complete eval run with its cells and trials.",
458
+ identifier: "EvalRun"
459
+ });
460
+ const EvalRunSummary = Schema.Struct({
461
+ trigger: Schema.optionalWith(Schema.NullOr(EvalTrigger), { default: () => null }),
462
+ caseCount: Schema.Int,
463
+ columns: Schema.Array(EvalTask),
464
+ commandMax: Schema.NullOr(Schema.Int),
465
+ commandMin: Schema.NullOr(Schema.Int),
466
+ failure: Schema.NullOr(Schema.String),
467
+ finishedAt: Schema.NullOr(EvalTimestamp),
468
+ firstCaseName: Schema.NullOr(Schema.String),
469
+ id: Schema.String,
470
+ name: Schema.NullOr(Schema.String),
471
+ passed: Schema.Int,
472
+ scored: Schema.Int,
473
+ startedAt: EvalTimestamp,
474
+ status: EvalRunStatus,
475
+ taskCount: Schema.Int,
476
+ voided: Schema.Int
477
+ }).annotations({
478
+ description: "A compact eval run returned by the run list.",
479
+ identifier: "EvalRunSummary"
480
+ });
481
+ const EvalPageCursor = Schema.Struct({
482
+ id: Schema.String,
483
+ startedAtMillis: Schema.Int
484
+ });
485
+ const EvalRunPage = Schema.Struct({
486
+ next: Schema.NullOr(EvalPageCursor),
487
+ runs: Schema.Array(EvalRunSummary),
488
+ total: Schema.Int
489
+ });
490
+ const EvalCellHistoryEntry = Schema.Struct({
491
+ trigger: Schema.optionalWith(Schema.NullOr(EvalTrigger), { default: () => null }),
492
+ distribution: EvalDistribution,
493
+ finishedAt: Schema.NullOr(EvalTimestamp),
494
+ harnessVersion: Schema.String,
495
+ internalId: Schema.String,
496
+ profileVersion: Schema.NullOr(Schema.String),
497
+ runId: Schema.String,
498
+ trials: Schema.Array(EvalTrial)
499
+ }).annotations({
500
+ description: "A previous scored result for the same cell identity.",
501
+ identifier: "EvalCellHistoryEntry"
502
+ });
503
+ const PlaygroundCaseView = Schema.Struct({
504
+ name: Schema.String,
505
+ setup: Schema.NullOr(Schema.String),
506
+ source: EvalSource,
507
+ variables: Schema.optionalWith(EvalVariables, { default: () => ({}) }),
508
+ verify: Schema.NullOr(Schema.String)
509
+ });
510
+ const PlaygroundColumnView = Schema.Struct({
511
+ harness: EvalHarness,
512
+ model: Schema.String,
513
+ sandbox: EvalSandbox
514
+ });
515
+ const PlaygroundConfigView = Schema.Struct({
516
+ cases: Schema.Array(PlaygroundCaseView),
517
+ columns: Schema.Array(PlaygroundColumnView),
518
+ connections: Schema.optionalWith(CredentialSelections, { default: () => ({}) }),
519
+ prompt: Schema.String,
520
+ trials: Schema.Int.pipe(Schema.between(1, 10))
521
+ });
522
+ const EvalDraftCase = Schema.Struct({
523
+ name: Schema.String.pipe(Schema.minLength(1), Schema.annotations({ message: () => "Name this case." })),
524
+ setup: Schema.NullOr(Schema.String),
525
+ source: EvalSource,
526
+ variables: EvalVariables.pipe(Schema.filter((values) => Object.values(values).some((value) => value.trim() !== ""), { message: () => "Say what the agent should do." })),
527
+ verify: Schema.NullOr(Schema.String)
528
+ });
529
+ const EvalAgent = Schema.Struct({
530
+ harness: EvalHarness,
531
+ model: Schema.String
532
+ });
533
+ Schema.Struct({
534
+ agents: Schema.mutable(Schema.Array(EvalAgent)).pipe(Schema.minItems(1), Schema.annotations({ message: () => "Choose at least one agent." })),
535
+ cases: Schema.mutable(Schema.Array(EvalDraftCase)).pipe(Schema.minItems(1), Schema.annotations({ message: () => "Add at least one case." })),
536
+ connections: CredentialSelections,
537
+ name: Schema.String,
538
+ prompt: Schema.String,
539
+ sandboxes: Schema.mutable(Schema.Array(EvalSandbox)),
540
+ trials: Schema.Int.pipe(Schema.between(1, 10), Schema.annotations({ message: () => "Run between 1 and 10 trials." }))
541
+ });
542
+ Schema.Struct({
543
+ config: PlaygroundConfigView,
544
+ id: Schema.String,
545
+ lastRunId: Schema.NullOr(Schema.String),
546
+ name: Schema.String,
547
+ problems: Schema.Array(Schema.String),
548
+ ungated: Schema.Array(Schema.String),
549
+ updatedAt: Schema.DateTimeUtc
550
+ });
551
+ Schema.Struct({ name: Schema.String.pipe(Schema.minLength(1)) });
552
+ Schema.Struct({
553
+ config: PlaygroundConfigView,
554
+ name: Schema.String.pipe(Schema.minLength(1))
555
+ });
556
+ const StartedEval = Schema.Struct({ id: Schema.String }).annotations({
557
+ description: "The id of an eval run accepted for background execution.",
558
+ identifier: "StartedEval"
559
+ });
560
+ const RerunCellRequest = Schema.Struct({
561
+ trigger: Schema.optional(EvalTrigger),
562
+ trials: Schema.Int.pipe(Schema.between(1, 10))
563
+ });
564
+ const CatalogueModel = Schema.Struct({
565
+ displayName: Schema.String,
566
+ id: Schema.String,
567
+ summary: Schema.NullOr(Schema.String),
568
+ vendor: Schema.NullOr(Schema.String)
569
+ }).annotations({
570
+ description: "A model available to the installed harness.",
571
+ identifier: "CatalogueModel"
572
+ });
573
+ const ModelCatalogue = Schema.Struct({
574
+ harness: EvalHarness,
575
+ models: Schema.Array(CatalogueModel),
576
+ total: Schema.Int
577
+ }).annotations({
578
+ description: "Models available to the installed harness.",
579
+ identifier: "ModelCatalogue"
580
+ });
581
+ //#endregion
582
+ export { EvalTrigger as C, EvalVerify as E, EvalSourceFiles as S, EvalPrompt as T, HarnessProfile as _, EvalPageCursor as a, ProfilePath as b, EvalRunPage as c, EvalValidator as d, EvalVariables as f, EnvName as g, StartedEval as h, EvalName as i, EvalSandbox as l, RerunCellRequest as m, EvalCellHistoryEntry as n, EvalPrepare as o, ModelCatalogue as p, EvalHarness as r, EvalRun as s, CaseCache as t, EvalSource as u, PROFILE_HARNESS_RULE as v, EvalCaseName as w, profileFitsHarness as x, PROFILE_LIMITS as y };