anpord 0.1.11 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +56 -78
  2. package/dist/bin.cjs +211 -204
  3. package/dist/bin.mjs +211 -204
  4. package/dist/cli-runtime.cjs +2 -2
  5. package/dist/cli-runtime.d.cts +1 -1
  6. package/dist/cli-runtime.d.mts +1 -1
  7. package/dist/cli-runtime.mjs +2 -2
  8. package/dist/client-BfdjFtlV.mjs +305 -0
  9. package/dist/{client-C9m0uLCz.d.cts → client-BgRs8JvW.d.mts} +2150 -174
  10. package/dist/{client-TFCLuMv8.d.mts → client-BirFO3jz.d.cts} +2150 -174
  11. package/dist/client-C-P2I00z.cjs +346 -0
  12. package/dist/{compiler-DXlpU_In.mjs → compiler-D96wI7UP.mjs} +108 -75
  13. package/dist/{compiler-Ap8DM-JU.cjs → compiler-uwb1pkdK.cjs} +116 -83
  14. package/dist/config.cjs +1 -1
  15. package/dist/config.d.cts +1 -1
  16. package/dist/config.d.mts +1 -1
  17. package/dist/config.mjs +1 -1
  18. package/dist/{errors-C9bQUcA3.mjs → errors-B0YknR5V.mjs} +3 -19
  19. package/dist/{errors-Boum34f5.cjs → errors-BX1wry8K.cjs} +3 -19
  20. package/dist/{errors-BK3f7Rdr.d.cts → errors-aGrd21jy.d.cts} +0 -3
  21. package/dist/{errors-BK3f7Rdr.d.mts → errors-aGrd21jy.d.mts} +0 -3
  22. package/dist/eval-judges-BkmiVLLq.cjs +59 -0
  23. package/dist/eval-judges-CwTvTq7N.mjs +42 -0
  24. package/dist/eval-judges-DPPUftbh.d.cts +48 -0
  25. package/dist/eval-judges-DPPUftbh.d.mts +48 -0
  26. package/dist/eval-validations-BRdoZrTQ.mjs +113 -0
  27. package/dist/eval-validations-_MlnQ_uv.cjs +148 -0
  28. package/dist/eval.cjs +1 -1
  29. package/dist/eval.d.cts +60 -6
  30. package/dist/eval.d.mts +60 -6
  31. package/dist/eval.mjs +1 -1
  32. package/dist/evals-B6aT3xgH.d.cts +1364 -0
  33. package/dist/evals-B6aT3xgH.d.mts +1364 -0
  34. package/dist/evals-Dcc_x6jW.mjs +582 -0
  35. package/dist/evals-DouMsrsM.cjs +737 -0
  36. package/dist/index.cjs +23 -99
  37. package/dist/index.d.cts +138 -34
  38. package/dist/index.d.mts +138 -34
  39. package/dist/index.mjs +23 -99
  40. package/dist/source.d.cts +1 -1
  41. package/dist/source.d.mts +1 -1
  42. package/dist/{types-oaMF6-E1.d.mts → types-Codjaq-o.d.mts} +146 -45
  43. package/dist/{types-H9L450Iu.d.cts → types-ISsteq95.d.cts} +146 -45
  44. package/dist/validator-runtime.cjs +238 -0
  45. package/dist/validator-runtime.d.cts +9 -0
  46. package/dist/validator-runtime.d.mts +9 -0
  47. package/dist/validator-runtime.mjs +237 -0
  48. package/dist/validators.cjs +10 -0
  49. package/dist/validators.d.cts +8 -0
  50. package/dist/validators.d.mts +8 -0
  51. package/dist/validators.mjs +9 -0
  52. package/package.json +11 -1
  53. package/dist/client-5K3dMI5Q.mjs +0 -849
  54. package/dist/client-Ci0woWLW.cjs +0 -890
  55. package/dist/evals-BEgEmfiO.d.cts +0 -603
  56. package/dist/evals-BEgEmfiO.d.mts +0 -603
  57. package/dist/harness-profile-BMjsN460.mjs +0 -43
  58. package/dist/harness-profile-CDQpPLPd.cjs +0 -78
@@ -1,603 +0,0 @@
1
- import { Schema } from "effect";
2
- //#region ../schema/src/domain/evals.d.ts
3
- declare const EvalProvider: Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>;
4
- type EvalProvider = typeof EvalProvider.Type;
5
- declare const EvalHarness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command"]>;
6
- type EvalHarness = typeof EvalHarness.Type;
7
- declare const EvalTrialStatus: Schema.Literal<["queued", "running", "passed", "failed", "void"]>;
8
- type EvalTrialStatus = typeof EvalTrialStatus.Type;
9
- declare const EvalRunStatus: Schema.Literal<["running", "finished", "failed"]>;
10
- type EvalRunStatus = typeof EvalRunStatus.Type;
11
- declare const EvalSource: Schema.Union<[Schema.Struct<{
12
- kind: Schema.Literal<["empty"]>;
13
- }>, Schema.Struct<{
14
- kind: Schema.Literal<["repo"]>;
15
- ref: Schema.NullOr<typeof Schema.String>;
16
- url: Schema.refine<string, typeof Schema.String>;
17
- }>, Schema.Struct<{
18
- files: Schema.Record$<typeof Schema.String, typeof Schema.String>;
19
- kind: Schema.Literal<["files"]>;
20
- }>]>;
21
- type EvalSource = typeof EvalSource.Type;
22
- declare const EvalTask: Schema.Struct<{
23
- harness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command"]>;
24
- harnessVersion: typeof Schema.String;
25
- model: typeof Schema.String;
26
- profile: Schema.optional<Schema.NullOr<Schema.Struct<{
27
- name: typeof Schema.String;
28
- version: typeof Schema.String;
29
- }>>>;
30
- provider: Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>;
31
- }>;
32
- type EvalTask = typeof EvalTask.Type;
33
- declare const EvalUsage: Schema.Struct<{
34
- cacheReadTokens: typeof Schema.Int;
35
- cacheWriteTokens: typeof Schema.Int;
36
- costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
37
- inputTokens: typeof Schema.Int;
38
- outputTokens: typeof Schema.Int;
39
- totalTokens: typeof Schema.Int;
40
- }>;
41
- type EvalUsage = typeof EvalUsage.Type;
42
- declare const EvalJournalEntry: Schema.Union<[Schema.Struct<{
43
- _tag: Schema.Literal<["command"]>;
44
- command: typeof Schema.String;
45
- exitCode: Schema.NullOr<typeof Schema.Int>;
46
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
47
- output: typeof Schema.String;
48
- startedAtMillis: Schema.NullOr<typeof Schema.Number>;
49
- }>, Schema.Struct<{
50
- _tag: Schema.Literal<["message"]>;
51
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
52
- text: typeof Schema.String;
53
- usage: Schema.optional<Schema.NullOr<Schema.Struct<{
54
- cacheReadTokens: typeof Schema.Int;
55
- cacheWriteTokens: typeof Schema.Int;
56
- costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
57
- inputTokens: typeof Schema.Int;
58
- outputTokens: typeof Schema.Int;
59
- totalTokens: typeof Schema.Int;
60
- }>>>;
61
- }>, Schema.Struct<{
62
- _tag: Schema.Literal<["toolCall"]>;
63
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
64
- name: typeof Schema.String;
65
- startedAtMillis: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
66
- status: Schema.NullOr<typeof Schema.String>;
67
- }>, Schema.Struct<{
68
- _tag: Schema.Literal<["fileChange"]>;
69
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
70
- paths: Schema.Array$<typeof Schema.String>;
71
- }>]>;
72
- type EvalJournalEntry = typeof EvalJournalEntry.Type;
73
- declare const EvalTrial: Schema.Struct<{
74
- commands: typeof Schema.Int;
75
- costs: Schema.NullOr<Schema.Struct<{
76
- allocatedUsd: typeof Schema.Number;
77
- components: Schema.Array$<Schema.Struct<{
78
- classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
79
- component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
80
- detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
81
- explanation: typeof Schema.String;
82
- source: typeof Schema.String;
83
- usd: Schema.NullOr<typeof Schema.Number>;
84
- }>>;
85
- estimatedEquivalentUsd: typeof Schema.Number;
86
- incomplete: typeof Schema.Boolean;
87
- knownActualUsd: typeof Schema.Number;
88
- }>>;
89
- prepared: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
90
- exitCode: typeof Schema.Int;
91
- failedCommands: typeof Schema.Int;
92
- filesChanged: Schema.Array$<typeof Schema.String>;
93
- modelMs: typeof Schema.Int;
94
- ordinal: typeof Schema.Int;
95
- passed: typeof Schema.Boolean;
96
- sandboxId: Schema.NullOr<typeof Schema.String>;
97
- sandboxMs: typeof Schema.Int;
98
- status: Schema.Literal<["queued", "running", "passed", "failed", "void"]>;
99
- timed: typeof Schema.Boolean;
100
- trajectory: Schema.Array$<Schema.Union<[Schema.Struct<{
101
- _tag: Schema.Literal<["command"]>;
102
- command: typeof Schema.String;
103
- exitCode: Schema.NullOr<typeof Schema.Int>;
104
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
105
- output: typeof Schema.String;
106
- startedAtMillis: Schema.NullOr<typeof Schema.Number>;
107
- }>, Schema.Struct<{
108
- _tag: Schema.Literal<["message"]>;
109
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
110
- text: typeof Schema.String;
111
- usage: Schema.optional<Schema.NullOr<Schema.Struct<{
112
- cacheReadTokens: typeof Schema.Int;
113
- cacheWriteTokens: typeof Schema.Int;
114
- costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
115
- inputTokens: typeof Schema.Int;
116
- outputTokens: typeof Schema.Int;
117
- totalTokens: typeof Schema.Int;
118
- }>>>;
119
- }>, Schema.Struct<{
120
- _tag: Schema.Literal<["toolCall"]>;
121
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
122
- name: typeof Schema.String;
123
- startedAtMillis: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
124
- status: Schema.NullOr<typeof Schema.String>;
125
- }>, Schema.Struct<{
126
- _tag: Schema.Literal<["fileChange"]>;
127
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
128
- paths: Schema.Array$<typeof Schema.String>;
129
- }>]>>;
130
- usage: Schema.NullOr<Schema.Struct<{
131
- cacheReadTokens: typeof Schema.Int;
132
- cacheWriteTokens: typeof Schema.Int;
133
- costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
134
- inputTokens: typeof Schema.Int;
135
- outputTokens: typeof Schema.Int;
136
- totalTokens: typeof Schema.Int;
137
- }>>;
138
- verifySteps: Schema.Array$<Schema.Struct<{
139
- command: typeof Schema.String;
140
- exitCode: typeof Schema.Int;
141
- }>>;
142
- voidFields: Schema.Array$<typeof Schema.String>;
143
- }>;
144
- type EvalTrial = typeof EvalTrial.Type;
145
- declare const EvalDistribution: Schema.Struct<{
146
- commandMax: typeof Schema.Int;
147
- commandMedian: typeof Schema.Number;
148
- commandMin: typeof Schema.Int;
149
- deterministic: typeof Schema.Boolean;
150
- failed: typeof Schema.Int;
151
- passRate: typeof Schema.Number;
152
- passed: typeof Schema.Int;
153
- scored: typeof Schema.Int;
154
- trials: typeof Schema.Int;
155
- voided: typeof Schema.Int;
156
- }>;
157
- type EvalDistribution = typeof EvalDistribution.Type;
158
- declare const EvalVerdict: Schema.Literal<["improved", "incomparable", "regressed", "unchanged"]>;
159
- type EvalVerdict = typeof EvalVerdict.Type;
160
- declare const EvalComparison: Schema.Struct<{
161
- baselineHarnessVersion: typeof Schema.String;
162
- baselinePassRate: typeof Schema.Number;
163
- baselineProfileVersion: Schema.NullOr<typeof Schema.String>;
164
- candidateHarnessVersion: typeof Schema.String;
165
- candidatePassRate: typeof Schema.Number;
166
- candidateProfileVersion: Schema.NullOr<typeof Schema.String>;
167
- delta: typeof Schema.Number;
168
- determinismLost: typeof Schema.Boolean;
169
- reason: Schema.NullOr<typeof Schema.String>;
170
- verdict: Schema.Literal<["improved", "incomparable", "regressed", "unchanged"]>;
171
- }>;
172
- type EvalComparison = typeof EvalComparison.Type;
173
- declare const EvalCell: Schema.Struct<{
174
- caseName: typeof Schema.String;
175
- cellKey: Schema.NullOr<typeof Schema.String>;
176
- costs: Schema.NullOr<Schema.Struct<{
177
- allocatedUsd: typeof Schema.Number;
178
- components: Schema.Array$<Schema.Struct<{
179
- classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
180
- component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
181
- detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
182
- explanation: typeof Schema.String;
183
- source: typeof Schema.String;
184
- usd: Schema.NullOr<typeof Schema.Number>;
185
- }>>;
186
- estimatedEquivalentUsd: typeof Schema.Number;
187
- incomplete: typeof Schema.Boolean;
188
- knownActualUsd: typeof Schema.Number;
189
- }>>;
190
- comparison: Schema.NullOr<Schema.Struct<{
191
- baselineHarnessVersion: typeof Schema.String;
192
- baselinePassRate: typeof Schema.Number;
193
- baselineProfileVersion: Schema.NullOr<typeof Schema.String>;
194
- candidateHarnessVersion: typeof Schema.String;
195
- candidatePassRate: typeof Schema.Number;
196
- candidateProfileVersion: Schema.NullOr<typeof Schema.String>;
197
- delta: typeof Schema.Number;
198
- determinismLost: typeof Schema.Boolean;
199
- reason: Schema.NullOr<typeof Schema.String>;
200
- verdict: Schema.Literal<["improved", "incomparable", "regressed", "unchanged"]>;
201
- }>>;
202
- distribution: Schema.NullOr<Schema.Struct<{
203
- commandMax: typeof Schema.Int;
204
- commandMedian: typeof Schema.Number;
205
- commandMin: typeof Schema.Int;
206
- deterministic: typeof Schema.Boolean;
207
- failed: typeof Schema.Int;
208
- passRate: typeof Schema.Number;
209
- passed: typeof Schema.Int;
210
- scored: typeof Schema.Int;
211
- trials: typeof Schema.Int;
212
- voided: typeof Schema.Int;
213
- }>>;
214
- internalId: Schema.NullOr<typeof Schema.String>;
215
- setup: Schema.NullOr<Schema.Struct<{
216
- prompt: typeof Schema.String;
217
- repoRef: Schema.NullOr<typeof Schema.String>;
218
- repoUrl: Schema.NullOr<typeof Schema.String>;
219
- prepareName: Schema.NullOr<typeof Schema.String>;
220
- validatorName: Schema.NullOr<typeof Schema.String>;
221
- verifyCommand: Schema.NullOr<typeof Schema.String>;
222
- workspace: typeof Schema.String;
223
- }>>;
224
- status: Schema.Literal<["running", "finished", "failed"]>;
225
- taskIndex: typeof Schema.Int;
226
- trials: Schema.Array$<Schema.Struct<{
227
- commands: typeof Schema.Int;
228
- costs: Schema.NullOr<Schema.Struct<{
229
- allocatedUsd: typeof Schema.Number;
230
- components: Schema.Array$<Schema.Struct<{
231
- classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
232
- component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
233
- detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
234
- explanation: typeof Schema.String;
235
- source: typeof Schema.String;
236
- usd: Schema.NullOr<typeof Schema.Number>;
237
- }>>;
238
- estimatedEquivalentUsd: typeof Schema.Number;
239
- incomplete: typeof Schema.Boolean;
240
- knownActualUsd: typeof Schema.Number;
241
- }>>;
242
- prepared: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
243
- exitCode: typeof Schema.Int;
244
- failedCommands: typeof Schema.Int;
245
- filesChanged: Schema.Array$<typeof Schema.String>;
246
- modelMs: typeof Schema.Int;
247
- ordinal: typeof Schema.Int;
248
- passed: typeof Schema.Boolean;
249
- sandboxId: Schema.NullOr<typeof Schema.String>;
250
- sandboxMs: typeof Schema.Int;
251
- status: Schema.Literal<["queued", "running", "passed", "failed", "void"]>;
252
- timed: typeof Schema.Boolean;
253
- trajectory: Schema.Array$<Schema.Union<[Schema.Struct<{
254
- _tag: Schema.Literal<["command"]>;
255
- command: typeof Schema.String;
256
- exitCode: Schema.NullOr<typeof Schema.Int>;
257
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
258
- output: typeof Schema.String;
259
- startedAtMillis: Schema.NullOr<typeof Schema.Number>;
260
- }>, Schema.Struct<{
261
- _tag: Schema.Literal<["message"]>;
262
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
263
- text: typeof Schema.String;
264
- usage: Schema.optional<Schema.NullOr<Schema.Struct<{
265
- cacheReadTokens: typeof Schema.Int;
266
- cacheWriteTokens: typeof Schema.Int;
267
- costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
268
- inputTokens: typeof Schema.Int;
269
- outputTokens: typeof Schema.Int;
270
- totalTokens: typeof Schema.Int;
271
- }>>>;
272
- }>, Schema.Struct<{
273
- _tag: Schema.Literal<["toolCall"]>;
274
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
275
- name: typeof Schema.String;
276
- startedAtMillis: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
277
- status: Schema.NullOr<typeof Schema.String>;
278
- }>, Schema.Struct<{
279
- _tag: Schema.Literal<["fileChange"]>;
280
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
281
- paths: Schema.Array$<typeof Schema.String>;
282
- }>]>>;
283
- usage: Schema.NullOr<Schema.Struct<{
284
- cacheReadTokens: typeof Schema.Int;
285
- cacheWriteTokens: typeof Schema.Int;
286
- costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
287
- inputTokens: typeof Schema.Int;
288
- outputTokens: typeof Schema.Int;
289
- totalTokens: typeof Schema.Int;
290
- }>>;
291
- verifySteps: Schema.Array$<Schema.Struct<{
292
- command: typeof Schema.String;
293
- exitCode: typeof Schema.Int;
294
- }>>;
295
- voidFields: Schema.Array$<typeof Schema.String>;
296
- }>>;
297
- }>;
298
- type EvalCell = typeof EvalCell.Type;
299
- declare const EvalRun: Schema.Struct<{
300
- cases: Schema.Array$<typeof Schema.String>;
301
- cells: Schema.Array$<Schema.Struct<{
302
- caseName: typeof Schema.String;
303
- cellKey: Schema.NullOr<typeof Schema.String>;
304
- costs: Schema.NullOr<Schema.Struct<{
305
- allocatedUsd: typeof Schema.Number;
306
- components: Schema.Array$<Schema.Struct<{
307
- classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
308
- component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
309
- detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
310
- explanation: typeof Schema.String;
311
- source: typeof Schema.String;
312
- usd: Schema.NullOr<typeof Schema.Number>;
313
- }>>;
314
- estimatedEquivalentUsd: typeof Schema.Number;
315
- incomplete: typeof Schema.Boolean;
316
- knownActualUsd: typeof Schema.Number;
317
- }>>;
318
- comparison: Schema.NullOr<Schema.Struct<{
319
- baselineHarnessVersion: typeof Schema.String;
320
- baselinePassRate: typeof Schema.Number;
321
- baselineProfileVersion: Schema.NullOr<typeof Schema.String>;
322
- candidateHarnessVersion: typeof Schema.String;
323
- candidatePassRate: typeof Schema.Number;
324
- candidateProfileVersion: Schema.NullOr<typeof Schema.String>;
325
- delta: typeof Schema.Number;
326
- determinismLost: typeof Schema.Boolean;
327
- reason: Schema.NullOr<typeof Schema.String>;
328
- verdict: Schema.Literal<["improved", "incomparable", "regressed", "unchanged"]>;
329
- }>>;
330
- distribution: Schema.NullOr<Schema.Struct<{
331
- commandMax: typeof Schema.Int;
332
- commandMedian: typeof Schema.Number;
333
- commandMin: typeof Schema.Int;
334
- deterministic: typeof Schema.Boolean;
335
- failed: typeof Schema.Int;
336
- passRate: typeof Schema.Number;
337
- passed: typeof Schema.Int;
338
- scored: typeof Schema.Int;
339
- trials: typeof Schema.Int;
340
- voided: typeof Schema.Int;
341
- }>>;
342
- internalId: Schema.NullOr<typeof Schema.String>;
343
- setup: Schema.NullOr<Schema.Struct<{
344
- prompt: typeof Schema.String;
345
- repoRef: Schema.NullOr<typeof Schema.String>;
346
- repoUrl: Schema.NullOr<typeof Schema.String>;
347
- prepareName: Schema.NullOr<typeof Schema.String>;
348
- validatorName: Schema.NullOr<typeof Schema.String>;
349
- verifyCommand: Schema.NullOr<typeof Schema.String>;
350
- workspace: typeof Schema.String;
351
- }>>;
352
- status: Schema.Literal<["running", "finished", "failed"]>;
353
- taskIndex: typeof Schema.Int;
354
- trials: Schema.Array$<Schema.Struct<{
355
- commands: typeof Schema.Int;
356
- costs: Schema.NullOr<Schema.Struct<{
357
- allocatedUsd: typeof Schema.Number;
358
- components: Schema.Array$<Schema.Struct<{
359
- classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
360
- component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
361
- detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
362
- explanation: typeof Schema.String;
363
- source: typeof Schema.String;
364
- usd: Schema.NullOr<typeof Schema.Number>;
365
- }>>;
366
- estimatedEquivalentUsd: typeof Schema.Number;
367
- incomplete: typeof Schema.Boolean;
368
- knownActualUsd: typeof Schema.Number;
369
- }>>;
370
- prepared: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
371
- exitCode: typeof Schema.Int;
372
- failedCommands: typeof Schema.Int;
373
- filesChanged: Schema.Array$<typeof Schema.String>;
374
- modelMs: typeof Schema.Int;
375
- ordinal: typeof Schema.Int;
376
- passed: typeof Schema.Boolean;
377
- sandboxId: Schema.NullOr<typeof Schema.String>;
378
- sandboxMs: typeof Schema.Int;
379
- status: Schema.Literal<["queued", "running", "passed", "failed", "void"]>;
380
- timed: typeof Schema.Boolean;
381
- trajectory: Schema.Array$<Schema.Union<[Schema.Struct<{
382
- _tag: Schema.Literal<["command"]>;
383
- command: typeof Schema.String;
384
- exitCode: Schema.NullOr<typeof Schema.Int>;
385
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
386
- output: typeof Schema.String;
387
- startedAtMillis: Schema.NullOr<typeof Schema.Number>;
388
- }>, Schema.Struct<{
389
- _tag: Schema.Literal<["message"]>;
390
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
391
- text: typeof Schema.String;
392
- usage: Schema.optional<Schema.NullOr<Schema.Struct<{
393
- cacheReadTokens: typeof Schema.Int;
394
- cacheWriteTokens: typeof Schema.Int;
395
- costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
396
- inputTokens: typeof Schema.Int;
397
- outputTokens: typeof Schema.Int;
398
- totalTokens: typeof Schema.Int;
399
- }>>>;
400
- }>, Schema.Struct<{
401
- _tag: Schema.Literal<["toolCall"]>;
402
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
403
- name: typeof Schema.String;
404
- startedAtMillis: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
405
- status: Schema.NullOr<typeof Schema.String>;
406
- }>, Schema.Struct<{
407
- _tag: Schema.Literal<["fileChange"]>;
408
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
409
- paths: Schema.Array$<typeof Schema.String>;
410
- }>]>>;
411
- usage: Schema.NullOr<Schema.Struct<{
412
- cacheReadTokens: typeof Schema.Int;
413
- cacheWriteTokens: typeof Schema.Int;
414
- costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
415
- inputTokens: typeof Schema.Int;
416
- outputTokens: typeof Schema.Int;
417
- totalTokens: typeof Schema.Int;
418
- }>>;
419
- verifySteps: Schema.Array$<Schema.Struct<{
420
- command: typeof Schema.String;
421
- exitCode: typeof Schema.Int;
422
- }>>;
423
- voidFields: Schema.Array$<typeof Schema.String>;
424
- }>>;
425
- }>>;
426
- costs: Schema.NullOr<Schema.Struct<{
427
- allocatedUsd: typeof Schema.Number;
428
- components: Schema.Array$<Schema.Struct<{
429
- classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
430
- component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
431
- detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
432
- explanation: typeof Schema.String;
433
- source: typeof Schema.String;
434
- usd: Schema.NullOr<typeof Schema.Number>;
435
- }>>;
436
- estimatedEquivalentUsd: typeof Schema.Number;
437
- incomplete: typeof Schema.Boolean;
438
- knownActualUsd: typeof Schema.Number;
439
- }>>;
440
- failure: Schema.NullOr<typeof Schema.String>;
441
- finishedAt: Schema.NullOr<Schema.transformOrFail<Schema.SchemaClass<string, string, never>, typeof Schema.DateTimeUtcFromSelf, never>>;
442
- id: typeof Schema.String;
443
- name: Schema.NullOr<Schema.filter<Schema.filter<typeof Schema.String>>>;
444
- startedAt: Schema.transformOrFail<Schema.SchemaClass<string, string, never>, typeof Schema.DateTimeUtcFromSelf, never>;
445
- status: Schema.Literal<["running", "finished", "failed"]>;
446
- tasks: Schema.Array$<Schema.Struct<{
447
- harness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command"]>;
448
- harnessVersion: typeof Schema.String;
449
- model: typeof Schema.String;
450
- profile: Schema.optional<Schema.NullOr<Schema.Struct<{
451
- name: typeof Schema.String;
452
- version: typeof Schema.String;
453
- }>>>;
454
- provider: Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>;
455
- }>>;
456
- }>;
457
- type EvalRun = typeof EvalRun.Type;
458
- declare const EvalRunSummary: Schema.Struct<{
459
- caseCount: typeof Schema.Int;
460
- columns: Schema.Array$<Schema.Struct<{
461
- harness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command"]>;
462
- harnessVersion: typeof Schema.String;
463
- model: typeof Schema.String;
464
- profile: Schema.optional<Schema.NullOr<Schema.Struct<{
465
- name: typeof Schema.String;
466
- version: typeof Schema.String;
467
- }>>>;
468
- provider: Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>;
469
- }>>;
470
- commandMax: Schema.NullOr<typeof Schema.Int>;
471
- commandMin: Schema.NullOr<typeof Schema.Int>;
472
- failure: Schema.NullOr<typeof Schema.String>;
473
- finishedAt: Schema.NullOr<Schema.transformOrFail<Schema.SchemaClass<string, string, never>, typeof Schema.DateTimeUtcFromSelf, never>>;
474
- firstCaseName: Schema.NullOr<typeof Schema.String>;
475
- id: typeof Schema.String;
476
- name: Schema.NullOr<typeof Schema.String>;
477
- passed: typeof Schema.Int;
478
- scored: typeof Schema.Int;
479
- startedAt: Schema.transformOrFail<Schema.SchemaClass<string, string, never>, typeof Schema.DateTimeUtcFromSelf, never>;
480
- status: Schema.Literal<["running", "finished", "failed"]>;
481
- taskCount: typeof Schema.Int;
482
- voided: typeof Schema.Int;
483
- }>;
484
- type EvalRunSummary = typeof EvalRunSummary.Type;
485
- declare const EvalCellHistoryEntry: Schema.Struct<{
486
- distribution: Schema.Struct<{
487
- commandMax: typeof Schema.Int;
488
- commandMedian: typeof Schema.Number;
489
- commandMin: typeof Schema.Int;
490
- deterministic: typeof Schema.Boolean;
491
- failed: typeof Schema.Int;
492
- passRate: typeof Schema.Number;
493
- passed: typeof Schema.Int;
494
- scored: typeof Schema.Int;
495
- trials: typeof Schema.Int;
496
- voided: typeof Schema.Int;
497
- }>;
498
- finishedAt: Schema.NullOr<Schema.transformOrFail<Schema.SchemaClass<string, string, never>, typeof Schema.DateTimeUtcFromSelf, never>>;
499
- harnessVersion: typeof Schema.String;
500
- internalId: typeof Schema.String;
501
- profileVersion: Schema.NullOr<typeof Schema.String>;
502
- runId: typeof Schema.String;
503
- trials: Schema.Array$<Schema.Struct<{
504
- commands: typeof Schema.Int;
505
- costs: Schema.NullOr<Schema.Struct<{
506
- allocatedUsd: typeof Schema.Number;
507
- components: Schema.Array$<Schema.Struct<{
508
- classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
509
- component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
510
- detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
511
- explanation: typeof Schema.String;
512
- source: typeof Schema.String;
513
- usd: Schema.NullOr<typeof Schema.Number>;
514
- }>>;
515
- estimatedEquivalentUsd: typeof Schema.Number;
516
- incomplete: typeof Schema.Boolean;
517
- knownActualUsd: typeof Schema.Number;
518
- }>>;
519
- prepared: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
520
- exitCode: typeof Schema.Int;
521
- failedCommands: typeof Schema.Int;
522
- filesChanged: Schema.Array$<typeof Schema.String>;
523
- modelMs: typeof Schema.Int;
524
- ordinal: typeof Schema.Int;
525
- passed: typeof Schema.Boolean;
526
- sandboxId: Schema.NullOr<typeof Schema.String>;
527
- sandboxMs: typeof Schema.Int;
528
- status: Schema.Literal<["queued", "running", "passed", "failed", "void"]>;
529
- timed: typeof Schema.Boolean;
530
- trajectory: Schema.Array$<Schema.Union<[Schema.Struct<{
531
- _tag: Schema.Literal<["command"]>;
532
- command: typeof Schema.String;
533
- exitCode: Schema.NullOr<typeof Schema.Int>;
534
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
535
- output: typeof Schema.String;
536
- startedAtMillis: Schema.NullOr<typeof Schema.Number>;
537
- }>, Schema.Struct<{
538
- _tag: Schema.Literal<["message"]>;
539
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
540
- text: typeof Schema.String;
541
- usage: Schema.optional<Schema.NullOr<Schema.Struct<{
542
- cacheReadTokens: typeof Schema.Int;
543
- cacheWriteTokens: typeof Schema.Int;
544
- costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
545
- inputTokens: typeof Schema.Int;
546
- outputTokens: typeof Schema.Int;
547
- totalTokens: typeof Schema.Int;
548
- }>>>;
549
- }>, Schema.Struct<{
550
- _tag: Schema.Literal<["toolCall"]>;
551
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
552
- name: typeof Schema.String;
553
- startedAtMillis: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
554
- status: Schema.NullOr<typeof Schema.String>;
555
- }>, Schema.Struct<{
556
- _tag: Schema.Literal<["fileChange"]>;
557
- finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
558
- paths: Schema.Array$<typeof Schema.String>;
559
- }>]>>;
560
- usage: Schema.NullOr<Schema.Struct<{
561
- cacheReadTokens: typeof Schema.Int;
562
- cacheWriteTokens: typeof Schema.Int;
563
- costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
564
- inputTokens: typeof Schema.Int;
565
- outputTokens: typeof Schema.Int;
566
- totalTokens: typeof Schema.Int;
567
- }>>;
568
- verifySteps: Schema.Array$<Schema.Struct<{
569
- command: typeof Schema.String;
570
- exitCode: typeof Schema.Int;
571
- }>>;
572
- voidFields: Schema.Array$<typeof Schema.String>;
573
- }>>;
574
- }>;
575
- type EvalCellHistoryEntry = typeof EvalCellHistoryEntry.Type;
576
- declare const StartedEval: Schema.Struct<{
577
- id: typeof Schema.String;
578
- }>;
579
- declare const RerunCellRequest: Schema.Struct<{
580
- trials: Schema.filter<typeof Schema.Int>;
581
- }>;
582
- type RerunCellRequest = typeof RerunCellRequest.Type;
583
- type StartedEval = typeof StartedEval.Type;
584
- declare const CatalogueModel: Schema.Struct<{
585
- displayName: typeof Schema.String;
586
- id: typeof Schema.String;
587
- summary: Schema.NullOr<typeof Schema.String>;
588
- vendor: Schema.NullOr<typeof Schema.String>;
589
- }>;
590
- type CatalogueModel = typeof CatalogueModel.Type;
591
- declare const ModelCatalogue: Schema.Struct<{
592
- harness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command"]>;
593
- models: Schema.Array$<Schema.Struct<{
594
- displayName: typeof Schema.String;
595
- id: typeof Schema.String;
596
- summary: Schema.NullOr<typeof Schema.String>;
597
- vendor: Schema.NullOr<typeof Schema.String>;
598
- }>>;
599
- total: typeof Schema.Int;
600
- }>;
601
- type ModelCatalogue = typeof ModelCatalogue.Type;
602
- //#endregion
603
- export { EvalVerdict as _, EvalDistribution as a, StartedEval as b, EvalProvider as c, EvalRunSummary as d, EvalSource as f, EvalUsage as g, EvalTrialStatus as h, EvalComparison as i, EvalRun as l, EvalTrial as m, EvalCell as n, EvalHarness as o, EvalTask as p, EvalCellHistoryEntry as r, EvalJournalEntry as s, CatalogueModel as t, EvalRunStatus as u, ModelCatalogue as v, RerunCellRequest as y };
@@ -1,43 +0,0 @@
1
- import { Schema } from "effect";
2
- //#region ../schema/src/domain/harness-profile.ts
3
- const PROFILE_LIMITS = {
4
- fileChars: 96e3,
5
- files: 256,
6
- totalChars: 2e6
7
- };
8
- const ProfileName = Schema.String.pipe(Schema.pattern(/^[a-z0-9][a-z0-9-]{0,63}$/), Schema.annotations({
9
- description: "Lowercase letters, digits and hyphens, at most 64 characters, starting with a letter or digit.",
10
- identifier: "ProfileName"
11
- }));
12
- const ProfilePath = Schema.String.pipe(Schema.pattern(/^(home|workspace)(?:\/(?!\.\.?(?:\/|$))[^/\0]+)+$/), Schema.maxLength(512), Schema.annotations({
13
- description: "A relative path under home/ or workspace/, without a .. segment.",
14
- identifier: "ProfilePath"
15
- }));
16
- const EnvName = Schema.String.pipe(Schema.pattern(/^[A-Z_][A-Z0-9_]*$/), Schema.annotations({
17
- description: "An environment variable name.",
18
- identifier: "EnvName"
19
- }));
20
- const ProfileFiles = Schema.Record({
21
- key: ProfilePath,
22
- value: Schema.String.pipe(Schema.maxLength(PROFILE_LIMITS.fileChars))
23
- }).pipe(Schema.filter((files) => Object.keys(files).length <= PROFILE_LIMITS.files, { message: () => `A profile holds at most ${PROFILE_LIMITS.files} files.` }), Schema.filter((files) => Object.values(files).reduce((sum, content) => sum + content.length, 0) <= PROFILE_LIMITS.totalChars, { message: () => `A profile's files hold at most ${PROFILE_LIMITS.totalChars} characters in total.` }));
24
- const HarnessProfile = Schema.Struct({
25
- env: Schema.optional(Schema.Record({
26
- key: EnvName,
27
- value: Schema.String
28
- })),
29
- files: ProfileFiles,
30
- install: Schema.optional(Schema.String),
31
- name: ProfileName,
32
- run: Schema.optional(Schema.String),
33
- systemPrompt: Schema.optional(Schema.String)
34
- }).annotations({
35
- description: "Configuration layered on a harness: files written under the sandbox home and workspace, a system prompt, environment, an install command run before the harness, and for the command harness the run command.",
36
- identifier: "HarnessProfile"
37
- });
38
- /** The one rule the wire schema cannot express in JSON Schema, so every tool
39
- * description repeats it. */
40
- const profileFitsHarness = (task) => task.harness === "command" ? task.profile?.run !== void 0 : task.profile?.run === void 0;
41
- const PROFILE_HARNESS_RULE = "The command harness needs a profile with a run command; other harnesses take no run. Any harness may take an install command.";
42
- //#endregion
43
- export { ProfilePath as a, PROFILE_LIMITS as i, HarnessProfile as n, profileFitsHarness as o, PROFILE_HARNESS_RULE as r, EnvName as t };