anpord 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -78
- package/dist/bin.cjs +211 -204
- package/dist/bin.mjs +211 -204
- package/dist/cli-runtime.cjs +2 -2
- package/dist/cli-runtime.d.cts +1 -1
- package/dist/cli-runtime.d.mts +1 -1
- package/dist/cli-runtime.mjs +2 -2
- package/dist/client-BfdjFtlV.mjs +305 -0
- package/dist/{client-C9m0uLCz.d.cts → client-BgRs8JvW.d.mts} +2150 -174
- package/dist/{client-TFCLuMv8.d.mts → client-BirFO3jz.d.cts} +2150 -174
- package/dist/client-C-P2I00z.cjs +346 -0
- package/dist/{compiler-DXlpU_In.mjs → compiler-D96wI7UP.mjs} +108 -75
- package/dist/{compiler-Ap8DM-JU.cjs → compiler-uwb1pkdK.cjs} +116 -83
- package/dist/config.cjs +1 -1
- package/dist/config.d.cts +1 -1
- package/dist/config.d.mts +1 -1
- package/dist/config.mjs +1 -1
- package/dist/{errors-C9bQUcA3.mjs → errors-B0YknR5V.mjs} +3 -19
- package/dist/{errors-Boum34f5.cjs → errors-BX1wry8K.cjs} +3 -19
- package/dist/{errors-BK3f7Rdr.d.cts → errors-aGrd21jy.d.cts} +0 -3
- package/dist/{errors-BK3f7Rdr.d.mts → errors-aGrd21jy.d.mts} +0 -3
- package/dist/eval-judges-BkmiVLLq.cjs +59 -0
- package/dist/eval-judges-CwTvTq7N.mjs +42 -0
- package/dist/eval-judges-DPPUftbh.d.cts +48 -0
- package/dist/eval-judges-DPPUftbh.d.mts +48 -0
- package/dist/eval-validations-BRdoZrTQ.mjs +113 -0
- package/dist/eval-validations-_MlnQ_uv.cjs +148 -0
- package/dist/eval.cjs +1 -1
- package/dist/eval.d.cts +60 -6
- package/dist/eval.d.mts +60 -6
- package/dist/eval.mjs +1 -1
- package/dist/evals-B6aT3xgH.d.cts +1364 -0
- package/dist/evals-B6aT3xgH.d.mts +1364 -0
- package/dist/evals-Dcc_x6jW.mjs +582 -0
- package/dist/evals-DouMsrsM.cjs +737 -0
- package/dist/index.cjs +23 -99
- package/dist/index.d.cts +138 -34
- package/dist/index.d.mts +138 -34
- package/dist/index.mjs +23 -99
- package/dist/source.d.cts +1 -1
- package/dist/source.d.mts +1 -1
- package/dist/{types-oaMF6-E1.d.mts → types-Codjaq-o.d.mts} +146 -45
- package/dist/{types-H9L450Iu.d.cts → types-ISsteq95.d.cts} +146 -45
- package/dist/validator-runtime.cjs +238 -0
- package/dist/validator-runtime.d.cts +9 -0
- package/dist/validator-runtime.d.mts +9 -0
- package/dist/validator-runtime.mjs +237 -0
- package/dist/validators.cjs +10 -0
- package/dist/validators.d.cts +8 -0
- package/dist/validators.d.mts +8 -0
- package/dist/validators.mjs +9 -0
- package/package.json +11 -1
- package/dist/client-5K3dMI5Q.mjs +0 -849
- package/dist/client-Ci0woWLW.cjs +0 -890
- package/dist/evals-BEgEmfiO.d.cts +0 -603
- package/dist/evals-BEgEmfiO.d.mts +0 -603
- package/dist/harness-profile-BMjsN460.mjs +0 -43
- package/dist/harness-profile-CDQpPLPd.cjs +0 -78
|
@@ -1,603 +0,0 @@
|
|
|
1
|
-
import { Schema } from "effect";
|
|
2
|
-
//#region ../schema/src/domain/evals.d.ts
|
|
3
|
-
declare const EvalProvider: Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>;
|
|
4
|
-
type EvalProvider = typeof EvalProvider.Type;
|
|
5
|
-
declare const EvalHarness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command"]>;
|
|
6
|
-
type EvalHarness = typeof EvalHarness.Type;
|
|
7
|
-
declare const EvalTrialStatus: Schema.Literal<["queued", "running", "passed", "failed", "void"]>;
|
|
8
|
-
type EvalTrialStatus = typeof EvalTrialStatus.Type;
|
|
9
|
-
declare const EvalRunStatus: Schema.Literal<["running", "finished", "failed"]>;
|
|
10
|
-
type EvalRunStatus = typeof EvalRunStatus.Type;
|
|
11
|
-
declare const EvalSource: Schema.Union<[Schema.Struct<{
|
|
12
|
-
kind: Schema.Literal<["empty"]>;
|
|
13
|
-
}>, Schema.Struct<{
|
|
14
|
-
kind: Schema.Literal<["repo"]>;
|
|
15
|
-
ref: Schema.NullOr<typeof Schema.String>;
|
|
16
|
-
url: Schema.refine<string, typeof Schema.String>;
|
|
17
|
-
}>, Schema.Struct<{
|
|
18
|
-
files: Schema.Record$<typeof Schema.String, typeof Schema.String>;
|
|
19
|
-
kind: Schema.Literal<["files"]>;
|
|
20
|
-
}>]>;
|
|
21
|
-
type EvalSource = typeof EvalSource.Type;
|
|
22
|
-
declare const EvalTask: Schema.Struct<{
|
|
23
|
-
harness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command"]>;
|
|
24
|
-
harnessVersion: typeof Schema.String;
|
|
25
|
-
model: typeof Schema.String;
|
|
26
|
-
profile: Schema.optional<Schema.NullOr<Schema.Struct<{
|
|
27
|
-
name: typeof Schema.String;
|
|
28
|
-
version: typeof Schema.String;
|
|
29
|
-
}>>>;
|
|
30
|
-
provider: Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>;
|
|
31
|
-
}>;
|
|
32
|
-
type EvalTask = typeof EvalTask.Type;
|
|
33
|
-
declare const EvalUsage: Schema.Struct<{
|
|
34
|
-
cacheReadTokens: typeof Schema.Int;
|
|
35
|
-
cacheWriteTokens: typeof Schema.Int;
|
|
36
|
-
costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
37
|
-
inputTokens: typeof Schema.Int;
|
|
38
|
-
outputTokens: typeof Schema.Int;
|
|
39
|
-
totalTokens: typeof Schema.Int;
|
|
40
|
-
}>;
|
|
41
|
-
type EvalUsage = typeof EvalUsage.Type;
|
|
42
|
-
declare const EvalJournalEntry: Schema.Union<[Schema.Struct<{
|
|
43
|
-
_tag: Schema.Literal<["command"]>;
|
|
44
|
-
command: typeof Schema.String;
|
|
45
|
-
exitCode: Schema.NullOr<typeof Schema.Int>;
|
|
46
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
47
|
-
output: typeof Schema.String;
|
|
48
|
-
startedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
49
|
-
}>, Schema.Struct<{
|
|
50
|
-
_tag: Schema.Literal<["message"]>;
|
|
51
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
52
|
-
text: typeof Schema.String;
|
|
53
|
-
usage: Schema.optional<Schema.NullOr<Schema.Struct<{
|
|
54
|
-
cacheReadTokens: typeof Schema.Int;
|
|
55
|
-
cacheWriteTokens: typeof Schema.Int;
|
|
56
|
-
costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
57
|
-
inputTokens: typeof Schema.Int;
|
|
58
|
-
outputTokens: typeof Schema.Int;
|
|
59
|
-
totalTokens: typeof Schema.Int;
|
|
60
|
-
}>>>;
|
|
61
|
-
}>, Schema.Struct<{
|
|
62
|
-
_tag: Schema.Literal<["toolCall"]>;
|
|
63
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
64
|
-
name: typeof Schema.String;
|
|
65
|
-
startedAtMillis: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
66
|
-
status: Schema.NullOr<typeof Schema.String>;
|
|
67
|
-
}>, Schema.Struct<{
|
|
68
|
-
_tag: Schema.Literal<["fileChange"]>;
|
|
69
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
70
|
-
paths: Schema.Array$<typeof Schema.String>;
|
|
71
|
-
}>]>;
|
|
72
|
-
type EvalJournalEntry = typeof EvalJournalEntry.Type;
|
|
73
|
-
declare const EvalTrial: Schema.Struct<{
|
|
74
|
-
commands: typeof Schema.Int;
|
|
75
|
-
costs: Schema.NullOr<Schema.Struct<{
|
|
76
|
-
allocatedUsd: typeof Schema.Number;
|
|
77
|
-
components: Schema.Array$<Schema.Struct<{
|
|
78
|
-
classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
|
|
79
|
-
component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
|
|
80
|
-
detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
|
|
81
|
-
explanation: typeof Schema.String;
|
|
82
|
-
source: typeof Schema.String;
|
|
83
|
-
usd: Schema.NullOr<typeof Schema.Number>;
|
|
84
|
-
}>>;
|
|
85
|
-
estimatedEquivalentUsd: typeof Schema.Number;
|
|
86
|
-
incomplete: typeof Schema.Boolean;
|
|
87
|
-
knownActualUsd: typeof Schema.Number;
|
|
88
|
-
}>>;
|
|
89
|
-
prepared: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
|
|
90
|
-
exitCode: typeof Schema.Int;
|
|
91
|
-
failedCommands: typeof Schema.Int;
|
|
92
|
-
filesChanged: Schema.Array$<typeof Schema.String>;
|
|
93
|
-
modelMs: typeof Schema.Int;
|
|
94
|
-
ordinal: typeof Schema.Int;
|
|
95
|
-
passed: typeof Schema.Boolean;
|
|
96
|
-
sandboxId: Schema.NullOr<typeof Schema.String>;
|
|
97
|
-
sandboxMs: typeof Schema.Int;
|
|
98
|
-
status: Schema.Literal<["queued", "running", "passed", "failed", "void"]>;
|
|
99
|
-
timed: typeof Schema.Boolean;
|
|
100
|
-
trajectory: Schema.Array$<Schema.Union<[Schema.Struct<{
|
|
101
|
-
_tag: Schema.Literal<["command"]>;
|
|
102
|
-
command: typeof Schema.String;
|
|
103
|
-
exitCode: Schema.NullOr<typeof Schema.Int>;
|
|
104
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
105
|
-
output: typeof Schema.String;
|
|
106
|
-
startedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
107
|
-
}>, Schema.Struct<{
|
|
108
|
-
_tag: Schema.Literal<["message"]>;
|
|
109
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
110
|
-
text: typeof Schema.String;
|
|
111
|
-
usage: Schema.optional<Schema.NullOr<Schema.Struct<{
|
|
112
|
-
cacheReadTokens: typeof Schema.Int;
|
|
113
|
-
cacheWriteTokens: typeof Schema.Int;
|
|
114
|
-
costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
115
|
-
inputTokens: typeof Schema.Int;
|
|
116
|
-
outputTokens: typeof Schema.Int;
|
|
117
|
-
totalTokens: typeof Schema.Int;
|
|
118
|
-
}>>>;
|
|
119
|
-
}>, Schema.Struct<{
|
|
120
|
-
_tag: Schema.Literal<["toolCall"]>;
|
|
121
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
122
|
-
name: typeof Schema.String;
|
|
123
|
-
startedAtMillis: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
124
|
-
status: Schema.NullOr<typeof Schema.String>;
|
|
125
|
-
}>, Schema.Struct<{
|
|
126
|
-
_tag: Schema.Literal<["fileChange"]>;
|
|
127
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
128
|
-
paths: Schema.Array$<typeof Schema.String>;
|
|
129
|
-
}>]>>;
|
|
130
|
-
usage: Schema.NullOr<Schema.Struct<{
|
|
131
|
-
cacheReadTokens: typeof Schema.Int;
|
|
132
|
-
cacheWriteTokens: typeof Schema.Int;
|
|
133
|
-
costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
134
|
-
inputTokens: typeof Schema.Int;
|
|
135
|
-
outputTokens: typeof Schema.Int;
|
|
136
|
-
totalTokens: typeof Schema.Int;
|
|
137
|
-
}>>;
|
|
138
|
-
verifySteps: Schema.Array$<Schema.Struct<{
|
|
139
|
-
command: typeof Schema.String;
|
|
140
|
-
exitCode: typeof Schema.Int;
|
|
141
|
-
}>>;
|
|
142
|
-
voidFields: Schema.Array$<typeof Schema.String>;
|
|
143
|
-
}>;
|
|
144
|
-
type EvalTrial = typeof EvalTrial.Type;
|
|
145
|
-
declare const EvalDistribution: Schema.Struct<{
|
|
146
|
-
commandMax: typeof Schema.Int;
|
|
147
|
-
commandMedian: typeof Schema.Number;
|
|
148
|
-
commandMin: typeof Schema.Int;
|
|
149
|
-
deterministic: typeof Schema.Boolean;
|
|
150
|
-
failed: typeof Schema.Int;
|
|
151
|
-
passRate: typeof Schema.Number;
|
|
152
|
-
passed: typeof Schema.Int;
|
|
153
|
-
scored: typeof Schema.Int;
|
|
154
|
-
trials: typeof Schema.Int;
|
|
155
|
-
voided: typeof Schema.Int;
|
|
156
|
-
}>;
|
|
157
|
-
type EvalDistribution = typeof EvalDistribution.Type;
|
|
158
|
-
declare const EvalVerdict: Schema.Literal<["improved", "incomparable", "regressed", "unchanged"]>;
|
|
159
|
-
type EvalVerdict = typeof EvalVerdict.Type;
|
|
160
|
-
declare const EvalComparison: Schema.Struct<{
|
|
161
|
-
baselineHarnessVersion: typeof Schema.String;
|
|
162
|
-
baselinePassRate: typeof Schema.Number;
|
|
163
|
-
baselineProfileVersion: Schema.NullOr<typeof Schema.String>;
|
|
164
|
-
candidateHarnessVersion: typeof Schema.String;
|
|
165
|
-
candidatePassRate: typeof Schema.Number;
|
|
166
|
-
candidateProfileVersion: Schema.NullOr<typeof Schema.String>;
|
|
167
|
-
delta: typeof Schema.Number;
|
|
168
|
-
determinismLost: typeof Schema.Boolean;
|
|
169
|
-
reason: Schema.NullOr<typeof Schema.String>;
|
|
170
|
-
verdict: Schema.Literal<["improved", "incomparable", "regressed", "unchanged"]>;
|
|
171
|
-
}>;
|
|
172
|
-
type EvalComparison = typeof EvalComparison.Type;
|
|
173
|
-
declare const EvalCell: Schema.Struct<{
|
|
174
|
-
caseName: typeof Schema.String;
|
|
175
|
-
cellKey: Schema.NullOr<typeof Schema.String>;
|
|
176
|
-
costs: Schema.NullOr<Schema.Struct<{
|
|
177
|
-
allocatedUsd: typeof Schema.Number;
|
|
178
|
-
components: Schema.Array$<Schema.Struct<{
|
|
179
|
-
classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
|
|
180
|
-
component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
|
|
181
|
-
detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
|
|
182
|
-
explanation: typeof Schema.String;
|
|
183
|
-
source: typeof Schema.String;
|
|
184
|
-
usd: Schema.NullOr<typeof Schema.Number>;
|
|
185
|
-
}>>;
|
|
186
|
-
estimatedEquivalentUsd: typeof Schema.Number;
|
|
187
|
-
incomplete: typeof Schema.Boolean;
|
|
188
|
-
knownActualUsd: typeof Schema.Number;
|
|
189
|
-
}>>;
|
|
190
|
-
comparison: Schema.NullOr<Schema.Struct<{
|
|
191
|
-
baselineHarnessVersion: typeof Schema.String;
|
|
192
|
-
baselinePassRate: typeof Schema.Number;
|
|
193
|
-
baselineProfileVersion: Schema.NullOr<typeof Schema.String>;
|
|
194
|
-
candidateHarnessVersion: typeof Schema.String;
|
|
195
|
-
candidatePassRate: typeof Schema.Number;
|
|
196
|
-
candidateProfileVersion: Schema.NullOr<typeof Schema.String>;
|
|
197
|
-
delta: typeof Schema.Number;
|
|
198
|
-
determinismLost: typeof Schema.Boolean;
|
|
199
|
-
reason: Schema.NullOr<typeof Schema.String>;
|
|
200
|
-
verdict: Schema.Literal<["improved", "incomparable", "regressed", "unchanged"]>;
|
|
201
|
-
}>>;
|
|
202
|
-
distribution: Schema.NullOr<Schema.Struct<{
|
|
203
|
-
commandMax: typeof Schema.Int;
|
|
204
|
-
commandMedian: typeof Schema.Number;
|
|
205
|
-
commandMin: typeof Schema.Int;
|
|
206
|
-
deterministic: typeof Schema.Boolean;
|
|
207
|
-
failed: typeof Schema.Int;
|
|
208
|
-
passRate: typeof Schema.Number;
|
|
209
|
-
passed: typeof Schema.Int;
|
|
210
|
-
scored: typeof Schema.Int;
|
|
211
|
-
trials: typeof Schema.Int;
|
|
212
|
-
voided: typeof Schema.Int;
|
|
213
|
-
}>>;
|
|
214
|
-
internalId: Schema.NullOr<typeof Schema.String>;
|
|
215
|
-
setup: Schema.NullOr<Schema.Struct<{
|
|
216
|
-
prompt: typeof Schema.String;
|
|
217
|
-
repoRef: Schema.NullOr<typeof Schema.String>;
|
|
218
|
-
repoUrl: Schema.NullOr<typeof Schema.String>;
|
|
219
|
-
prepareName: Schema.NullOr<typeof Schema.String>;
|
|
220
|
-
validatorName: Schema.NullOr<typeof Schema.String>;
|
|
221
|
-
verifyCommand: Schema.NullOr<typeof Schema.String>;
|
|
222
|
-
workspace: typeof Schema.String;
|
|
223
|
-
}>>;
|
|
224
|
-
status: Schema.Literal<["running", "finished", "failed"]>;
|
|
225
|
-
taskIndex: typeof Schema.Int;
|
|
226
|
-
trials: Schema.Array$<Schema.Struct<{
|
|
227
|
-
commands: typeof Schema.Int;
|
|
228
|
-
costs: Schema.NullOr<Schema.Struct<{
|
|
229
|
-
allocatedUsd: typeof Schema.Number;
|
|
230
|
-
components: Schema.Array$<Schema.Struct<{
|
|
231
|
-
classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
|
|
232
|
-
component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
|
|
233
|
-
detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
|
|
234
|
-
explanation: typeof Schema.String;
|
|
235
|
-
source: typeof Schema.String;
|
|
236
|
-
usd: Schema.NullOr<typeof Schema.Number>;
|
|
237
|
-
}>>;
|
|
238
|
-
estimatedEquivalentUsd: typeof Schema.Number;
|
|
239
|
-
incomplete: typeof Schema.Boolean;
|
|
240
|
-
knownActualUsd: typeof Schema.Number;
|
|
241
|
-
}>>;
|
|
242
|
-
prepared: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
|
|
243
|
-
exitCode: typeof Schema.Int;
|
|
244
|
-
failedCommands: typeof Schema.Int;
|
|
245
|
-
filesChanged: Schema.Array$<typeof Schema.String>;
|
|
246
|
-
modelMs: typeof Schema.Int;
|
|
247
|
-
ordinal: typeof Schema.Int;
|
|
248
|
-
passed: typeof Schema.Boolean;
|
|
249
|
-
sandboxId: Schema.NullOr<typeof Schema.String>;
|
|
250
|
-
sandboxMs: typeof Schema.Int;
|
|
251
|
-
status: Schema.Literal<["queued", "running", "passed", "failed", "void"]>;
|
|
252
|
-
timed: typeof Schema.Boolean;
|
|
253
|
-
trajectory: Schema.Array$<Schema.Union<[Schema.Struct<{
|
|
254
|
-
_tag: Schema.Literal<["command"]>;
|
|
255
|
-
command: typeof Schema.String;
|
|
256
|
-
exitCode: Schema.NullOr<typeof Schema.Int>;
|
|
257
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
258
|
-
output: typeof Schema.String;
|
|
259
|
-
startedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
260
|
-
}>, Schema.Struct<{
|
|
261
|
-
_tag: Schema.Literal<["message"]>;
|
|
262
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
263
|
-
text: typeof Schema.String;
|
|
264
|
-
usage: Schema.optional<Schema.NullOr<Schema.Struct<{
|
|
265
|
-
cacheReadTokens: typeof Schema.Int;
|
|
266
|
-
cacheWriteTokens: typeof Schema.Int;
|
|
267
|
-
costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
268
|
-
inputTokens: typeof Schema.Int;
|
|
269
|
-
outputTokens: typeof Schema.Int;
|
|
270
|
-
totalTokens: typeof Schema.Int;
|
|
271
|
-
}>>>;
|
|
272
|
-
}>, Schema.Struct<{
|
|
273
|
-
_tag: Schema.Literal<["toolCall"]>;
|
|
274
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
275
|
-
name: typeof Schema.String;
|
|
276
|
-
startedAtMillis: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
277
|
-
status: Schema.NullOr<typeof Schema.String>;
|
|
278
|
-
}>, Schema.Struct<{
|
|
279
|
-
_tag: Schema.Literal<["fileChange"]>;
|
|
280
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
281
|
-
paths: Schema.Array$<typeof Schema.String>;
|
|
282
|
-
}>]>>;
|
|
283
|
-
usage: Schema.NullOr<Schema.Struct<{
|
|
284
|
-
cacheReadTokens: typeof Schema.Int;
|
|
285
|
-
cacheWriteTokens: typeof Schema.Int;
|
|
286
|
-
costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
287
|
-
inputTokens: typeof Schema.Int;
|
|
288
|
-
outputTokens: typeof Schema.Int;
|
|
289
|
-
totalTokens: typeof Schema.Int;
|
|
290
|
-
}>>;
|
|
291
|
-
verifySteps: Schema.Array$<Schema.Struct<{
|
|
292
|
-
command: typeof Schema.String;
|
|
293
|
-
exitCode: typeof Schema.Int;
|
|
294
|
-
}>>;
|
|
295
|
-
voidFields: Schema.Array$<typeof Schema.String>;
|
|
296
|
-
}>>;
|
|
297
|
-
}>;
|
|
298
|
-
type EvalCell = typeof EvalCell.Type;
|
|
299
|
-
declare const EvalRun: Schema.Struct<{
|
|
300
|
-
cases: Schema.Array$<typeof Schema.String>;
|
|
301
|
-
cells: Schema.Array$<Schema.Struct<{
|
|
302
|
-
caseName: typeof Schema.String;
|
|
303
|
-
cellKey: Schema.NullOr<typeof Schema.String>;
|
|
304
|
-
costs: Schema.NullOr<Schema.Struct<{
|
|
305
|
-
allocatedUsd: typeof Schema.Number;
|
|
306
|
-
components: Schema.Array$<Schema.Struct<{
|
|
307
|
-
classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
|
|
308
|
-
component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
|
|
309
|
-
detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
|
|
310
|
-
explanation: typeof Schema.String;
|
|
311
|
-
source: typeof Schema.String;
|
|
312
|
-
usd: Schema.NullOr<typeof Schema.Number>;
|
|
313
|
-
}>>;
|
|
314
|
-
estimatedEquivalentUsd: typeof Schema.Number;
|
|
315
|
-
incomplete: typeof Schema.Boolean;
|
|
316
|
-
knownActualUsd: typeof Schema.Number;
|
|
317
|
-
}>>;
|
|
318
|
-
comparison: Schema.NullOr<Schema.Struct<{
|
|
319
|
-
baselineHarnessVersion: typeof Schema.String;
|
|
320
|
-
baselinePassRate: typeof Schema.Number;
|
|
321
|
-
baselineProfileVersion: Schema.NullOr<typeof Schema.String>;
|
|
322
|
-
candidateHarnessVersion: typeof Schema.String;
|
|
323
|
-
candidatePassRate: typeof Schema.Number;
|
|
324
|
-
candidateProfileVersion: Schema.NullOr<typeof Schema.String>;
|
|
325
|
-
delta: typeof Schema.Number;
|
|
326
|
-
determinismLost: typeof Schema.Boolean;
|
|
327
|
-
reason: Schema.NullOr<typeof Schema.String>;
|
|
328
|
-
verdict: Schema.Literal<["improved", "incomparable", "regressed", "unchanged"]>;
|
|
329
|
-
}>>;
|
|
330
|
-
distribution: Schema.NullOr<Schema.Struct<{
|
|
331
|
-
commandMax: typeof Schema.Int;
|
|
332
|
-
commandMedian: typeof Schema.Number;
|
|
333
|
-
commandMin: typeof Schema.Int;
|
|
334
|
-
deterministic: typeof Schema.Boolean;
|
|
335
|
-
failed: typeof Schema.Int;
|
|
336
|
-
passRate: typeof Schema.Number;
|
|
337
|
-
passed: typeof Schema.Int;
|
|
338
|
-
scored: typeof Schema.Int;
|
|
339
|
-
trials: typeof Schema.Int;
|
|
340
|
-
voided: typeof Schema.Int;
|
|
341
|
-
}>>;
|
|
342
|
-
internalId: Schema.NullOr<typeof Schema.String>;
|
|
343
|
-
setup: Schema.NullOr<Schema.Struct<{
|
|
344
|
-
prompt: typeof Schema.String;
|
|
345
|
-
repoRef: Schema.NullOr<typeof Schema.String>;
|
|
346
|
-
repoUrl: Schema.NullOr<typeof Schema.String>;
|
|
347
|
-
prepareName: Schema.NullOr<typeof Schema.String>;
|
|
348
|
-
validatorName: Schema.NullOr<typeof Schema.String>;
|
|
349
|
-
verifyCommand: Schema.NullOr<typeof Schema.String>;
|
|
350
|
-
workspace: typeof Schema.String;
|
|
351
|
-
}>>;
|
|
352
|
-
status: Schema.Literal<["running", "finished", "failed"]>;
|
|
353
|
-
taskIndex: typeof Schema.Int;
|
|
354
|
-
trials: Schema.Array$<Schema.Struct<{
|
|
355
|
-
commands: typeof Schema.Int;
|
|
356
|
-
costs: Schema.NullOr<Schema.Struct<{
|
|
357
|
-
allocatedUsd: typeof Schema.Number;
|
|
358
|
-
components: Schema.Array$<Schema.Struct<{
|
|
359
|
-
classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
|
|
360
|
-
component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
|
|
361
|
-
detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
|
|
362
|
-
explanation: typeof Schema.String;
|
|
363
|
-
source: typeof Schema.String;
|
|
364
|
-
usd: Schema.NullOr<typeof Schema.Number>;
|
|
365
|
-
}>>;
|
|
366
|
-
estimatedEquivalentUsd: typeof Schema.Number;
|
|
367
|
-
incomplete: typeof Schema.Boolean;
|
|
368
|
-
knownActualUsd: typeof Schema.Number;
|
|
369
|
-
}>>;
|
|
370
|
-
prepared: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
|
|
371
|
-
exitCode: typeof Schema.Int;
|
|
372
|
-
failedCommands: typeof Schema.Int;
|
|
373
|
-
filesChanged: Schema.Array$<typeof Schema.String>;
|
|
374
|
-
modelMs: typeof Schema.Int;
|
|
375
|
-
ordinal: typeof Schema.Int;
|
|
376
|
-
passed: typeof Schema.Boolean;
|
|
377
|
-
sandboxId: Schema.NullOr<typeof Schema.String>;
|
|
378
|
-
sandboxMs: typeof Schema.Int;
|
|
379
|
-
status: Schema.Literal<["queued", "running", "passed", "failed", "void"]>;
|
|
380
|
-
timed: typeof Schema.Boolean;
|
|
381
|
-
trajectory: Schema.Array$<Schema.Union<[Schema.Struct<{
|
|
382
|
-
_tag: Schema.Literal<["command"]>;
|
|
383
|
-
command: typeof Schema.String;
|
|
384
|
-
exitCode: Schema.NullOr<typeof Schema.Int>;
|
|
385
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
386
|
-
output: typeof Schema.String;
|
|
387
|
-
startedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
388
|
-
}>, Schema.Struct<{
|
|
389
|
-
_tag: Schema.Literal<["message"]>;
|
|
390
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
391
|
-
text: typeof Schema.String;
|
|
392
|
-
usage: Schema.optional<Schema.NullOr<Schema.Struct<{
|
|
393
|
-
cacheReadTokens: typeof Schema.Int;
|
|
394
|
-
cacheWriteTokens: typeof Schema.Int;
|
|
395
|
-
costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
396
|
-
inputTokens: typeof Schema.Int;
|
|
397
|
-
outputTokens: typeof Schema.Int;
|
|
398
|
-
totalTokens: typeof Schema.Int;
|
|
399
|
-
}>>>;
|
|
400
|
-
}>, Schema.Struct<{
|
|
401
|
-
_tag: Schema.Literal<["toolCall"]>;
|
|
402
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
403
|
-
name: typeof Schema.String;
|
|
404
|
-
startedAtMillis: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
405
|
-
status: Schema.NullOr<typeof Schema.String>;
|
|
406
|
-
}>, Schema.Struct<{
|
|
407
|
-
_tag: Schema.Literal<["fileChange"]>;
|
|
408
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
409
|
-
paths: Schema.Array$<typeof Schema.String>;
|
|
410
|
-
}>]>>;
|
|
411
|
-
usage: Schema.NullOr<Schema.Struct<{
|
|
412
|
-
cacheReadTokens: typeof Schema.Int;
|
|
413
|
-
cacheWriteTokens: typeof Schema.Int;
|
|
414
|
-
costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
415
|
-
inputTokens: typeof Schema.Int;
|
|
416
|
-
outputTokens: typeof Schema.Int;
|
|
417
|
-
totalTokens: typeof Schema.Int;
|
|
418
|
-
}>>;
|
|
419
|
-
verifySteps: Schema.Array$<Schema.Struct<{
|
|
420
|
-
command: typeof Schema.String;
|
|
421
|
-
exitCode: typeof Schema.Int;
|
|
422
|
-
}>>;
|
|
423
|
-
voidFields: Schema.Array$<typeof Schema.String>;
|
|
424
|
-
}>>;
|
|
425
|
-
}>>;
|
|
426
|
-
costs: Schema.NullOr<Schema.Struct<{
|
|
427
|
-
allocatedUsd: typeof Schema.Number;
|
|
428
|
-
components: Schema.Array$<Schema.Struct<{
|
|
429
|
-
classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
|
|
430
|
-
component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
|
|
431
|
-
detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
|
|
432
|
-
explanation: typeof Schema.String;
|
|
433
|
-
source: typeof Schema.String;
|
|
434
|
-
usd: Schema.NullOr<typeof Schema.Number>;
|
|
435
|
-
}>>;
|
|
436
|
-
estimatedEquivalentUsd: typeof Schema.Number;
|
|
437
|
-
incomplete: typeof Schema.Boolean;
|
|
438
|
-
knownActualUsd: typeof Schema.Number;
|
|
439
|
-
}>>;
|
|
440
|
-
failure: Schema.NullOr<typeof Schema.String>;
|
|
441
|
-
finishedAt: Schema.NullOr<Schema.transformOrFail<Schema.SchemaClass<string, string, never>, typeof Schema.DateTimeUtcFromSelf, never>>;
|
|
442
|
-
id: typeof Schema.String;
|
|
443
|
-
name: Schema.NullOr<Schema.filter<Schema.filter<typeof Schema.String>>>;
|
|
444
|
-
startedAt: Schema.transformOrFail<Schema.SchemaClass<string, string, never>, typeof Schema.DateTimeUtcFromSelf, never>;
|
|
445
|
-
status: Schema.Literal<["running", "finished", "failed"]>;
|
|
446
|
-
tasks: Schema.Array$<Schema.Struct<{
|
|
447
|
-
harness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command"]>;
|
|
448
|
-
harnessVersion: typeof Schema.String;
|
|
449
|
-
model: typeof Schema.String;
|
|
450
|
-
profile: Schema.optional<Schema.NullOr<Schema.Struct<{
|
|
451
|
-
name: typeof Schema.String;
|
|
452
|
-
version: typeof Schema.String;
|
|
453
|
-
}>>>;
|
|
454
|
-
provider: Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>;
|
|
455
|
-
}>>;
|
|
456
|
-
}>;
|
|
457
|
-
type EvalRun = typeof EvalRun.Type;
|
|
458
|
-
declare const EvalRunSummary: Schema.Struct<{
|
|
459
|
-
caseCount: typeof Schema.Int;
|
|
460
|
-
columns: Schema.Array$<Schema.Struct<{
|
|
461
|
-
harness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command"]>;
|
|
462
|
-
harnessVersion: typeof Schema.String;
|
|
463
|
-
model: typeof Schema.String;
|
|
464
|
-
profile: Schema.optional<Schema.NullOr<Schema.Struct<{
|
|
465
|
-
name: typeof Schema.String;
|
|
466
|
-
version: typeof Schema.String;
|
|
467
|
-
}>>>;
|
|
468
|
-
provider: Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>;
|
|
469
|
-
}>>;
|
|
470
|
-
commandMax: Schema.NullOr<typeof Schema.Int>;
|
|
471
|
-
commandMin: Schema.NullOr<typeof Schema.Int>;
|
|
472
|
-
failure: Schema.NullOr<typeof Schema.String>;
|
|
473
|
-
finishedAt: Schema.NullOr<Schema.transformOrFail<Schema.SchemaClass<string, string, never>, typeof Schema.DateTimeUtcFromSelf, never>>;
|
|
474
|
-
firstCaseName: Schema.NullOr<typeof Schema.String>;
|
|
475
|
-
id: typeof Schema.String;
|
|
476
|
-
name: Schema.NullOr<typeof Schema.String>;
|
|
477
|
-
passed: typeof Schema.Int;
|
|
478
|
-
scored: typeof Schema.Int;
|
|
479
|
-
startedAt: Schema.transformOrFail<Schema.SchemaClass<string, string, never>, typeof Schema.DateTimeUtcFromSelf, never>;
|
|
480
|
-
status: Schema.Literal<["running", "finished", "failed"]>;
|
|
481
|
-
taskCount: typeof Schema.Int;
|
|
482
|
-
voided: typeof Schema.Int;
|
|
483
|
-
}>;
|
|
484
|
-
type EvalRunSummary = typeof EvalRunSummary.Type;
|
|
485
|
-
declare const EvalCellHistoryEntry: Schema.Struct<{
|
|
486
|
-
distribution: Schema.Struct<{
|
|
487
|
-
commandMax: typeof Schema.Int;
|
|
488
|
-
commandMedian: typeof Schema.Number;
|
|
489
|
-
commandMin: typeof Schema.Int;
|
|
490
|
-
deterministic: typeof Schema.Boolean;
|
|
491
|
-
failed: typeof Schema.Int;
|
|
492
|
-
passRate: typeof Schema.Number;
|
|
493
|
-
passed: typeof Schema.Int;
|
|
494
|
-
scored: typeof Schema.Int;
|
|
495
|
-
trials: typeof Schema.Int;
|
|
496
|
-
voided: typeof Schema.Int;
|
|
497
|
-
}>;
|
|
498
|
-
finishedAt: Schema.NullOr<Schema.transformOrFail<Schema.SchemaClass<string, string, never>, typeof Schema.DateTimeUtcFromSelf, never>>;
|
|
499
|
-
harnessVersion: typeof Schema.String;
|
|
500
|
-
internalId: typeof Schema.String;
|
|
501
|
-
profileVersion: Schema.NullOr<typeof Schema.String>;
|
|
502
|
-
runId: typeof Schema.String;
|
|
503
|
-
trials: Schema.Array$<Schema.Struct<{
|
|
504
|
-
commands: typeof Schema.Int;
|
|
505
|
-
costs: Schema.NullOr<Schema.Struct<{
|
|
506
|
-
allocatedUsd: typeof Schema.Number;
|
|
507
|
-
components: Schema.Array$<Schema.Struct<{
|
|
508
|
-
classification: Schema.Literal<["actual", "allocated", "estimate", "included", "managed", "unknown"]>;
|
|
509
|
-
component: Schema.Literal<["harness", "model", "platform", "sandbox"]>;
|
|
510
|
-
detail: Schema.Record$<typeof Schema.String, typeof Schema.Unknown>;
|
|
511
|
-
explanation: typeof Schema.String;
|
|
512
|
-
source: typeof Schema.String;
|
|
513
|
-
usd: Schema.NullOr<typeof Schema.Number>;
|
|
514
|
-
}>>;
|
|
515
|
-
estimatedEquivalentUsd: typeof Schema.Number;
|
|
516
|
-
incomplete: typeof Schema.Boolean;
|
|
517
|
-
knownActualUsd: typeof Schema.Number;
|
|
518
|
-
}>>;
|
|
519
|
-
prepared: Schema.NullOr<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
|
|
520
|
-
exitCode: typeof Schema.Int;
|
|
521
|
-
failedCommands: typeof Schema.Int;
|
|
522
|
-
filesChanged: Schema.Array$<typeof Schema.String>;
|
|
523
|
-
modelMs: typeof Schema.Int;
|
|
524
|
-
ordinal: typeof Schema.Int;
|
|
525
|
-
passed: typeof Schema.Boolean;
|
|
526
|
-
sandboxId: Schema.NullOr<typeof Schema.String>;
|
|
527
|
-
sandboxMs: typeof Schema.Int;
|
|
528
|
-
status: Schema.Literal<["queued", "running", "passed", "failed", "void"]>;
|
|
529
|
-
timed: typeof Schema.Boolean;
|
|
530
|
-
trajectory: Schema.Array$<Schema.Union<[Schema.Struct<{
|
|
531
|
-
_tag: Schema.Literal<["command"]>;
|
|
532
|
-
command: typeof Schema.String;
|
|
533
|
-
exitCode: Schema.NullOr<typeof Schema.Int>;
|
|
534
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
535
|
-
output: typeof Schema.String;
|
|
536
|
-
startedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
537
|
-
}>, Schema.Struct<{
|
|
538
|
-
_tag: Schema.Literal<["message"]>;
|
|
539
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
540
|
-
text: typeof Schema.String;
|
|
541
|
-
usage: Schema.optional<Schema.NullOr<Schema.Struct<{
|
|
542
|
-
cacheReadTokens: typeof Schema.Int;
|
|
543
|
-
cacheWriteTokens: typeof Schema.Int;
|
|
544
|
-
costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
545
|
-
inputTokens: typeof Schema.Int;
|
|
546
|
-
outputTokens: typeof Schema.Int;
|
|
547
|
-
totalTokens: typeof Schema.Int;
|
|
548
|
-
}>>>;
|
|
549
|
-
}>, Schema.Struct<{
|
|
550
|
-
_tag: Schema.Literal<["toolCall"]>;
|
|
551
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
552
|
-
name: typeof Schema.String;
|
|
553
|
-
startedAtMillis: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
554
|
-
status: Schema.NullOr<typeof Schema.String>;
|
|
555
|
-
}>, Schema.Struct<{
|
|
556
|
-
_tag: Schema.Literal<["fileChange"]>;
|
|
557
|
-
finishedAtMillis: Schema.NullOr<typeof Schema.Number>;
|
|
558
|
-
paths: Schema.Array$<typeof Schema.String>;
|
|
559
|
-
}>]>>;
|
|
560
|
-
usage: Schema.NullOr<Schema.Struct<{
|
|
561
|
-
cacheReadTokens: typeof Schema.Int;
|
|
562
|
-
cacheWriteTokens: typeof Schema.Int;
|
|
563
|
-
costUsd: Schema.optional<Schema.NullOr<typeof Schema.Number>>;
|
|
564
|
-
inputTokens: typeof Schema.Int;
|
|
565
|
-
outputTokens: typeof Schema.Int;
|
|
566
|
-
totalTokens: typeof Schema.Int;
|
|
567
|
-
}>>;
|
|
568
|
-
verifySteps: Schema.Array$<Schema.Struct<{
|
|
569
|
-
command: typeof Schema.String;
|
|
570
|
-
exitCode: typeof Schema.Int;
|
|
571
|
-
}>>;
|
|
572
|
-
voidFields: Schema.Array$<typeof Schema.String>;
|
|
573
|
-
}>>;
|
|
574
|
-
}>;
|
|
575
|
-
type EvalCellHistoryEntry = typeof EvalCellHistoryEntry.Type;
|
|
576
|
-
declare const StartedEval: Schema.Struct<{
|
|
577
|
-
id: typeof Schema.String;
|
|
578
|
-
}>;
|
|
579
|
-
declare const RerunCellRequest: Schema.Struct<{
|
|
580
|
-
trials: Schema.filter<typeof Schema.Int>;
|
|
581
|
-
}>;
|
|
582
|
-
type RerunCellRequest = typeof RerunCellRequest.Type;
|
|
583
|
-
type StartedEval = typeof StartedEval.Type;
|
|
584
|
-
declare const CatalogueModel: Schema.Struct<{
|
|
585
|
-
displayName: typeof Schema.String;
|
|
586
|
-
id: typeof Schema.String;
|
|
587
|
-
summary: Schema.NullOr<typeof Schema.String>;
|
|
588
|
-
vendor: Schema.NullOr<typeof Schema.String>;
|
|
589
|
-
}>;
|
|
590
|
-
type CatalogueModel = typeof CatalogueModel.Type;
|
|
591
|
-
declare const ModelCatalogue: Schema.Struct<{
|
|
592
|
-
harness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command"]>;
|
|
593
|
-
models: Schema.Array$<Schema.Struct<{
|
|
594
|
-
displayName: typeof Schema.String;
|
|
595
|
-
id: typeof Schema.String;
|
|
596
|
-
summary: Schema.NullOr<typeof Schema.String>;
|
|
597
|
-
vendor: Schema.NullOr<typeof Schema.String>;
|
|
598
|
-
}>>;
|
|
599
|
-
total: typeof Schema.Int;
|
|
600
|
-
}>;
|
|
601
|
-
type ModelCatalogue = typeof ModelCatalogue.Type;
|
|
602
|
-
//#endregion
|
|
603
|
-
export { EvalVerdict as _, EvalDistribution as a, StartedEval as b, EvalProvider as c, EvalRunSummary as d, EvalSource as f, EvalUsage as g, EvalTrialStatus as h, EvalComparison as i, EvalRun as l, EvalTrial as m, EvalCell as n, EvalHarness as o, EvalTask as p, EvalCellHistoryEntry as r, EvalJournalEntry as s, CatalogueModel as t, EvalRunStatus as u, ModelCatalogue as v, RerunCellRequest as y };
|