@openpond/evals 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -5
- package/dist/benchmarks.js +218 -0
- package/dist/builtin-benchmarks/harness-refiner.js +1268 -0
- package/dist/index.js +1 -0
- package/dist/types/benchmarks.d.ts +293 -0
- package/dist/types/benchmarks.d.ts.map +1 -0
- package/dist/types/builtin-benchmarks/harness-refiner.d.ts +119 -0
- package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -0
- package/dist/types/evidence/conformance.d.ts +8 -8
- package/dist/types/evidence/contracts.d.ts +8 -8
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/review-conformance.d.ts +10 -10
- package/package.json +5 -1
package/dist/index.js
CHANGED
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { type AttemptReceipt, type EvaluationResult } from "./runs.js";
|
|
3
|
+
export { harnessRefinerBenchmarkAssets, harnessRefinerBenchmarkRelease, } from "./builtin-benchmarks/harness-refiner.js";
|
|
4
|
+
export declare const BenchmarkMetricSchema: z.ZodEnum<{
|
|
5
|
+
foreground_tokens: "foreground_tokens";
|
|
6
|
+
success_rate: "success_rate";
|
|
7
|
+
latency_ms: "latency_ms";
|
|
8
|
+
cost_usd: "cost_usd";
|
|
9
|
+
}>;
|
|
10
|
+
export declare const BenchmarkRunPhaseSchema: z.ZodEnum<{
|
|
11
|
+
baseline: "baseline";
|
|
12
|
+
candidate: "candidate";
|
|
13
|
+
}>;
|
|
14
|
+
export declare const BenchmarkProtocolSchema: z.ZodObject<{
|
|
15
|
+
split: z.ZodEnum<{
|
|
16
|
+
train: "train";
|
|
17
|
+
validation: "validation";
|
|
18
|
+
test: "test";
|
|
19
|
+
frozen_eval: "frozen_eval";
|
|
20
|
+
}>;
|
|
21
|
+
taskIds: z.ZodArray<z.ZodString>;
|
|
22
|
+
seeds: z.ZodArray<z.ZodString>;
|
|
23
|
+
repetitions: z.ZodNumber;
|
|
24
|
+
runtimeTargetHash: z.ZodString;
|
|
25
|
+
environmentHash: z.ZodString;
|
|
26
|
+
toolContractHash: z.ZodString;
|
|
27
|
+
limitsHash: z.ZodString;
|
|
28
|
+
}, z.core.$strict>;
|
|
29
|
+
export declare const BenchmarkDefinitionSchema: z.ZodObject<{
|
|
30
|
+
schemaVersion: z.ZodLiteral<"openpond.benchmarkDefinition.v1">;
|
|
31
|
+
id: z.ZodString;
|
|
32
|
+
title: z.ZodString;
|
|
33
|
+
description: z.ZodString;
|
|
34
|
+
tasksetRelease: z.ZodObject<{
|
|
35
|
+
id: z.ZodString;
|
|
36
|
+
contentHash: z.ZodString;
|
|
37
|
+
}, z.core.$strict>;
|
|
38
|
+
adaptationSplit: z.ZodEnum<{
|
|
39
|
+
train: "train";
|
|
40
|
+
validation: "validation";
|
|
41
|
+
test: "test";
|
|
42
|
+
frozen_eval: "frozen_eval";
|
|
43
|
+
}>;
|
|
44
|
+
evaluationSplit: z.ZodEnum<{
|
|
45
|
+
train: "train";
|
|
46
|
+
validation: "validation";
|
|
47
|
+
test: "test";
|
|
48
|
+
frozen_eval: "frozen_eval";
|
|
49
|
+
}>;
|
|
50
|
+
primaryMetric: z.ZodEnum<{
|
|
51
|
+
foreground_tokens: "foreground_tokens";
|
|
52
|
+
success_rate: "success_rate";
|
|
53
|
+
latency_ms: "latency_ms";
|
|
54
|
+
cost_usd: "cost_usd";
|
|
55
|
+
}>;
|
|
56
|
+
qualityGate: z.ZodEnum<{
|
|
57
|
+
none: "none";
|
|
58
|
+
non_regression: "non_regression";
|
|
59
|
+
all_pass: "all_pass";
|
|
60
|
+
}>;
|
|
61
|
+
caseCounts: z.ZodObject<{
|
|
62
|
+
adaptation: z.ZodNumber;
|
|
63
|
+
evaluation: z.ZodNumber;
|
|
64
|
+
}, z.core.$strict>;
|
|
65
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
66
|
+
}, z.core.$strict>;
|
|
67
|
+
export declare const BenchmarkRunRequestSchema: z.ZodObject<{
|
|
68
|
+
schemaVersion: z.ZodLiteral<"openpond.benchmarkRunRequest.v1">;
|
|
69
|
+
phase: z.ZodEnum<{
|
|
70
|
+
baseline: "baseline";
|
|
71
|
+
candidate: "candidate";
|
|
72
|
+
}>;
|
|
73
|
+
model: z.ZodObject<{
|
|
74
|
+
provider: z.ZodString;
|
|
75
|
+
model: z.ZodString;
|
|
76
|
+
revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
77
|
+
artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
78
|
+
tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
79
|
+
chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
80
|
+
}, z.core.$strict>;
|
|
81
|
+
reasoningEffort: z.ZodNullable<z.ZodString>;
|
|
82
|
+
split: z.ZodEnum<{
|
|
83
|
+
train: "train";
|
|
84
|
+
validation: "validation";
|
|
85
|
+
test: "test";
|
|
86
|
+
frozen_eval: "frozen_eval";
|
|
87
|
+
}>;
|
|
88
|
+
seeds: z.ZodArray<z.ZodString>;
|
|
89
|
+
repetitions: z.ZodNumber;
|
|
90
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
91
|
+
}, z.core.$strict>;
|
|
92
|
+
export declare const BenchmarkRunSummaryContentSchema: z.ZodObject<{
|
|
93
|
+
schemaVersion: z.ZodLiteral<"openpond.benchmarkRunSummary.v1">;
|
|
94
|
+
id: z.ZodString;
|
|
95
|
+
phase: z.ZodEnum<{
|
|
96
|
+
baseline: "baseline";
|
|
97
|
+
candidate: "candidate";
|
|
98
|
+
}>;
|
|
99
|
+
tasksetRelease: z.ZodObject<{
|
|
100
|
+
id: z.ZodString;
|
|
101
|
+
contentHash: z.ZodString;
|
|
102
|
+
}, z.core.$strict>;
|
|
103
|
+
harnessRelease: z.ZodObject<{
|
|
104
|
+
id: z.ZodString;
|
|
105
|
+
contentHash: z.ZodString;
|
|
106
|
+
}, z.core.$strict>;
|
|
107
|
+
evaluationResult: z.ZodObject<{
|
|
108
|
+
id: z.ZodString;
|
|
109
|
+
contentHash: z.ZodString;
|
|
110
|
+
}, z.core.$strict>;
|
|
111
|
+
model: z.ZodObject<{
|
|
112
|
+
provider: z.ZodString;
|
|
113
|
+
model: z.ZodString;
|
|
114
|
+
revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
115
|
+
artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
116
|
+
tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
117
|
+
chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
118
|
+
}, z.core.$strict>;
|
|
119
|
+
reasoningEffort: z.ZodNullable<z.ZodString>;
|
|
120
|
+
protocol: z.ZodObject<{
|
|
121
|
+
split: z.ZodEnum<{
|
|
122
|
+
train: "train";
|
|
123
|
+
validation: "validation";
|
|
124
|
+
test: "test";
|
|
125
|
+
frozen_eval: "frozen_eval";
|
|
126
|
+
}>;
|
|
127
|
+
taskIds: z.ZodArray<z.ZodString>;
|
|
128
|
+
seeds: z.ZodArray<z.ZodString>;
|
|
129
|
+
repetitions: z.ZodNumber;
|
|
130
|
+
runtimeTargetHash: z.ZodString;
|
|
131
|
+
environmentHash: z.ZodString;
|
|
132
|
+
toolContractHash: z.ZodString;
|
|
133
|
+
limitsHash: z.ZodString;
|
|
134
|
+
}, z.core.$strict>;
|
|
135
|
+
attemptCount: z.ZodNumber;
|
|
136
|
+
passedCount: z.ZodNumber;
|
|
137
|
+
terminalCount: z.ZodNumber;
|
|
138
|
+
usage: z.ZodObject<{
|
|
139
|
+
inputTokens: z.ZodNumber;
|
|
140
|
+
outputTokens: z.ZodNumber;
|
|
141
|
+
totalTokens: z.ZodNumber;
|
|
142
|
+
}, z.core.$strict>;
|
|
143
|
+
costUsd: z.ZodNullable<z.ZodNumber>;
|
|
144
|
+
latencyMs: z.ZodNumber;
|
|
145
|
+
createdAt: z.ZodString;
|
|
146
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
147
|
+
}, z.core.$strict>;
|
|
148
|
+
export declare const BenchmarkRunSummarySchema: z.ZodObject<{
|
|
149
|
+
schemaVersion: z.ZodLiteral<"openpond.benchmarkRunSummary.v1">;
|
|
150
|
+
id: z.ZodString;
|
|
151
|
+
phase: z.ZodEnum<{
|
|
152
|
+
baseline: "baseline";
|
|
153
|
+
candidate: "candidate";
|
|
154
|
+
}>;
|
|
155
|
+
tasksetRelease: z.ZodObject<{
|
|
156
|
+
id: z.ZodString;
|
|
157
|
+
contentHash: z.ZodString;
|
|
158
|
+
}, z.core.$strict>;
|
|
159
|
+
harnessRelease: z.ZodObject<{
|
|
160
|
+
id: z.ZodString;
|
|
161
|
+
contentHash: z.ZodString;
|
|
162
|
+
}, z.core.$strict>;
|
|
163
|
+
evaluationResult: z.ZodObject<{
|
|
164
|
+
id: z.ZodString;
|
|
165
|
+
contentHash: z.ZodString;
|
|
166
|
+
}, z.core.$strict>;
|
|
167
|
+
model: z.ZodObject<{
|
|
168
|
+
provider: z.ZodString;
|
|
169
|
+
model: z.ZodString;
|
|
170
|
+
revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
171
|
+
artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
172
|
+
tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
173
|
+
chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
174
|
+
}, z.core.$strict>;
|
|
175
|
+
reasoningEffort: z.ZodNullable<z.ZodString>;
|
|
176
|
+
protocol: z.ZodObject<{
|
|
177
|
+
split: z.ZodEnum<{
|
|
178
|
+
train: "train";
|
|
179
|
+
validation: "validation";
|
|
180
|
+
test: "test";
|
|
181
|
+
frozen_eval: "frozen_eval";
|
|
182
|
+
}>;
|
|
183
|
+
taskIds: z.ZodArray<z.ZodString>;
|
|
184
|
+
seeds: z.ZodArray<z.ZodString>;
|
|
185
|
+
repetitions: z.ZodNumber;
|
|
186
|
+
runtimeTargetHash: z.ZodString;
|
|
187
|
+
environmentHash: z.ZodString;
|
|
188
|
+
toolContractHash: z.ZodString;
|
|
189
|
+
limitsHash: z.ZodString;
|
|
190
|
+
}, z.core.$strict>;
|
|
191
|
+
attemptCount: z.ZodNumber;
|
|
192
|
+
passedCount: z.ZodNumber;
|
|
193
|
+
terminalCount: z.ZodNumber;
|
|
194
|
+
usage: z.ZodObject<{
|
|
195
|
+
inputTokens: z.ZodNumber;
|
|
196
|
+
outputTokens: z.ZodNumber;
|
|
197
|
+
totalTokens: z.ZodNumber;
|
|
198
|
+
}, z.core.$strict>;
|
|
199
|
+
costUsd: z.ZodNullable<z.ZodNumber>;
|
|
200
|
+
latencyMs: z.ZodNumber;
|
|
201
|
+
createdAt: z.ZodString;
|
|
202
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
203
|
+
contentHash: z.ZodString;
|
|
204
|
+
}, z.core.$strict>;
|
|
205
|
+
export declare const BenchmarkComparisonContentSchema: z.ZodObject<{
|
|
206
|
+
schemaVersion: z.ZodLiteral<"openpond.benchmarkComparison.v1">;
|
|
207
|
+
id: z.ZodString;
|
|
208
|
+
baseline: z.ZodObject<{
|
|
209
|
+
id: z.ZodString;
|
|
210
|
+
contentHash: z.ZodString;
|
|
211
|
+
}, z.core.$strict>;
|
|
212
|
+
candidate: z.ZodObject<{
|
|
213
|
+
id: z.ZodString;
|
|
214
|
+
contentHash: z.ZodString;
|
|
215
|
+
}, z.core.$strict>;
|
|
216
|
+
tasksetRelease: z.ZodObject<{
|
|
217
|
+
id: z.ZodString;
|
|
218
|
+
contentHash: z.ZodString;
|
|
219
|
+
}, z.core.$strict>;
|
|
220
|
+
primaryMetric: z.ZodEnum<{
|
|
221
|
+
foreground_tokens: "foreground_tokens";
|
|
222
|
+
success_rate: "success_rate";
|
|
223
|
+
latency_ms: "latency_ms";
|
|
224
|
+
cost_usd: "cost_usd";
|
|
225
|
+
}>;
|
|
226
|
+
qualityPassed: z.ZodBoolean;
|
|
227
|
+
baselinePassRate: z.ZodNumber;
|
|
228
|
+
candidatePassRate: z.ZodNumber;
|
|
229
|
+
foregroundTokenDelta: z.ZodNumber;
|
|
230
|
+
foregroundTokenDeltaPercent: z.ZodNullable<z.ZodNumber>;
|
|
231
|
+
improved: z.ZodBoolean;
|
|
232
|
+
createdAt: z.ZodString;
|
|
233
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
234
|
+
}, z.core.$strict>;
|
|
235
|
+
export declare const BenchmarkComparisonSchema: z.ZodObject<{
|
|
236
|
+
schemaVersion: z.ZodLiteral<"openpond.benchmarkComparison.v1">;
|
|
237
|
+
id: z.ZodString;
|
|
238
|
+
baseline: z.ZodObject<{
|
|
239
|
+
id: z.ZodString;
|
|
240
|
+
contentHash: z.ZodString;
|
|
241
|
+
}, z.core.$strict>;
|
|
242
|
+
candidate: z.ZodObject<{
|
|
243
|
+
id: z.ZodString;
|
|
244
|
+
contentHash: z.ZodString;
|
|
245
|
+
}, z.core.$strict>;
|
|
246
|
+
tasksetRelease: z.ZodObject<{
|
|
247
|
+
id: z.ZodString;
|
|
248
|
+
contentHash: z.ZodString;
|
|
249
|
+
}, z.core.$strict>;
|
|
250
|
+
primaryMetric: z.ZodEnum<{
|
|
251
|
+
foreground_tokens: "foreground_tokens";
|
|
252
|
+
success_rate: "success_rate";
|
|
253
|
+
latency_ms: "latency_ms";
|
|
254
|
+
cost_usd: "cost_usd";
|
|
255
|
+
}>;
|
|
256
|
+
qualityPassed: z.ZodBoolean;
|
|
257
|
+
baselinePassRate: z.ZodNumber;
|
|
258
|
+
candidatePassRate: z.ZodNumber;
|
|
259
|
+
foregroundTokenDelta: z.ZodNumber;
|
|
260
|
+
foregroundTokenDeltaPercent: z.ZodNullable<z.ZodNumber>;
|
|
261
|
+
improved: z.ZodBoolean;
|
|
262
|
+
createdAt: z.ZodString;
|
|
263
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
264
|
+
contentHash: z.ZodString;
|
|
265
|
+
}, z.core.$strict>;
|
|
266
|
+
export declare function createBenchmarkDefinition(input: z.input<typeof BenchmarkDefinitionSchema>): BenchmarkDefinition;
|
|
267
|
+
export declare function createBenchmarkRunSummary(input: {
|
|
268
|
+
id: string;
|
|
269
|
+
phase: BenchmarkRunPhase;
|
|
270
|
+
evaluation: EvaluationResult;
|
|
271
|
+
receipts: AttemptReceipt[];
|
|
272
|
+
reasoningEffort: string | null;
|
|
273
|
+
protocol: BenchmarkProtocol;
|
|
274
|
+
createdAt: string;
|
|
275
|
+
metadata?: Record<string, unknown>;
|
|
276
|
+
}): BenchmarkRunSummary;
|
|
277
|
+
export declare function compareBenchmarkRuns(input: {
|
|
278
|
+
id: string;
|
|
279
|
+
baseline: BenchmarkRunSummary;
|
|
280
|
+
candidate: BenchmarkRunSummary;
|
|
281
|
+
primaryMetric: BenchmarkMetric;
|
|
282
|
+
qualityGate: "none" | "non_regression" | "all_pass";
|
|
283
|
+
createdAt: string;
|
|
284
|
+
metadata?: Record<string, unknown>;
|
|
285
|
+
}): BenchmarkComparison;
|
|
286
|
+
export type BenchmarkMetric = z.infer<typeof BenchmarkMetricSchema>;
|
|
287
|
+
export type BenchmarkRunPhase = z.infer<typeof BenchmarkRunPhaseSchema>;
|
|
288
|
+
export type BenchmarkProtocol = z.infer<typeof BenchmarkProtocolSchema>;
|
|
289
|
+
export type BenchmarkDefinition = z.infer<typeof BenchmarkDefinitionSchema>;
|
|
290
|
+
export type BenchmarkRunRequest = z.infer<typeof BenchmarkRunRequestSchema>;
|
|
291
|
+
export type BenchmarkRunSummary = z.infer<typeof BenchmarkRunSummarySchema>;
|
|
292
|
+
export type BenchmarkComparison = z.infer<typeof BenchmarkComparisonSchema>;
|
|
293
|
+
//# sourceMappingURL=benchmarks.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"benchmarks.d.ts","sourceRoot":"","sources":["../../../../src/benchmarks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAGvE,OAAO,EACL,6BAA6B,EAC7B,8BAA8B,GAC/B,MAAM,yCAAyC,CAAC;AAEjD,eAAO,MAAM,qBAAqB;;;;;EAKhC,CAAC;AACH,eAAO,MAAM,uBAAuB;;;EAAoC,CAAC;AAEzE,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;kBASzB,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAe3B,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;kBAS3B,CAAC;AAQZ,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkBlC,CAAC;AACZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE3B,CAAC;AAEZ,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAelC,CAAC;AACZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE3B,CAAC;AAEZ,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,GAC/C,mBAAmB,CAErB;AAED,wBAAgB,yBAAyB,CAAC,KAAK,EAAE;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,iBAAiB,CAAC;IACzB,UAAU,EAAE,gBAAgB,CAAC;IAC7B,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,mBAAmB,CAuCtB;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,SAAS,EAAE,mBAAmB,CAAC;IAC/B,aAAa,EAAE,eAAe,CAAC;IAC/B,WAAW,EAAE,MAAM,GAAG,gBAAgB,GAAG,UAAU,CAAC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,mBAAmB,CAuDtB;AAkDD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
export declare const harnessRefinerBenchmarkRelease: {
|
|
2
|
+
schemaVersion: "openpond.tasksetRelease.v2";
|
|
3
|
+
id: string;
|
|
4
|
+
revision: number;
|
|
5
|
+
policy: {
|
|
6
|
+
policyVisibleFields: string[];
|
|
7
|
+
privilegedFields: string[];
|
|
8
|
+
hiddenGraderRefs: string[];
|
|
9
|
+
connectedAppScopes: string[];
|
|
10
|
+
};
|
|
11
|
+
environment: {
|
|
12
|
+
protocolVersion: "openpond.environment.v1";
|
|
13
|
+
kind: "text" | "agent" | "work" | "custom_program";
|
|
14
|
+
entrypoint: string;
|
|
15
|
+
stateful: boolean;
|
|
16
|
+
deterministicSeeds: boolean;
|
|
17
|
+
lifecycle: ("create" | "reset" | "step" | "collect" | "destroy")[];
|
|
18
|
+
networkPolicy: "none" | "declared_read_only" | "declared_scoped";
|
|
19
|
+
defaultTimeoutMs: number;
|
|
20
|
+
};
|
|
21
|
+
tools: {
|
|
22
|
+
name: string;
|
|
23
|
+
description: string;
|
|
24
|
+
inputSchema: Record<string, unknown>;
|
|
25
|
+
inputSchemaHash: string;
|
|
26
|
+
sideEffect: "read" | "write";
|
|
27
|
+
timeoutMs: number;
|
|
28
|
+
}[];
|
|
29
|
+
capabilities: {
|
|
30
|
+
id: string;
|
|
31
|
+
required: boolean;
|
|
32
|
+
scopes: string[];
|
|
33
|
+
portability: "portable" | "host_adapter" | "local_only" | "hosted_only";
|
|
34
|
+
}[];
|
|
35
|
+
tasks: {
|
|
36
|
+
id: string;
|
|
37
|
+
clusterKey: string;
|
|
38
|
+
split: "train" | "validation" | "test" | "frozen_eval";
|
|
39
|
+
input: Record<string, unknown>;
|
|
40
|
+
expectedOutput: Record<string, unknown> | null;
|
|
41
|
+
policyVisibleContext: Record<string, unknown>;
|
|
42
|
+
privilegedContextRef: string | null;
|
|
43
|
+
artifactRefs: {
|
|
44
|
+
id: string;
|
|
45
|
+
path: string;
|
|
46
|
+
contentHash: string;
|
|
47
|
+
sizeBytes: number;
|
|
48
|
+
mediaType: string;
|
|
49
|
+
visibility: "policy" | "verifier" | "host_private";
|
|
50
|
+
}[];
|
|
51
|
+
tags: string[];
|
|
52
|
+
}[];
|
|
53
|
+
graders: ({
|
|
54
|
+
id: string;
|
|
55
|
+
version: string;
|
|
56
|
+
weight: number;
|
|
57
|
+
hardGate: boolean;
|
|
58
|
+
rewardEligible: boolean;
|
|
59
|
+
privileged: boolean;
|
|
60
|
+
kind: "content" | "schema" | "artifact" | "runtime_event" | "state";
|
|
61
|
+
config: Record<string, unknown>;
|
|
62
|
+
} | {
|
|
63
|
+
id: string;
|
|
64
|
+
version: string;
|
|
65
|
+
weight: number;
|
|
66
|
+
hardGate: boolean;
|
|
67
|
+
rewardEligible: boolean;
|
|
68
|
+
privileged: boolean;
|
|
69
|
+
kind: "model_judge";
|
|
70
|
+
rubricRef: {
|
|
71
|
+
id: string;
|
|
72
|
+
path: string;
|
|
73
|
+
contentHash: string;
|
|
74
|
+
sizeBytes: number;
|
|
75
|
+
mediaType: string;
|
|
76
|
+
visibility: "policy" | "verifier" | "host_private";
|
|
77
|
+
};
|
|
78
|
+
calibrationStatus: "pending" | "passed" | "failed";
|
|
79
|
+
} | {
|
|
80
|
+
id: string;
|
|
81
|
+
version: string;
|
|
82
|
+
weight: number;
|
|
83
|
+
hardGate: boolean;
|
|
84
|
+
rewardEligible: boolean;
|
|
85
|
+
privileged: boolean;
|
|
86
|
+
kind: "custom_verifier";
|
|
87
|
+
verifierRef: {
|
|
88
|
+
id: string;
|
|
89
|
+
path: string;
|
|
90
|
+
contentHash: string;
|
|
91
|
+
sizeBytes: number;
|
|
92
|
+
mediaType: string;
|
|
93
|
+
visibility: "policy" | "verifier" | "host_private";
|
|
94
|
+
};
|
|
95
|
+
timeoutMs: number;
|
|
96
|
+
networkPolicy: "none";
|
|
97
|
+
} | {
|
|
98
|
+
id: string;
|
|
99
|
+
version: string;
|
|
100
|
+
weight: number;
|
|
101
|
+
hardGate: boolean;
|
|
102
|
+
rewardEligible: boolean;
|
|
103
|
+
privileged: boolean;
|
|
104
|
+
kind: "human";
|
|
105
|
+
rubricRef: {
|
|
106
|
+
id: string;
|
|
107
|
+
path: string;
|
|
108
|
+
contentHash: string;
|
|
109
|
+
sizeBytes: number;
|
|
110
|
+
mediaType: string;
|
|
111
|
+
visibility: "policy" | "verifier" | "host_private";
|
|
112
|
+
};
|
|
113
|
+
reviewerRole: string;
|
|
114
|
+
})[];
|
|
115
|
+
metadata: Record<string, unknown>;
|
|
116
|
+
contentHash: string;
|
|
117
|
+
};
|
|
118
|
+
export declare const harnessRefinerBenchmarkAssets: Readonly<Record<string, string>>;
|
|
119
|
+
//# sourceMappingURL=harness-refiner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwuC1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}
|
|
@@ -23,8 +23,8 @@ export declare const completeWorkProcessTraceFixture: {
|
|
|
23
23
|
id: string;
|
|
24
24
|
}[];
|
|
25
25
|
attributes: {
|
|
26
|
-
toolCategory: "agent" | "
|
|
27
|
-
validationKind: "test" | "
|
|
26
|
+
toolCategory: "agent" | "command" | "filesystem" | "source_control" | "browser" | "connected_app" | "sandbox" | "other" | null;
|
|
27
|
+
validationKind: "test" | "structural" | "visual" | "user_review" | "other" | null;
|
|
28
28
|
transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
|
|
29
29
|
interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
|
|
30
30
|
artifactCount: number;
|
|
@@ -200,8 +200,8 @@ export declare const incompleteWorkProcessTraceFixture: {
|
|
|
200
200
|
id: string;
|
|
201
201
|
}[];
|
|
202
202
|
attributes: {
|
|
203
|
-
toolCategory: "agent" | "
|
|
204
|
-
validationKind: "test" | "
|
|
203
|
+
toolCategory: "agent" | "command" | "filesystem" | "source_control" | "browser" | "connected_app" | "sandbox" | "other" | null;
|
|
204
|
+
validationKind: "test" | "structural" | "visual" | "user_review" | "other" | null;
|
|
205
205
|
transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
|
|
206
206
|
interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
|
|
207
207
|
artifactCount: number;
|
|
@@ -406,8 +406,8 @@ export declare const workEvidenceConformance: {
|
|
|
406
406
|
id: string;
|
|
407
407
|
}[];
|
|
408
408
|
attributes: {
|
|
409
|
-
toolCategory: "agent" | "
|
|
410
|
-
validationKind: "test" | "
|
|
409
|
+
toolCategory: "agent" | "command" | "filesystem" | "source_control" | "browser" | "connected_app" | "sandbox" | "other" | null;
|
|
410
|
+
validationKind: "test" | "structural" | "visual" | "user_review" | "other" | null;
|
|
411
411
|
transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
|
|
412
412
|
interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
|
|
413
413
|
artifactCount: number;
|
|
@@ -639,8 +639,8 @@ export declare const workEvidenceConformance: {
|
|
|
639
639
|
id: string;
|
|
640
640
|
}[];
|
|
641
641
|
attributes: {
|
|
642
|
-
toolCategory: "agent" | "
|
|
643
|
-
validationKind: "test" | "
|
|
642
|
+
toolCategory: "agent" | "command" | "filesystem" | "source_control" | "browser" | "connected_app" | "sandbox" | "other" | null;
|
|
643
|
+
validationKind: "test" | "structural" | "visual" | "user_review" | "other" | null;
|
|
644
644
|
transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
|
|
645
645
|
interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
|
|
646
646
|
artifactCount: number;
|
|
@@ -57,9 +57,9 @@ export declare const WorkProcessStepStatusSchema: z.ZodEnum<{
|
|
|
57
57
|
}>;
|
|
58
58
|
export declare const WorkToolCategorySchema: z.ZodEnum<{
|
|
59
59
|
agent: "agent";
|
|
60
|
+
command: "command";
|
|
60
61
|
filesystem: "filesystem";
|
|
61
62
|
source_control: "source_control";
|
|
62
|
-
command: "command";
|
|
63
63
|
browser: "browser";
|
|
64
64
|
connected_app: "connected_app";
|
|
65
65
|
sandbox: "sandbox";
|
|
@@ -67,10 +67,10 @@ export declare const WorkToolCategorySchema: z.ZodEnum<{
|
|
|
67
67
|
}>;
|
|
68
68
|
export declare const WorkValidationKindSchema: z.ZodEnum<{
|
|
69
69
|
test: "test";
|
|
70
|
-
other: "other";
|
|
71
70
|
structural: "structural";
|
|
72
71
|
visual: "visual";
|
|
73
72
|
user_review: "user_review";
|
|
73
|
+
other: "other";
|
|
74
74
|
}>;
|
|
75
75
|
export declare const WorkTransitionStateSchema: z.ZodEnum<{
|
|
76
76
|
timeout: "timeout";
|
|
@@ -162,9 +162,9 @@ export declare const WorkProcessStepSchema: z.ZodObject<{
|
|
|
162
162
|
attributes: z.ZodObject<{
|
|
163
163
|
toolCategory: z.ZodNullable<z.ZodEnum<{
|
|
164
164
|
agent: "agent";
|
|
165
|
+
command: "command";
|
|
165
166
|
filesystem: "filesystem";
|
|
166
167
|
source_control: "source_control";
|
|
167
|
-
command: "command";
|
|
168
168
|
browser: "browser";
|
|
169
169
|
connected_app: "connected_app";
|
|
170
170
|
sandbox: "sandbox";
|
|
@@ -172,10 +172,10 @@ export declare const WorkProcessStepSchema: z.ZodObject<{
|
|
|
172
172
|
}>>;
|
|
173
173
|
validationKind: z.ZodNullable<z.ZodEnum<{
|
|
174
174
|
test: "test";
|
|
175
|
-
other: "other";
|
|
176
175
|
structural: "structural";
|
|
177
176
|
visual: "visual";
|
|
178
177
|
user_review: "user_review";
|
|
178
|
+
other: "other";
|
|
179
179
|
}>>;
|
|
180
180
|
transitionState: z.ZodNullable<z.ZodEnum<{
|
|
181
181
|
timeout: "timeout";
|
|
@@ -280,9 +280,9 @@ export declare const WorkProcessTraceContentSchema: z.ZodObject<{
|
|
|
280
280
|
attributes: z.ZodObject<{
|
|
281
281
|
toolCategory: z.ZodNullable<z.ZodEnum<{
|
|
282
282
|
agent: "agent";
|
|
283
|
+
command: "command";
|
|
283
284
|
filesystem: "filesystem";
|
|
284
285
|
source_control: "source_control";
|
|
285
|
-
command: "command";
|
|
286
286
|
browser: "browser";
|
|
287
287
|
connected_app: "connected_app";
|
|
288
288
|
sandbox: "sandbox";
|
|
@@ -290,10 +290,10 @@ export declare const WorkProcessTraceContentSchema: z.ZodObject<{
|
|
|
290
290
|
}>>;
|
|
291
291
|
validationKind: z.ZodNullable<z.ZodEnum<{
|
|
292
292
|
test: "test";
|
|
293
|
-
other: "other";
|
|
294
293
|
structural: "structural";
|
|
295
294
|
visual: "visual";
|
|
296
295
|
user_review: "user_review";
|
|
296
|
+
other: "other";
|
|
297
297
|
}>>;
|
|
298
298
|
transitionState: z.ZodNullable<z.ZodEnum<{
|
|
299
299
|
timeout: "timeout";
|
|
@@ -399,9 +399,9 @@ export declare const WorkProcessTraceSchema: z.ZodObject<{
|
|
|
399
399
|
attributes: z.ZodObject<{
|
|
400
400
|
toolCategory: z.ZodNullable<z.ZodEnum<{
|
|
401
401
|
agent: "agent";
|
|
402
|
+
command: "command";
|
|
402
403
|
filesystem: "filesystem";
|
|
403
404
|
source_control: "source_control";
|
|
404
|
-
command: "command";
|
|
405
405
|
browser: "browser";
|
|
406
406
|
connected_app: "connected_app";
|
|
407
407
|
sandbox: "sandbox";
|
|
@@ -409,10 +409,10 @@ export declare const WorkProcessTraceSchema: z.ZodObject<{
|
|
|
409
409
|
}>>;
|
|
410
410
|
validationKind: z.ZodNullable<z.ZodEnum<{
|
|
411
411
|
test: "test";
|
|
412
|
-
other: "other";
|
|
413
412
|
structural: "structural";
|
|
414
413
|
visual: "visual";
|
|
415
414
|
user_review: "user_review";
|
|
415
|
+
other: "other";
|
|
416
416
|
}>>;
|
|
417
417
|
transitionState: z.ZodNullable<z.ZodEnum<{
|
|
418
418
|
timeout: "timeout";
|
package/dist/types/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,sCAAsC,CAAC;AACrD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,sCAAsC,CAAC;AACrD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC"}
|
|
@@ -24,7 +24,7 @@ export declare const harnessEvaluationReviewConformance: {
|
|
|
24
24
|
id: string;
|
|
25
25
|
contentHash: string;
|
|
26
26
|
};
|
|
27
|
-
kind: "validation" | "observation" | "taskset" | "trigger" | "route_decision" | "refiner_outcome" | "proposal" | "apply_receipt" | "harness_advance" | "rollback" | "work_outcome" | "
|
|
27
|
+
kind: "validation" | "evaluation" | "observation" | "taskset" | "trigger" | "route_decision" | "refiner_outcome" | "proposal" | "apply_receipt" | "harness_advance" | "rollback" | "work_outcome" | "training_qualification" | "model_candidate";
|
|
28
28
|
sourceRef: string;
|
|
29
29
|
sourcePolicy: {
|
|
30
30
|
policy: {
|
|
@@ -61,7 +61,7 @@ export declare const harnessEvaluationReviewConformance: {
|
|
|
61
61
|
} | null;
|
|
62
62
|
classification: "runtime" | "taskset" | "no_action" | "harness_maintenance" | "product" | "model_improvement";
|
|
63
63
|
triage: {
|
|
64
|
-
layer: "model" | "tools" | "
|
|
64
|
+
layer: "model" | "tools" | "evaluation" | "runtime" | "product" | "harness" | "retrieval";
|
|
65
65
|
status: "resolved" | "not_applicable" | "unresolved" | "blocked";
|
|
66
66
|
reason: string;
|
|
67
67
|
evidenceRefs: {
|
|
@@ -114,7 +114,7 @@ export declare const harnessEvaluationReviewConformance: {
|
|
|
114
114
|
id: string;
|
|
115
115
|
contentHash: string;
|
|
116
116
|
};
|
|
117
|
-
kind: "validation" | "observation" | "taskset" | "trigger" | "route_decision" | "refiner_outcome" | "proposal" | "apply_receipt" | "harness_advance" | "rollback" | "work_outcome" | "
|
|
117
|
+
kind: "validation" | "evaluation" | "observation" | "taskset" | "trigger" | "route_decision" | "refiner_outcome" | "proposal" | "apply_receipt" | "harness_advance" | "rollback" | "work_outcome" | "training_qualification" | "model_candidate";
|
|
118
118
|
sourceRef: string;
|
|
119
119
|
sourcePolicy: {
|
|
120
120
|
policy: {
|
|
@@ -151,7 +151,7 @@ export declare const harnessEvaluationReviewConformance: {
|
|
|
151
151
|
} | null;
|
|
152
152
|
classification: "runtime" | "taskset" | "no_action" | "harness_maintenance" | "product" | "model_improvement";
|
|
153
153
|
triage: {
|
|
154
|
-
layer: "model" | "tools" | "
|
|
154
|
+
layer: "model" | "tools" | "evaluation" | "runtime" | "product" | "harness" | "retrieval";
|
|
155
155
|
status: "resolved" | "not_applicable" | "unresolved" | "blocked";
|
|
156
156
|
reason: string;
|
|
157
157
|
evidenceRefs: {
|
|
@@ -204,7 +204,7 @@ export declare const harnessEvaluationReviewConformance: {
|
|
|
204
204
|
id: string;
|
|
205
205
|
contentHash: string;
|
|
206
206
|
};
|
|
207
|
-
kind: "validation" | "observation" | "taskset" | "trigger" | "route_decision" | "refiner_outcome" | "proposal" | "apply_receipt" | "harness_advance" | "rollback" | "work_outcome" | "
|
|
207
|
+
kind: "validation" | "evaluation" | "observation" | "taskset" | "trigger" | "route_decision" | "refiner_outcome" | "proposal" | "apply_receipt" | "harness_advance" | "rollback" | "work_outcome" | "training_qualification" | "model_candidate";
|
|
208
208
|
sourceRef: string;
|
|
209
209
|
sourcePolicy: {
|
|
210
210
|
policy: {
|
|
@@ -241,7 +241,7 @@ export declare const harnessEvaluationReviewConformance: {
|
|
|
241
241
|
} | null;
|
|
242
242
|
classification: "runtime" | "taskset" | "no_action" | "harness_maintenance" | "product" | "model_improvement";
|
|
243
243
|
triage: {
|
|
244
|
-
layer: "model" | "tools" | "
|
|
244
|
+
layer: "model" | "tools" | "evaluation" | "runtime" | "product" | "harness" | "retrieval";
|
|
245
245
|
status: "resolved" | "not_applicable" | "unresolved" | "blocked";
|
|
246
246
|
reason: string;
|
|
247
247
|
evidenceRefs: {
|
|
@@ -294,7 +294,7 @@ export declare const harnessEvaluationReviewConformance: {
|
|
|
294
294
|
id: string;
|
|
295
295
|
contentHash: string;
|
|
296
296
|
};
|
|
297
|
-
kind: "validation" | "observation" | "taskset" | "trigger" | "route_decision" | "refiner_outcome" | "proposal" | "apply_receipt" | "harness_advance" | "rollback" | "work_outcome" | "
|
|
297
|
+
kind: "validation" | "evaluation" | "observation" | "taskset" | "trigger" | "route_decision" | "refiner_outcome" | "proposal" | "apply_receipt" | "harness_advance" | "rollback" | "work_outcome" | "training_qualification" | "model_candidate";
|
|
298
298
|
sourceRef: string;
|
|
299
299
|
sourcePolicy: {
|
|
300
300
|
policy: {
|
|
@@ -331,7 +331,7 @@ export declare const harnessEvaluationReviewConformance: {
|
|
|
331
331
|
} | null;
|
|
332
332
|
classification: "runtime" | "taskset" | "no_action" | "harness_maintenance" | "product" | "model_improvement";
|
|
333
333
|
triage: {
|
|
334
|
-
layer: "model" | "tools" | "
|
|
334
|
+
layer: "model" | "tools" | "evaluation" | "runtime" | "product" | "harness" | "retrieval";
|
|
335
335
|
status: "resolved" | "not_applicable" | "unresolved" | "blocked";
|
|
336
336
|
reason: string;
|
|
337
337
|
evidenceRefs: {
|
|
@@ -538,7 +538,7 @@ export declare const harnessEvaluationReviewConformance: {
|
|
|
538
538
|
id: string;
|
|
539
539
|
contentHash: string;
|
|
540
540
|
};
|
|
541
|
-
kind: "validation" | "observation" | "taskset" | "trigger" | "route_decision" | "refiner_outcome" | "proposal" | "apply_receipt" | "harness_advance" | "rollback" | "work_outcome" | "
|
|
541
|
+
kind: "validation" | "evaluation" | "observation" | "taskset" | "trigger" | "route_decision" | "refiner_outcome" | "proposal" | "apply_receipt" | "harness_advance" | "rollback" | "work_outcome" | "training_qualification" | "model_candidate";
|
|
542
542
|
sourceRef: string;
|
|
543
543
|
sourcePolicy: {
|
|
544
544
|
policy: {
|
|
@@ -575,7 +575,7 @@ export declare const harnessEvaluationReviewConformance: {
|
|
|
575
575
|
} | null;
|
|
576
576
|
classification: "runtime" | "taskset" | "no_action" | "harness_maintenance" | "product" | "model_improvement";
|
|
577
577
|
triage: {
|
|
578
|
-
layer: "model" | "tools" | "
|
|
578
|
+
layer: "model" | "tools" | "evaluation" | "runtime" | "product" | "harness" | "retrieval";
|
|
579
579
|
status: "resolved" | "not_applicable" | "unresolved" | "blocked";
|
|
580
580
|
reason: string;
|
|
581
581
|
evidenceRefs: {
|