@openpond/evals 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRACT.md +11 -3
- package/README.md +45 -9
- package/RELEASING.md +13 -46
- package/dist/benchmarks.js +218 -0
- package/dist/builtin-benchmarks/harness-refiner.js +1267 -0
- package/dist/compatibility.js +2 -2
- package/dist/conformance.js +1 -2
- package/dist/evidence/authoring.js +1 -1
- package/dist/evidence/conformance.js +1 -1
- package/dist/evidence/contracts.js +1 -2
- package/dist/evidence/eligibility.js +1 -1
- package/dist/graders.js +1 -1
- package/dist/harness.js +33 -101
- package/dist/index.js +3 -3
- package/dist/model-improvement-qualification.js +125 -0
- package/dist/review-conformance.js +209 -0
- package/dist/runs.js +1 -9
- package/dist/tasksets.js +2 -15
- package/dist/types/benchmarks.d.ts +293 -0
- package/dist/types/benchmarks.d.ts.map +1 -0
- package/dist/types/builtin-benchmarks/harness-refiner.d.ts +119 -0
- package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -0
- package/dist/types/compatibility.d.ts +1 -1
- package/dist/types/compatibility.d.ts.map +1 -1
- package/dist/types/conformance.d.ts +2 -0
- package/dist/types/conformance.d.ts.map +1 -1
- package/dist/types/evidence/conformance.d.ts +24 -24
- package/dist/types/evidence/contracts.d.ts +26 -26
- package/dist/types/harness.d.ts +3 -409
- package/dist/types/harness.d.ts.map +1 -1
- package/dist/types/index.d.ts +3 -3
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/model-improvement-qualification.d.ts +222 -0
- package/dist/types/model-improvement-qualification.d.ts.map +1 -0
- package/dist/types/review-conformance.d.ts +607 -0
- package/dist/types/review-conformance.d.ts.map +1 -0
- package/dist/types/runs.d.ts +1 -9
- package/dist/types/runs.d.ts.map +1 -1
- package/dist/types/tasksets.d.ts +2 -25
- package/dist/types/tasksets.d.ts.map +1 -1
- package/package.json +18 -10
- package/dist/common.js +0 -67
- package/dist/harness-improvements.js +0 -329
- package/dist/harness-workspaces.js +0 -368
- package/dist/sha256.js +0 -91
- package/dist/types/common.d.ts +0 -51
- package/dist/types/common.d.ts.map +0 -1
- package/dist/types/harness-improvements.d.ts +0 -523
- package/dist/types/harness-improvements.d.ts.map +0 -1
- package/dist/types/harness-workspaces.d.ts +0 -802
- package/dist/types/harness-workspaces.d.ts.map +0 -1
- package/dist/types/sha256.d.ts +0 -2
- package/dist/types/sha256.d.ts.map +0 -1
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { type AttemptReceipt, type EvaluationResult } from "./runs.js";
|
|
3
|
+
export { harnessRefinerBenchmarkAssets, harnessRefinerBenchmarkRelease, } from "./builtin-benchmarks/harness-refiner.js";
|
|
4
|
+
export declare const BenchmarkMetricSchema: z.ZodEnum<{
|
|
5
|
+
foreground_tokens: "foreground_tokens";
|
|
6
|
+
success_rate: "success_rate";
|
|
7
|
+
latency_ms: "latency_ms";
|
|
8
|
+
cost_usd: "cost_usd";
|
|
9
|
+
}>;
|
|
10
|
+
export declare const BenchmarkRunPhaseSchema: z.ZodEnum<{
|
|
11
|
+
baseline: "baseline";
|
|
12
|
+
candidate: "candidate";
|
|
13
|
+
}>;
|
|
14
|
+
export declare const BenchmarkProtocolSchema: z.ZodObject<{
|
|
15
|
+
split: z.ZodEnum<{
|
|
16
|
+
train: "train";
|
|
17
|
+
validation: "validation";
|
|
18
|
+
test: "test";
|
|
19
|
+
frozen_eval: "frozen_eval";
|
|
20
|
+
}>;
|
|
21
|
+
taskIds: z.ZodArray<z.ZodString>;
|
|
22
|
+
seeds: z.ZodArray<z.ZodString>;
|
|
23
|
+
repetitions: z.ZodNumber;
|
|
24
|
+
runtimeTargetHash: z.ZodString;
|
|
25
|
+
environmentHash: z.ZodString;
|
|
26
|
+
toolContractHash: z.ZodString;
|
|
27
|
+
limitsHash: z.ZodString;
|
|
28
|
+
}, z.core.$strict>;
|
|
29
|
+
export declare const BenchmarkDefinitionSchema: z.ZodObject<{
|
|
30
|
+
schemaVersion: z.ZodLiteral<"openpond.benchmarkDefinition.v1">;
|
|
31
|
+
id: z.ZodString;
|
|
32
|
+
title: z.ZodString;
|
|
33
|
+
description: z.ZodString;
|
|
34
|
+
tasksetRelease: z.ZodObject<{
|
|
35
|
+
id: z.ZodString;
|
|
36
|
+
contentHash: z.ZodString;
|
|
37
|
+
}, z.core.$strict>;
|
|
38
|
+
adaptationSplit: z.ZodEnum<{
|
|
39
|
+
train: "train";
|
|
40
|
+
validation: "validation";
|
|
41
|
+
test: "test";
|
|
42
|
+
frozen_eval: "frozen_eval";
|
|
43
|
+
}>;
|
|
44
|
+
evaluationSplit: z.ZodEnum<{
|
|
45
|
+
train: "train";
|
|
46
|
+
validation: "validation";
|
|
47
|
+
test: "test";
|
|
48
|
+
frozen_eval: "frozen_eval";
|
|
49
|
+
}>;
|
|
50
|
+
primaryMetric: z.ZodEnum<{
|
|
51
|
+
foreground_tokens: "foreground_tokens";
|
|
52
|
+
success_rate: "success_rate";
|
|
53
|
+
latency_ms: "latency_ms";
|
|
54
|
+
cost_usd: "cost_usd";
|
|
55
|
+
}>;
|
|
56
|
+
qualityGate: z.ZodEnum<{
|
|
57
|
+
none: "none";
|
|
58
|
+
non_regression: "non_regression";
|
|
59
|
+
all_pass: "all_pass";
|
|
60
|
+
}>;
|
|
61
|
+
caseCounts: z.ZodObject<{
|
|
62
|
+
adaptation: z.ZodNumber;
|
|
63
|
+
evaluation: z.ZodNumber;
|
|
64
|
+
}, z.core.$strict>;
|
|
65
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
66
|
+
}, z.core.$strict>;
|
|
67
|
+
export declare const BenchmarkRunRequestSchema: z.ZodObject<{
|
|
68
|
+
schemaVersion: z.ZodLiteral<"openpond.benchmarkRunRequest.v1">;
|
|
69
|
+
phase: z.ZodEnum<{
|
|
70
|
+
baseline: "baseline";
|
|
71
|
+
candidate: "candidate";
|
|
72
|
+
}>;
|
|
73
|
+
model: z.ZodObject<{
|
|
74
|
+
provider: z.ZodString;
|
|
75
|
+
model: z.ZodString;
|
|
76
|
+
revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
77
|
+
artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
78
|
+
tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
79
|
+
chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
80
|
+
}, z.core.$strict>;
|
|
81
|
+
reasoningEffort: z.ZodNullable<z.ZodString>;
|
|
82
|
+
split: z.ZodEnum<{
|
|
83
|
+
train: "train";
|
|
84
|
+
validation: "validation";
|
|
85
|
+
test: "test";
|
|
86
|
+
frozen_eval: "frozen_eval";
|
|
87
|
+
}>;
|
|
88
|
+
seeds: z.ZodArray<z.ZodString>;
|
|
89
|
+
repetitions: z.ZodNumber;
|
|
90
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
91
|
+
}, z.core.$strict>;
|
|
92
|
+
export declare const BenchmarkRunSummaryContentSchema: z.ZodObject<{
|
|
93
|
+
schemaVersion: z.ZodLiteral<"openpond.benchmarkRunSummary.v1">;
|
|
94
|
+
id: z.ZodString;
|
|
95
|
+
phase: z.ZodEnum<{
|
|
96
|
+
baseline: "baseline";
|
|
97
|
+
candidate: "candidate";
|
|
98
|
+
}>;
|
|
99
|
+
tasksetRelease: z.ZodObject<{
|
|
100
|
+
id: z.ZodString;
|
|
101
|
+
contentHash: z.ZodString;
|
|
102
|
+
}, z.core.$strict>;
|
|
103
|
+
harnessRelease: z.ZodObject<{
|
|
104
|
+
id: z.ZodString;
|
|
105
|
+
contentHash: z.ZodString;
|
|
106
|
+
}, z.core.$strict>;
|
|
107
|
+
evaluationResult: z.ZodObject<{
|
|
108
|
+
id: z.ZodString;
|
|
109
|
+
contentHash: z.ZodString;
|
|
110
|
+
}, z.core.$strict>;
|
|
111
|
+
model: z.ZodObject<{
|
|
112
|
+
provider: z.ZodString;
|
|
113
|
+
model: z.ZodString;
|
|
114
|
+
revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
115
|
+
artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
116
|
+
tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
117
|
+
chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
118
|
+
}, z.core.$strict>;
|
|
119
|
+
reasoningEffort: z.ZodNullable<z.ZodString>;
|
|
120
|
+
protocol: z.ZodObject<{
|
|
121
|
+
split: z.ZodEnum<{
|
|
122
|
+
train: "train";
|
|
123
|
+
validation: "validation";
|
|
124
|
+
test: "test";
|
|
125
|
+
frozen_eval: "frozen_eval";
|
|
126
|
+
}>;
|
|
127
|
+
taskIds: z.ZodArray<z.ZodString>;
|
|
128
|
+
seeds: z.ZodArray<z.ZodString>;
|
|
129
|
+
repetitions: z.ZodNumber;
|
|
130
|
+
runtimeTargetHash: z.ZodString;
|
|
131
|
+
environmentHash: z.ZodString;
|
|
132
|
+
toolContractHash: z.ZodString;
|
|
133
|
+
limitsHash: z.ZodString;
|
|
134
|
+
}, z.core.$strict>;
|
|
135
|
+
attemptCount: z.ZodNumber;
|
|
136
|
+
passedCount: z.ZodNumber;
|
|
137
|
+
terminalCount: z.ZodNumber;
|
|
138
|
+
usage: z.ZodObject<{
|
|
139
|
+
inputTokens: z.ZodNumber;
|
|
140
|
+
outputTokens: z.ZodNumber;
|
|
141
|
+
totalTokens: z.ZodNumber;
|
|
142
|
+
}, z.core.$strict>;
|
|
143
|
+
costUsd: z.ZodNullable<z.ZodNumber>;
|
|
144
|
+
latencyMs: z.ZodNumber;
|
|
145
|
+
createdAt: z.ZodString;
|
|
146
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
147
|
+
}, z.core.$strict>;
|
|
148
|
+
export declare const BenchmarkRunSummarySchema: z.ZodObject<{
|
|
149
|
+
schemaVersion: z.ZodLiteral<"openpond.benchmarkRunSummary.v1">;
|
|
150
|
+
id: z.ZodString;
|
|
151
|
+
phase: z.ZodEnum<{
|
|
152
|
+
baseline: "baseline";
|
|
153
|
+
candidate: "candidate";
|
|
154
|
+
}>;
|
|
155
|
+
tasksetRelease: z.ZodObject<{
|
|
156
|
+
id: z.ZodString;
|
|
157
|
+
contentHash: z.ZodString;
|
|
158
|
+
}, z.core.$strict>;
|
|
159
|
+
harnessRelease: z.ZodObject<{
|
|
160
|
+
id: z.ZodString;
|
|
161
|
+
contentHash: z.ZodString;
|
|
162
|
+
}, z.core.$strict>;
|
|
163
|
+
evaluationResult: z.ZodObject<{
|
|
164
|
+
id: z.ZodString;
|
|
165
|
+
contentHash: z.ZodString;
|
|
166
|
+
}, z.core.$strict>;
|
|
167
|
+
model: z.ZodObject<{
|
|
168
|
+
provider: z.ZodString;
|
|
169
|
+
model: z.ZodString;
|
|
170
|
+
revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
171
|
+
artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
172
|
+
tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
173
|
+
chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
174
|
+
}, z.core.$strict>;
|
|
175
|
+
reasoningEffort: z.ZodNullable<z.ZodString>;
|
|
176
|
+
protocol: z.ZodObject<{
|
|
177
|
+
split: z.ZodEnum<{
|
|
178
|
+
train: "train";
|
|
179
|
+
validation: "validation";
|
|
180
|
+
test: "test";
|
|
181
|
+
frozen_eval: "frozen_eval";
|
|
182
|
+
}>;
|
|
183
|
+
taskIds: z.ZodArray<z.ZodString>;
|
|
184
|
+
seeds: z.ZodArray<z.ZodString>;
|
|
185
|
+
repetitions: z.ZodNumber;
|
|
186
|
+
runtimeTargetHash: z.ZodString;
|
|
187
|
+
environmentHash: z.ZodString;
|
|
188
|
+
toolContractHash: z.ZodString;
|
|
189
|
+
limitsHash: z.ZodString;
|
|
190
|
+
}, z.core.$strict>;
|
|
191
|
+
attemptCount: z.ZodNumber;
|
|
192
|
+
passedCount: z.ZodNumber;
|
|
193
|
+
terminalCount: z.ZodNumber;
|
|
194
|
+
usage: z.ZodObject<{
|
|
195
|
+
inputTokens: z.ZodNumber;
|
|
196
|
+
outputTokens: z.ZodNumber;
|
|
197
|
+
totalTokens: z.ZodNumber;
|
|
198
|
+
}, z.core.$strict>;
|
|
199
|
+
costUsd: z.ZodNullable<z.ZodNumber>;
|
|
200
|
+
latencyMs: z.ZodNumber;
|
|
201
|
+
createdAt: z.ZodString;
|
|
202
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
203
|
+
contentHash: z.ZodString;
|
|
204
|
+
}, z.core.$strict>;
|
|
205
|
+
export declare const BenchmarkComparisonContentSchema: z.ZodObject<{
|
|
206
|
+
schemaVersion: z.ZodLiteral<"openpond.benchmarkComparison.v1">;
|
|
207
|
+
id: z.ZodString;
|
|
208
|
+
baseline: z.ZodObject<{
|
|
209
|
+
id: z.ZodString;
|
|
210
|
+
contentHash: z.ZodString;
|
|
211
|
+
}, z.core.$strict>;
|
|
212
|
+
candidate: z.ZodObject<{
|
|
213
|
+
id: z.ZodString;
|
|
214
|
+
contentHash: z.ZodString;
|
|
215
|
+
}, z.core.$strict>;
|
|
216
|
+
tasksetRelease: z.ZodObject<{
|
|
217
|
+
id: z.ZodString;
|
|
218
|
+
contentHash: z.ZodString;
|
|
219
|
+
}, z.core.$strict>;
|
|
220
|
+
primaryMetric: z.ZodEnum<{
|
|
221
|
+
foreground_tokens: "foreground_tokens";
|
|
222
|
+
success_rate: "success_rate";
|
|
223
|
+
latency_ms: "latency_ms";
|
|
224
|
+
cost_usd: "cost_usd";
|
|
225
|
+
}>;
|
|
226
|
+
qualityPassed: z.ZodBoolean;
|
|
227
|
+
baselinePassRate: z.ZodNumber;
|
|
228
|
+
candidatePassRate: z.ZodNumber;
|
|
229
|
+
foregroundTokenDelta: z.ZodNumber;
|
|
230
|
+
foregroundTokenDeltaPercent: z.ZodNullable<z.ZodNumber>;
|
|
231
|
+
improved: z.ZodBoolean;
|
|
232
|
+
createdAt: z.ZodString;
|
|
233
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
234
|
+
}, z.core.$strict>;
|
|
235
|
+
export declare const BenchmarkComparisonSchema: z.ZodObject<{
|
|
236
|
+
schemaVersion: z.ZodLiteral<"openpond.benchmarkComparison.v1">;
|
|
237
|
+
id: z.ZodString;
|
|
238
|
+
baseline: z.ZodObject<{
|
|
239
|
+
id: z.ZodString;
|
|
240
|
+
contentHash: z.ZodString;
|
|
241
|
+
}, z.core.$strict>;
|
|
242
|
+
candidate: z.ZodObject<{
|
|
243
|
+
id: z.ZodString;
|
|
244
|
+
contentHash: z.ZodString;
|
|
245
|
+
}, z.core.$strict>;
|
|
246
|
+
tasksetRelease: z.ZodObject<{
|
|
247
|
+
id: z.ZodString;
|
|
248
|
+
contentHash: z.ZodString;
|
|
249
|
+
}, z.core.$strict>;
|
|
250
|
+
primaryMetric: z.ZodEnum<{
|
|
251
|
+
foreground_tokens: "foreground_tokens";
|
|
252
|
+
success_rate: "success_rate";
|
|
253
|
+
latency_ms: "latency_ms";
|
|
254
|
+
cost_usd: "cost_usd";
|
|
255
|
+
}>;
|
|
256
|
+
qualityPassed: z.ZodBoolean;
|
|
257
|
+
baselinePassRate: z.ZodNumber;
|
|
258
|
+
candidatePassRate: z.ZodNumber;
|
|
259
|
+
foregroundTokenDelta: z.ZodNumber;
|
|
260
|
+
foregroundTokenDeltaPercent: z.ZodNullable<z.ZodNumber>;
|
|
261
|
+
improved: z.ZodBoolean;
|
|
262
|
+
createdAt: z.ZodString;
|
|
263
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
264
|
+
contentHash: z.ZodString;
|
|
265
|
+
}, z.core.$strict>;
|
|
266
|
+
export declare function createBenchmarkDefinition(input: z.input<typeof BenchmarkDefinitionSchema>): BenchmarkDefinition;
|
|
267
|
+
export declare function createBenchmarkRunSummary(input: {
|
|
268
|
+
id: string;
|
|
269
|
+
phase: BenchmarkRunPhase;
|
|
270
|
+
evaluation: EvaluationResult;
|
|
271
|
+
receipts: AttemptReceipt[];
|
|
272
|
+
reasoningEffort: string | null;
|
|
273
|
+
protocol: BenchmarkProtocol;
|
|
274
|
+
createdAt: string;
|
|
275
|
+
metadata?: Record<string, unknown>;
|
|
276
|
+
}): BenchmarkRunSummary;
|
|
277
|
+
export declare function compareBenchmarkRuns(input: {
|
|
278
|
+
id: string;
|
|
279
|
+
baseline: BenchmarkRunSummary;
|
|
280
|
+
candidate: BenchmarkRunSummary;
|
|
281
|
+
primaryMetric: BenchmarkMetric;
|
|
282
|
+
qualityGate: "none" | "non_regression" | "all_pass";
|
|
283
|
+
createdAt: string;
|
|
284
|
+
metadata?: Record<string, unknown>;
|
|
285
|
+
}): BenchmarkComparison;
|
|
286
|
+
export type BenchmarkMetric = z.infer<typeof BenchmarkMetricSchema>;
|
|
287
|
+
export type BenchmarkRunPhase = z.infer<typeof BenchmarkRunPhaseSchema>;
|
|
288
|
+
export type BenchmarkProtocol = z.infer<typeof BenchmarkProtocolSchema>;
|
|
289
|
+
export type BenchmarkDefinition = z.infer<typeof BenchmarkDefinitionSchema>;
|
|
290
|
+
export type BenchmarkRunRequest = z.infer<typeof BenchmarkRunRequestSchema>;
|
|
291
|
+
export type BenchmarkRunSummary = z.infer<typeof BenchmarkRunSummarySchema>;
|
|
292
|
+
export type BenchmarkComparison = z.infer<typeof BenchmarkComparisonSchema>;
|
|
293
|
+
//# sourceMappingURL=benchmarks.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"benchmarks.d.ts","sourceRoot":"","sources":["../../../../src/benchmarks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAGvE,OAAO,EACL,6BAA6B,EAC7B,8BAA8B,GAC/B,MAAM,yCAAyC,CAAC;AAEjD,eAAO,MAAM,qBAAqB;;;;;EAKhC,CAAC;AACH,eAAO,MAAM,uBAAuB;;;EAAoC,CAAC;AAEzE,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;kBASzB,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAe3B,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;kBAS3B,CAAC;AAQZ,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkBlC,CAAC;AACZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE3B,CAAC;AAEZ,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAelC,CAAC;AACZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE3B,CAAC;AAEZ,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,GAC/C,mBAAmB,CAErB;AAED,wBAAgB,yBAAyB,CAAC,KAAK,EAAE;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,iBAAiB,CAAC;IACzB,UAAU,EAAE,gBAAgB,CAAC;IAC7B,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,mBAAmB,CAuCtB;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,SAAS,EAAE,mBAAmB,CAAC;IAC/B,aAAa,EAAE,eAAe,CAAC;IAC/B,WAAW,EAAE,MAAM,GAAG,gBAAgB,GAAG,UAAU,CAAC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,mBAAmB,CAuDtB;AAkDD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
export declare const harnessRefinerBenchmarkRelease: {
|
|
2
|
+
schemaVersion: "openpond.tasksetRelease.v2";
|
|
3
|
+
id: string;
|
|
4
|
+
revision: number;
|
|
5
|
+
policy: {
|
|
6
|
+
policyVisibleFields: string[];
|
|
7
|
+
privilegedFields: string[];
|
|
8
|
+
hiddenGraderRefs: string[];
|
|
9
|
+
connectedAppScopes: string[];
|
|
10
|
+
};
|
|
11
|
+
environment: {
|
|
12
|
+
protocolVersion: "openpond.environment.v1";
|
|
13
|
+
kind: "text" | "agent" | "work" | "custom_program";
|
|
14
|
+
entrypoint: string;
|
|
15
|
+
stateful: boolean;
|
|
16
|
+
deterministicSeeds: boolean;
|
|
17
|
+
lifecycle: ("create" | "reset" | "step" | "collect" | "destroy")[];
|
|
18
|
+
networkPolicy: "none" | "declared_read_only" | "declared_scoped";
|
|
19
|
+
defaultTimeoutMs: number;
|
|
20
|
+
};
|
|
21
|
+
tools: {
|
|
22
|
+
name: string;
|
|
23
|
+
description: string;
|
|
24
|
+
inputSchema: Record<string, unknown>;
|
|
25
|
+
inputSchemaHash: string;
|
|
26
|
+
sideEffect: "read" | "write";
|
|
27
|
+
timeoutMs: number;
|
|
28
|
+
}[];
|
|
29
|
+
capabilities: {
|
|
30
|
+
id: string;
|
|
31
|
+
required: boolean;
|
|
32
|
+
scopes: string[];
|
|
33
|
+
portability: "portable" | "host_adapter" | "local_only" | "hosted_only";
|
|
34
|
+
}[];
|
|
35
|
+
tasks: {
|
|
36
|
+
id: string;
|
|
37
|
+
clusterKey: string;
|
|
38
|
+
split: "train" | "validation" | "test" | "frozen_eval";
|
|
39
|
+
input: Record<string, unknown>;
|
|
40
|
+
expectedOutput: Record<string, unknown> | null;
|
|
41
|
+
policyVisibleContext: Record<string, unknown>;
|
|
42
|
+
privilegedContextRef: string | null;
|
|
43
|
+
artifactRefs: {
|
|
44
|
+
id: string;
|
|
45
|
+
path: string;
|
|
46
|
+
contentHash: string;
|
|
47
|
+
sizeBytes: number;
|
|
48
|
+
mediaType: string;
|
|
49
|
+
visibility: "policy" | "verifier" | "host_private";
|
|
50
|
+
}[];
|
|
51
|
+
tags: string[];
|
|
52
|
+
}[];
|
|
53
|
+
graders: ({
|
|
54
|
+
id: string;
|
|
55
|
+
version: string;
|
|
56
|
+
weight: number;
|
|
57
|
+
hardGate: boolean;
|
|
58
|
+
rewardEligible: boolean;
|
|
59
|
+
privileged: boolean;
|
|
60
|
+
kind: "content" | "schema" | "artifact" | "runtime_event" | "state";
|
|
61
|
+
config: Record<string, unknown>;
|
|
62
|
+
} | {
|
|
63
|
+
id: string;
|
|
64
|
+
version: string;
|
|
65
|
+
weight: number;
|
|
66
|
+
hardGate: boolean;
|
|
67
|
+
rewardEligible: boolean;
|
|
68
|
+
privileged: boolean;
|
|
69
|
+
kind: "model_judge";
|
|
70
|
+
rubricRef: {
|
|
71
|
+
id: string;
|
|
72
|
+
path: string;
|
|
73
|
+
contentHash: string;
|
|
74
|
+
sizeBytes: number;
|
|
75
|
+
mediaType: string;
|
|
76
|
+
visibility: "policy" | "verifier" | "host_private";
|
|
77
|
+
};
|
|
78
|
+
calibrationStatus: "pending" | "passed" | "failed";
|
|
79
|
+
} | {
|
|
80
|
+
id: string;
|
|
81
|
+
version: string;
|
|
82
|
+
weight: number;
|
|
83
|
+
hardGate: boolean;
|
|
84
|
+
rewardEligible: boolean;
|
|
85
|
+
privileged: boolean;
|
|
86
|
+
kind: "custom_verifier";
|
|
87
|
+
verifierRef: {
|
|
88
|
+
id: string;
|
|
89
|
+
path: string;
|
|
90
|
+
contentHash: string;
|
|
91
|
+
sizeBytes: number;
|
|
92
|
+
mediaType: string;
|
|
93
|
+
visibility: "policy" | "verifier" | "host_private";
|
|
94
|
+
};
|
|
95
|
+
timeoutMs: number;
|
|
96
|
+
networkPolicy: "none";
|
|
97
|
+
} | {
|
|
98
|
+
id: string;
|
|
99
|
+
version: string;
|
|
100
|
+
weight: number;
|
|
101
|
+
hardGate: boolean;
|
|
102
|
+
rewardEligible: boolean;
|
|
103
|
+
privileged: boolean;
|
|
104
|
+
kind: "human";
|
|
105
|
+
rubricRef: {
|
|
106
|
+
id: string;
|
|
107
|
+
path: string;
|
|
108
|
+
contentHash: string;
|
|
109
|
+
sizeBytes: number;
|
|
110
|
+
mediaType: string;
|
|
111
|
+
visibility: "policy" | "verifier" | "host_private";
|
|
112
|
+
};
|
|
113
|
+
reviewerRole: string;
|
|
114
|
+
})[];
|
|
115
|
+
metadata: Record<string, unknown>;
|
|
116
|
+
contentHash: string;
|
|
117
|
+
};
|
|
118
|
+
export declare const harnessRefinerBenchmarkAssets: Readonly<Record<string, string>>;
|
|
119
|
+
//# sourceMappingURL=harness-refiner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuuC1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type HarnessRelease } from "
|
|
1
|
+
import { type HarnessRelease } from "@openpond/harness";
|
|
2
2
|
import { type HarnessCompatibilityReceipt } from "./runs.js";
|
|
3
3
|
import { type TasksetRelease } from "./tasksets.js";
|
|
4
4
|
export declare function createVerifiedHarnessCompatibilityReceipt(input: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compatibility.d.ts","sourceRoot":"","sources":["../../../../src/compatibility.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,
|
|
1
|
+
{"version":3,"file":"compatibility.d.ts","sourceRoot":"","sources":["../../../../src/compatibility.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,WAAW,CAAC;AACnB,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,eAAe,CAAC;AAEvB,wBAAgB,yCAAyC,CAAC,KAAK,EAAE;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,kBAAkB,EAAE,cAAc,CAAC;IACnC,uBAAuB,EAAE,cAAc,CAAC;IACxC,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,2BAA2B,CA4C9B"}
|
|
@@ -42,6 +42,7 @@ export declare const genericToolConformance: {
|
|
|
42
42
|
id: string;
|
|
43
43
|
required: boolean;
|
|
44
44
|
scopes: string[];
|
|
45
|
+
portability: "portable" | "host_adapter" | "local_only" | "hosted_only";
|
|
45
46
|
}[];
|
|
46
47
|
dependencyLock: {
|
|
47
48
|
id: string;
|
|
@@ -308,6 +309,7 @@ export declare const marketingPortfolioConformance: {
|
|
|
308
309
|
id: string;
|
|
309
310
|
required: boolean;
|
|
310
311
|
scopes: string[];
|
|
312
|
+
portability: "portable" | "host_adapter" | "local_only" | "hosted_only";
|
|
311
313
|
}[];
|
|
312
314
|
dependencyLock: {
|
|
313
315
|
id: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAEjC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGxC,CAAC"}
|
|
@@ -9,8 +9,8 @@ export declare const completeWorkProcessTraceFixture: {
|
|
|
9
9
|
sequence: number;
|
|
10
10
|
timestamp: string;
|
|
11
11
|
layer: "agent" | "environment";
|
|
12
|
-
kind: "
|
|
13
|
-
action: "
|
|
12
|
+
kind: "approval" | "validation" | "artifact" | "tool" | "state_transition" | "question" | "cleanup";
|
|
13
|
+
action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
|
|
14
14
|
status: "cancelled" | "failed" | "started" | "completed";
|
|
15
15
|
inputHash: string | null;
|
|
16
16
|
outputHash: string | null;
|
|
@@ -23,8 +23,8 @@ export declare const completeWorkProcessTraceFixture: {
|
|
|
23
23
|
id: string;
|
|
24
24
|
}[];
|
|
25
25
|
attributes: {
|
|
26
|
-
toolCategory: "agent" | "
|
|
27
|
-
validationKind: "test" | "
|
|
26
|
+
toolCategory: "agent" | "command" | "filesystem" | "source_control" | "browser" | "connected_app" | "sandbox" | "other" | null;
|
|
27
|
+
validationKind: "test" | "structural" | "visual" | "user_review" | "other" | null;
|
|
28
28
|
transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
|
|
29
29
|
interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
|
|
30
30
|
artifactCount: number;
|
|
@@ -32,7 +32,7 @@ export declare const completeWorkProcessTraceFixture: {
|
|
|
32
32
|
durationMs: number | null;
|
|
33
33
|
cpuTimeMs: number | null;
|
|
34
34
|
memoryPeakBytes: number | null;
|
|
35
|
-
errorClass: "unknown" | "
|
|
35
|
+
errorClass: "unknown" | "timeout" | "cancelled" | "validation" | "policy" | "environment" | "infrastructure" | null;
|
|
36
36
|
};
|
|
37
37
|
}[];
|
|
38
38
|
contentHash: string;
|
|
@@ -41,7 +41,7 @@ export declare const completeWorkEvidenceFixture: {
|
|
|
41
41
|
schemaVersion: "openpond.workEvidenceReceipt.v1";
|
|
42
42
|
id: string;
|
|
43
43
|
source: {
|
|
44
|
-
surface: "
|
|
44
|
+
surface: "desktop" | "hosted";
|
|
45
45
|
experience: "work" | "development";
|
|
46
46
|
opaqueRef: string;
|
|
47
47
|
revisionHash: string;
|
|
@@ -186,8 +186,8 @@ export declare const incompleteWorkProcessTraceFixture: {
|
|
|
186
186
|
sequence: number;
|
|
187
187
|
timestamp: string;
|
|
188
188
|
layer: "agent" | "environment";
|
|
189
|
-
kind: "
|
|
190
|
-
action: "
|
|
189
|
+
kind: "approval" | "validation" | "artifact" | "tool" | "state_transition" | "question" | "cleanup";
|
|
190
|
+
action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
|
|
191
191
|
status: "cancelled" | "failed" | "started" | "completed";
|
|
192
192
|
inputHash: string | null;
|
|
193
193
|
outputHash: string | null;
|
|
@@ -200,8 +200,8 @@ export declare const incompleteWorkProcessTraceFixture: {
|
|
|
200
200
|
id: string;
|
|
201
201
|
}[];
|
|
202
202
|
attributes: {
|
|
203
|
-
toolCategory: "agent" | "
|
|
204
|
-
validationKind: "test" | "
|
|
203
|
+
toolCategory: "agent" | "command" | "filesystem" | "source_control" | "browser" | "connected_app" | "sandbox" | "other" | null;
|
|
204
|
+
validationKind: "test" | "structural" | "visual" | "user_review" | "other" | null;
|
|
205
205
|
transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
|
|
206
206
|
interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
|
|
207
207
|
artifactCount: number;
|
|
@@ -209,7 +209,7 @@ export declare const incompleteWorkProcessTraceFixture: {
|
|
|
209
209
|
durationMs: number | null;
|
|
210
210
|
cpuTimeMs: number | null;
|
|
211
211
|
memoryPeakBytes: number | null;
|
|
212
|
-
errorClass: "unknown" | "
|
|
212
|
+
errorClass: "unknown" | "timeout" | "cancelled" | "validation" | "policy" | "environment" | "infrastructure" | null;
|
|
213
213
|
};
|
|
214
214
|
}[];
|
|
215
215
|
contentHash: string;
|
|
@@ -273,7 +273,7 @@ export declare const revokedWorkEvidenceEligibilityFixture: {
|
|
|
273
273
|
export declare const invalidRawEvidenceFixture: {
|
|
274
274
|
source: {
|
|
275
275
|
opaqueRef: string;
|
|
276
|
-
surface: "
|
|
276
|
+
surface: "desktop" | "hosted";
|
|
277
277
|
experience: "work" | "development";
|
|
278
278
|
revisionHash: string;
|
|
279
279
|
};
|
|
@@ -392,8 +392,8 @@ export declare const workEvidenceConformance: {
|
|
|
392
392
|
sequence: number;
|
|
393
393
|
timestamp: string;
|
|
394
394
|
layer: "agent" | "environment";
|
|
395
|
-
kind: "
|
|
396
|
-
action: "
|
|
395
|
+
kind: "approval" | "validation" | "artifact" | "tool" | "state_transition" | "question" | "cleanup";
|
|
396
|
+
action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
|
|
397
397
|
status: "cancelled" | "failed" | "started" | "completed";
|
|
398
398
|
inputHash: string | null;
|
|
399
399
|
outputHash: string | null;
|
|
@@ -406,8 +406,8 @@ export declare const workEvidenceConformance: {
|
|
|
406
406
|
id: string;
|
|
407
407
|
}[];
|
|
408
408
|
attributes: {
|
|
409
|
-
toolCategory: "agent" | "
|
|
410
|
-
validationKind: "test" | "
|
|
409
|
+
toolCategory: "agent" | "command" | "filesystem" | "source_control" | "browser" | "connected_app" | "sandbox" | "other" | null;
|
|
410
|
+
validationKind: "test" | "structural" | "visual" | "user_review" | "other" | null;
|
|
411
411
|
transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
|
|
412
412
|
interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
|
|
413
413
|
artifactCount: number;
|
|
@@ -415,7 +415,7 @@ export declare const workEvidenceConformance: {
|
|
|
415
415
|
durationMs: number | null;
|
|
416
416
|
cpuTimeMs: number | null;
|
|
417
417
|
memoryPeakBytes: number | null;
|
|
418
|
-
errorClass: "unknown" | "
|
|
418
|
+
errorClass: "unknown" | "timeout" | "cancelled" | "validation" | "policy" | "environment" | "infrastructure" | null;
|
|
419
419
|
};
|
|
420
420
|
}[];
|
|
421
421
|
contentHash: string;
|
|
@@ -424,7 +424,7 @@ export declare const workEvidenceConformance: {
|
|
|
424
424
|
schemaVersion: "openpond.workEvidenceReceipt.v1";
|
|
425
425
|
id: string;
|
|
426
426
|
source: {
|
|
427
|
-
surface: "
|
|
427
|
+
surface: "desktop" | "hosted";
|
|
428
428
|
experience: "work" | "development";
|
|
429
429
|
opaqueRef: string;
|
|
430
430
|
revisionHash: string;
|
|
@@ -625,8 +625,8 @@ export declare const workEvidenceConformance: {
|
|
|
625
625
|
sequence: number;
|
|
626
626
|
timestamp: string;
|
|
627
627
|
layer: "agent" | "environment";
|
|
628
|
-
kind: "
|
|
629
|
-
action: "
|
|
628
|
+
kind: "approval" | "validation" | "artifact" | "tool" | "state_transition" | "question" | "cleanup";
|
|
629
|
+
action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
|
|
630
630
|
status: "cancelled" | "failed" | "started" | "completed";
|
|
631
631
|
inputHash: string | null;
|
|
632
632
|
outputHash: string | null;
|
|
@@ -639,8 +639,8 @@ export declare const workEvidenceConformance: {
|
|
|
639
639
|
id: string;
|
|
640
640
|
}[];
|
|
641
641
|
attributes: {
|
|
642
|
-
toolCategory: "agent" | "
|
|
643
|
-
validationKind: "test" | "
|
|
642
|
+
toolCategory: "agent" | "command" | "filesystem" | "source_control" | "browser" | "connected_app" | "sandbox" | "other" | null;
|
|
643
|
+
validationKind: "test" | "structural" | "visual" | "user_review" | "other" | null;
|
|
644
644
|
transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
|
|
645
645
|
interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
|
|
646
646
|
artifactCount: number;
|
|
@@ -648,7 +648,7 @@ export declare const workEvidenceConformance: {
|
|
|
648
648
|
durationMs: number | null;
|
|
649
649
|
cpuTimeMs: number | null;
|
|
650
650
|
memoryPeakBytes: number | null;
|
|
651
|
-
errorClass: "unknown" | "
|
|
651
|
+
errorClass: "unknown" | "timeout" | "cancelled" | "validation" | "policy" | "environment" | "infrastructure" | null;
|
|
652
652
|
};
|
|
653
653
|
}[];
|
|
654
654
|
contentHash: string;
|
|
@@ -656,7 +656,7 @@ export declare const workEvidenceConformance: {
|
|
|
656
656
|
readonly invalidRawEvidence: {
|
|
657
657
|
source: {
|
|
658
658
|
opaqueRef: string;
|
|
659
|
-
surface: "
|
|
659
|
+
surface: "desktop" | "hosted";
|
|
660
660
|
experience: "work" | "development";
|
|
661
661
|
revisionHash: string;
|
|
662
662
|
};
|