@openpond/evals 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRACT.md +13 -2
- package/RELEASING.md +6 -5
- package/dist/compatibility.js +45 -0
- package/dist/conformance.js +6 -9
- package/dist/harness-improvements.js +329 -0
- package/dist/harness-workspaces.js +368 -0
- package/dist/harness.js +4 -6
- package/dist/index.js +3 -0
- package/dist/runs.js +33 -5
- package/dist/tasksets.js +2 -3
- package/dist/types/compatibility.d.ts +11 -0
- package/dist/types/compatibility.d.ts.map +1 -0
- package/dist/types/conformance.d.ts +8 -48
- package/dist/types/conformance.d.ts.map +1 -1
- package/dist/types/evidence/conformance.d.ts +12 -12
- package/dist/types/evidence/contracts.d.ts +10 -10
- package/dist/types/harness-improvements.d.ts +523 -0
- package/dist/types/harness-improvements.d.ts.map +1 -0
- package/dist/types/harness-workspaces.d.ts +802 -0
- package/dist/types/harness-workspaces.d.ts.map +1 -0
- package/dist/types/harness.d.ts +6 -68
- package/dist/types/harness.d.ts.map +1 -1
- package/dist/types/index.d.ts +3 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/runs.d.ts +46 -1
- package/dist/types/runs.d.ts.map +1 -1
- package/dist/types/tasksets.d.ts +2 -10
- package/dist/types/tasksets.d.ts.map +1 -1
- package/package.json +9 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"harness-workspaces.d.ts","sourceRoot":"","sources":["../../../../src/harness-workspaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAcxB,eAAO,MAAM,uBAAuB;;;;;;kBAKzB,CAAC;AAEZ,eAAO,MAAM,2BAA2B;;;;;;;kBAM7B,CAAC;AAEZ;;;GAGG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAexB,CAAC;AAEZ,eAAO,MAAM,6BAA6B;;;;;;;;;EASxC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;EAcpC,CAAC;AAEH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAuCjC,CAAC;AAEL,eAAO,MAAM,iCAAiC;;;;;kBAOnC,CAAC;AAEZ,eAAO,MAAM,+BAA+B;;;;kBAMjC,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;kBAW3B,CAAC;AAEZ,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAchC,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAEzB,CAAC;AAEZ,eAAO,MAAM,gCAAgC;;;;;;;;;;;kBAalC,CAAC;AAEZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;kBAiBjC,CAAC;AAEZ,eAAO,MAAM,uCAAuC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA8ChD,CAAC;AAEL,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAGhC,CAAC;AAEd,eAAO,MAAM,6CAA6C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAa/C,CAAC;AAEZ,eAAO,MAAM,sCAAsC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAGtC,CAAC;AAEd,eAAO,MAAM,4BAA4B;;;;;EAKvC,CAAC;AAEH,eAAO,MAAM,kCAAkC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA4C3C,CAAC;AAEL,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE7B,CAAC;AAEZ,eAAO,MAAM,uCAAuC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA+BhD,CAAC;AAEL,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAGhC,CAAC;AAEd,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,GACtD,iBAAiB,CAGnB;AAED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,uCAAuC,CAAC,GAC/D,0BAA0B,CAM5B;AAED,wBAAgB,sCAAsC,CACpD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,6CAA6C,CAAC,GACrE,gCAAgC,CAMlC;AAED,wBAAgB,2BAA2B,CACzC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,kCAAkC,CAAC,GAC1D,qBAAqB,CAMvB;AAED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,uCAAuC,CAAC,GAC/D,0BAA0B,CAM5B;AAED,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAChF,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AACpF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,MAAM,MAAM,2BAA2B,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,CAAC;AAC5F,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,yBAAyB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,CAAC;AACxF,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,0BAA0B,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gCAAgC,CAAC,CAAC;AAC1F,MAAM,MAAM,gCAAgC,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sCAAsC,CAAC,CAAC;AACtG,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAChF,MAAM,MAAM,0BAA0B,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gCAAgC,CAAC,CAAC"}
|
package/dist/types/harness.d.ts
CHANGED
|
@@ -8,9 +8,9 @@ export declare const PortabilityReportSchema: z.ZodObject<{
|
|
|
8
8
|
hostPrivateAssetRefs: z.ZodArray<z.ZodString>;
|
|
9
9
|
}, z.core.$strict>;
|
|
10
10
|
export declare const AgentSnapshotContentSchema: z.ZodObject<{
|
|
11
|
-
schemaVersion: z.ZodLiteral<"openpond.agentSnapshot.
|
|
11
|
+
schemaVersion: z.ZodLiteral<"openpond.agentSnapshot.v2">;
|
|
12
12
|
id: z.ZodString;
|
|
13
|
-
|
|
13
|
+
sourceRelease: z.ZodNullable<z.ZodObject<{
|
|
14
14
|
id: z.ZodString;
|
|
15
15
|
contentHash: z.ZodString;
|
|
16
16
|
}, z.core.$strict>>;
|
|
@@ -87,9 +87,9 @@ export declare const AgentSnapshotContentSchema: z.ZodObject<{
|
|
|
87
87
|
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
88
88
|
}, z.core.$strict>;
|
|
89
89
|
export declare const AgentSnapshotSchema: z.ZodObject<{
|
|
90
|
-
schemaVersion: z.ZodLiteral<"openpond.agentSnapshot.
|
|
90
|
+
schemaVersion: z.ZodLiteral<"openpond.agentSnapshot.v2">;
|
|
91
91
|
id: z.ZodString;
|
|
92
|
-
|
|
92
|
+
sourceRelease: z.ZodNullable<z.ZodObject<{
|
|
93
93
|
id: z.ZodString;
|
|
94
94
|
contentHash: z.ZodString;
|
|
95
95
|
}, z.core.$strict>>;
|
|
@@ -183,7 +183,7 @@ export declare const GraderInterfaceContractSchema: z.ZodObject<{
|
|
|
183
183
|
privateVerifierIsolation: z.ZodBoolean;
|
|
184
184
|
}, z.core.$strict>;
|
|
185
185
|
export declare const HarnessReleaseContentSchema: z.ZodObject<{
|
|
186
|
-
schemaVersion: z.ZodLiteral<"openpond.harnessRelease.
|
|
186
|
+
schemaVersion: z.ZodLiteral<"openpond.harnessRelease.v2">;
|
|
187
187
|
id: z.ZodString;
|
|
188
188
|
agentSnapshot: z.ZodObject<{
|
|
189
189
|
id: z.ZodString;
|
|
@@ -201,31 +201,6 @@ export declare const HarnessReleaseContentSchema: z.ZodObject<{
|
|
|
201
201
|
host_private: "host_private";
|
|
202
202
|
}>;
|
|
203
203
|
}, z.core.$strict>;
|
|
204
|
-
environment: z.ZodObject<{
|
|
205
|
-
protocolVersion: z.ZodLiteral<"openpond.environment.v1">;
|
|
206
|
-
kind: z.ZodEnum<{
|
|
207
|
-
text: "text";
|
|
208
|
-
agent: "agent";
|
|
209
|
-
work: "work";
|
|
210
|
-
custom_program: "custom_program";
|
|
211
|
-
}>;
|
|
212
|
-
entrypoint: z.ZodString;
|
|
213
|
-
stateful: z.ZodBoolean;
|
|
214
|
-
deterministicSeeds: z.ZodBoolean;
|
|
215
|
-
lifecycle: z.ZodArray<z.ZodEnum<{
|
|
216
|
-
create: "create";
|
|
217
|
-
reset: "reset";
|
|
218
|
-
step: "step";
|
|
219
|
-
collect: "collect";
|
|
220
|
-
destroy: "destroy";
|
|
221
|
-
}>>;
|
|
222
|
-
networkPolicy: z.ZodEnum<{
|
|
223
|
-
none: "none";
|
|
224
|
-
declared_read_only: "declared_read_only";
|
|
225
|
-
declared_scoped: "declared_scoped";
|
|
226
|
-
}>;
|
|
227
|
-
defaultTimeoutMs: z.ZodNumber;
|
|
228
|
-
}, z.core.$strict>;
|
|
229
204
|
tools: z.ZodArray<z.ZodObject<{
|
|
230
205
|
name: z.ZodString;
|
|
231
206
|
description: z.ZodString;
|
|
@@ -253,12 +228,6 @@ export declare const HarnessReleaseContentSchema: z.ZodObject<{
|
|
|
253
228
|
privilegedEvidence: z.ZodArray<z.ZodString>;
|
|
254
229
|
privateVerifierIsolation: z.ZodBoolean;
|
|
255
230
|
}, z.core.$strict>;
|
|
256
|
-
policy: z.ZodObject<{
|
|
257
|
-
policyVisibleFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
258
|
-
privilegedFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
259
|
-
hiddenGraderRefs: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
260
|
-
connectedAppScopes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
261
|
-
}, z.core.$strict>;
|
|
262
231
|
files: z.ZodArray<z.ZodObject<{
|
|
263
232
|
id: z.ZodString;
|
|
264
233
|
path: z.ZodString;
|
|
@@ -274,7 +243,7 @@ export declare const HarnessReleaseContentSchema: z.ZodObject<{
|
|
|
274
243
|
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
275
244
|
}, z.core.$strict>;
|
|
276
245
|
export declare const HarnessReleaseSchema: z.ZodObject<{
|
|
277
|
-
schemaVersion: z.ZodLiteral<"openpond.harnessRelease.
|
|
246
|
+
schemaVersion: z.ZodLiteral<"openpond.harnessRelease.v2">;
|
|
278
247
|
id: z.ZodString;
|
|
279
248
|
agentSnapshot: z.ZodObject<{
|
|
280
249
|
id: z.ZodString;
|
|
@@ -292,31 +261,6 @@ export declare const HarnessReleaseSchema: z.ZodObject<{
|
|
|
292
261
|
host_private: "host_private";
|
|
293
262
|
}>;
|
|
294
263
|
}, z.core.$strict>;
|
|
295
|
-
environment: z.ZodObject<{
|
|
296
|
-
protocolVersion: z.ZodLiteral<"openpond.environment.v1">;
|
|
297
|
-
kind: z.ZodEnum<{
|
|
298
|
-
text: "text";
|
|
299
|
-
agent: "agent";
|
|
300
|
-
work: "work";
|
|
301
|
-
custom_program: "custom_program";
|
|
302
|
-
}>;
|
|
303
|
-
entrypoint: z.ZodString;
|
|
304
|
-
stateful: z.ZodBoolean;
|
|
305
|
-
deterministicSeeds: z.ZodBoolean;
|
|
306
|
-
lifecycle: z.ZodArray<z.ZodEnum<{
|
|
307
|
-
create: "create";
|
|
308
|
-
reset: "reset";
|
|
309
|
-
step: "step";
|
|
310
|
-
collect: "collect";
|
|
311
|
-
destroy: "destroy";
|
|
312
|
-
}>>;
|
|
313
|
-
networkPolicy: z.ZodEnum<{
|
|
314
|
-
none: "none";
|
|
315
|
-
declared_read_only: "declared_read_only";
|
|
316
|
-
declared_scoped: "declared_scoped";
|
|
317
|
-
}>;
|
|
318
|
-
defaultTimeoutMs: z.ZodNumber;
|
|
319
|
-
}, z.core.$strict>;
|
|
320
264
|
tools: z.ZodArray<z.ZodObject<{
|
|
321
265
|
name: z.ZodString;
|
|
322
266
|
description: z.ZodString;
|
|
@@ -344,12 +288,6 @@ export declare const HarnessReleaseSchema: z.ZodObject<{
|
|
|
344
288
|
privilegedEvidence: z.ZodArray<z.ZodString>;
|
|
345
289
|
privateVerifierIsolation: z.ZodBoolean;
|
|
346
290
|
}, z.core.$strict>;
|
|
347
|
-
policy: z.ZodObject<{
|
|
348
|
-
policyVisibleFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
349
|
-
privilegedFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
350
|
-
hiddenGraderRefs: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
351
|
-
connectedAppScopes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
352
|
-
}, z.core.$strict>;
|
|
353
291
|
files: z.ZodArray<z.ZodObject<{
|
|
354
292
|
id: z.ZodString;
|
|
355
293
|
path: z.ZodString;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"harness.d.ts","sourceRoot":"","sources":["../../../../src/harness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAEL,0BAA0B,EAO3B,MAAM,aAAa,CAAC;AAErB,OAAO,EAAwB,KAAK,cAAc,EAAE,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAExF,eAAO,MAAM,uBAAuB;;;;;kBAKzB,CAAC;AAEZ,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAgB5B,CAAC;AACZ,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAiF,CAAC;AAElH,eAAO,MAAM,uBAAuB;;;;;;;;;;kBAOzB,CAAC;AACZ,eAAO,MAAM,6BAA6B;;;;kBAI/B,CAAC;AAEZ,eAAO,MAAM,2BAA2B
|
|
1
|
+
{"version":3,"file":"harness.d.ts","sourceRoot":"","sources":["../../../../src/harness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAEL,0BAA0B,EAO3B,MAAM,aAAa,CAAC;AAErB,OAAO,EAAwB,KAAK,cAAc,EAAE,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAExF,eAAO,MAAM,uBAAuB;;;;;kBAKzB,CAAC;AAEZ,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAgB5B,CAAC;AACZ,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAiF,CAAC;AAElH,eAAO,MAAM,uBAAuB;;;;;;;;;;kBAOzB,CAAC;AACZ,eAAO,MAAM,6BAA6B;;;;kBAI/B,CAAC;AAEZ,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAU7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,eAAO,MAAM,iBAAiB;;;;;;;;;;;kBAOnB,CAAC;AACZ,eAAO,MAAM,qBAAqB;;;;;;;;;;;kBAMvB,CAAC;AAEZ,MAAM,MAAM,YAAY,GAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,CAAC;AAC7E,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,KAAK,EAAE;QAAE,QAAQ,EAAE,WAAW,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACnH,KAAK,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxF,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAC9F,OAAO,CAAC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,EAAE,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,CAAC;IAC9J,OAAO,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7C;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,QAAQ,EAAE,WAAW,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC;AACF,MAAM,MAAM,sBAAsB,GAAG;IACnC,OAAO,EAAE,cAAc,CAAC;IACxB,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC,CAAC;AACF,MAAM,WAAW,eAAe;IAC9B,OAAO,CAAC,KAAK,EAAE,qBAAqB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;CACxE;AACD,MAAM,WAAW,gBAAgB;IAC/B,GAAG,CAAC,KAAK,EAAE;QACT,QAAQ,EAAE,WAAW,CAAC;QACtB,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAAC;CACvC;AAED,wBAAsB,sBAAsB,CAAC,KAAK,EAAE;IAClD,QAAQ,EAAE,WAAW,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,OAAO,EAAE,cAAc,CAAC;IACxB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAuGlC;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,GAAG,aAAa,CAGpG;AACD,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,GAAG,cAAc,CAGvG;AAED,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;kBAK7B,CAAC;AACZ,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAYpB,CAAC;AAEZ,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC"}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
export * from "./common.js";
|
|
2
|
+
export * from "./compatibility.js";
|
|
2
3
|
export * from "./evidence/index.js";
|
|
3
4
|
export * from "./graders.js";
|
|
4
5
|
export * from "./harness.js";
|
|
6
|
+
export * from "./harness-improvements.js";
|
|
7
|
+
export * from "./harness-workspaces.js";
|
|
5
8
|
export * from "./runs.js";
|
|
6
9
|
export * from "./tasksets.js";
|
|
7
10
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC"}
|
package/dist/types/runs.d.ts
CHANGED
|
@@ -273,6 +273,49 @@ export declare const EvaluationResultSchema: z.ZodObject<{
|
|
|
273
273
|
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
274
274
|
contentHash: z.ZodString;
|
|
275
275
|
}, z.core.$strict>;
|
|
276
|
+
export declare const HarnessCompatibilityReceiptContentSchema: z.ZodObject<{
|
|
277
|
+
schemaVersion: z.ZodLiteral<"openpond.harnessCompatibility.v1">;
|
|
278
|
+
id: z.ZodString;
|
|
279
|
+
baseHarnessRelease: z.ZodObject<{
|
|
280
|
+
id: z.ZodString;
|
|
281
|
+
contentHash: z.ZodString;
|
|
282
|
+
}, z.core.$strict>;
|
|
283
|
+
candidateHarnessRelease: z.ZodObject<{
|
|
284
|
+
id: z.ZodString;
|
|
285
|
+
contentHash: z.ZodString;
|
|
286
|
+
}, z.core.$strict>;
|
|
287
|
+
tasksetRelease: z.ZodObject<{
|
|
288
|
+
id: z.ZodString;
|
|
289
|
+
contentHash: z.ZodString;
|
|
290
|
+
}, z.core.$strict>;
|
|
291
|
+
environmentHash: z.ZodString;
|
|
292
|
+
toolContractHash: z.ZodString;
|
|
293
|
+
policyHash: z.ZodString;
|
|
294
|
+
graderInterfaceHash: z.ZodString;
|
|
295
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
296
|
+
}, z.core.$strict>;
|
|
297
|
+
export declare const HarnessCompatibilityReceiptSchema: z.ZodObject<{
|
|
298
|
+
schemaVersion: z.ZodLiteral<"openpond.harnessCompatibility.v1">;
|
|
299
|
+
id: z.ZodString;
|
|
300
|
+
baseHarnessRelease: z.ZodObject<{
|
|
301
|
+
id: z.ZodString;
|
|
302
|
+
contentHash: z.ZodString;
|
|
303
|
+
}, z.core.$strict>;
|
|
304
|
+
candidateHarnessRelease: z.ZodObject<{
|
|
305
|
+
id: z.ZodString;
|
|
306
|
+
contentHash: z.ZodString;
|
|
307
|
+
}, z.core.$strict>;
|
|
308
|
+
tasksetRelease: z.ZodObject<{
|
|
309
|
+
id: z.ZodString;
|
|
310
|
+
contentHash: z.ZodString;
|
|
311
|
+
}, z.core.$strict>;
|
|
312
|
+
environmentHash: z.ZodString;
|
|
313
|
+
toolContractHash: z.ZodString;
|
|
314
|
+
policyHash: z.ZodString;
|
|
315
|
+
graderInterfaceHash: z.ZodString;
|
|
316
|
+
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
317
|
+
contentHash: z.ZodString;
|
|
318
|
+
}, z.core.$strict>;
|
|
276
319
|
export declare function createRunManifest(input: z.input<typeof RunManifestContentSchema>): RunManifest;
|
|
277
320
|
export declare function createAttemptReceipt(input: z.input<typeof AttemptReceiptContentSchema>): AttemptReceipt;
|
|
278
321
|
export declare function verifyAttemptReceipt(receipt: AttemptReceipt): boolean;
|
|
@@ -282,7 +325,8 @@ export declare function aggregateEvaluationReceipts(input: {
|
|
|
282
325
|
receipts: AttemptReceipt[];
|
|
283
326
|
metadata?: Record<string, unknown>;
|
|
284
327
|
}): EvaluationResult;
|
|
285
|
-
export declare function
|
|
328
|
+
export declare function createHarnessCompatibilityReceipt(input: z.input<typeof HarnessCompatibilityReceiptContentSchema>): z.infer<typeof HarnessCompatibilityReceiptSchema>;
|
|
329
|
+
export declare function assertComparableRunManifests(base: RunManifest, candidate: RunManifest, compatibility?: z.input<typeof HarnessCompatibilityReceiptSchema>): void;
|
|
286
330
|
export declare function rewardEligibleReceipts(receipts: AttemptReceipt[]): AttemptReceipt[];
|
|
287
331
|
export type ModelRef = z.infer<typeof ModelRefSchema>;
|
|
288
332
|
export type RuntimeTargetBinding = z.infer<typeof RuntimeTargetBindingSchema>;
|
|
@@ -290,4 +334,5 @@ export type RunLimits = z.infer<typeof RunLimitsSchema>;
|
|
|
290
334
|
export type RunManifest = z.infer<typeof RunManifestSchema>;
|
|
291
335
|
export type AttemptReceipt = z.infer<typeof AttemptReceiptSchema>;
|
|
292
336
|
export type EvaluationResult = z.infer<typeof EvaluationResultSchema>;
|
|
337
|
+
export type HarnessCompatibilityReceipt = z.infer<typeof HarnessCompatibilityReceiptSchema>;
|
|
293
338
|
//# sourceMappingURL=runs.d.ts.map
|
package/dist/types/runs.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runs.d.ts","sourceRoot":"","sources":["../../../../src/runs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"runs.d.ts","sourceRoot":"","sources":["../../../../src/runs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAcxB,eAAO,MAAM,cAAc;;;;;;;kBAOhB,CAAC;AAEZ,eAAO,MAAM,0BAA0B;;;;;;;;;;kBAK5B,CAAC;AAEZ,eAAO,MAAM,eAAe;;;;;kBAKjB,CAAC;AAEZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAW1B,CAAC;AACZ,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA+E,CAAC;AAE9G,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkB7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAc/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AAExH,eAAO,MAAM,wCAAwC;;;;;;;;;;;;;;;;;;;;kBAW1C,CAAC;AACZ,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;kBAEnC,CAAC;AAEZ,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,GAAG,WAAW,CAG9F;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,GAAG,cAAc,CAGvG;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAKrE;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,WAAW,CAAC;IACtB,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,gBAAgB,CAgCnB;AAED,wBAAgB,iCAAiC,CAC/C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,wCAAwC,CAAC,GAC9D,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,CAGnD;AAED,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,WAAW,EACjB,SAAS,EAAE,WAAW,EACtB,aAAa,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,GAChE,IAAI,CAiBN;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CASnF;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,2BAA2B,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,CAAC"}
|
package/dist/types/tasksets.d.ts
CHANGED
|
@@ -258,13 +258,9 @@ export declare const TaskRecordSchema: z.ZodObject<{
|
|
|
258
258
|
tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
259
259
|
}, z.core.$strict>;
|
|
260
260
|
export declare const TasksetReleaseContentSchema: z.ZodObject<{
|
|
261
|
-
schemaVersion: z.ZodLiteral<"openpond.tasksetRelease.
|
|
261
|
+
schemaVersion: z.ZodLiteral<"openpond.tasksetRelease.v2">;
|
|
262
262
|
id: z.ZodString;
|
|
263
263
|
revision: z.ZodNumber;
|
|
264
|
-
harnessRelease: z.ZodObject<{
|
|
265
|
-
id: z.ZodString;
|
|
266
|
-
contentHash: z.ZodString;
|
|
267
|
-
}, z.core.$strict>;
|
|
268
264
|
policy: z.ZodObject<{
|
|
269
265
|
policyVisibleFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
270
266
|
privilegedFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
@@ -432,13 +428,9 @@ export declare const TasksetReleaseContentSchema: z.ZodObject<{
|
|
|
432
428
|
metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
433
429
|
}, z.core.$strict>;
|
|
434
430
|
export declare const TasksetReleaseSchema: z.ZodObject<{
|
|
435
|
-
schemaVersion: z.ZodLiteral<"openpond.tasksetRelease.
|
|
431
|
+
schemaVersion: z.ZodLiteral<"openpond.tasksetRelease.v2">;
|
|
436
432
|
id: z.ZodString;
|
|
437
433
|
revision: z.ZodNumber;
|
|
438
|
-
harnessRelease: z.ZodObject<{
|
|
439
|
-
id: z.ZodString;
|
|
440
|
-
contentHash: z.ZodString;
|
|
441
|
-
}, z.core.$strict>;
|
|
442
434
|
policy: z.ZodObject<{
|
|
443
435
|
policyVisibleFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
444
436
|
privilegedFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tasksets.d.ts","sourceRoot":"","sources":["../../../../src/tasksets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EACL,uBAAuB,
|
|
1
|
+
{"version":3,"file":"tasksets.d.ts","sourceRoot":"","sources":["../../../../src/tasksets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EACL,uBAAuB,EAMxB,MAAM,aAAa,CAAC;AAErB,eAAO,MAAM,eAAe;;;;;EAAyD,CAAC;AACtF,eAAO,MAAM,oBAAoB;;;;;kBAKtB,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;;;;;kBAOvB,CAAC;AAEZ,eAAO,MAAM,2BAA2B;;;;;;;;;;kBAK7B,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;kBAS3B,CAAC;AAWZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;kBAG/B,CAAC;AACZ,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;kBAI5B,CAAC;AACZ,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;kBAKhC,CAAC;AACZ,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;kBAIvB,CAAC;AACZ,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAK3B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;kBAUlB,CAAC;AAEZ,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,SAAS,GAAG,OAAO,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG;IACtD,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,cAAc,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,sBAAsB,EAAE,CAAC;CAClC,CA8CA;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,CAKlE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,UAAU,GAAG;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9C,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC,CAAC;IAC7D,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB,CAQA;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,cAAc,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,EAAE,CAIpG;AAED,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAChF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AACpF,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openpond/evals",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Portable evaluation, Work evidence, Harness, Taskset, grader, and receipt contracts for OpenPond",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -22,6 +22,14 @@
|
|
|
22
22
|
"types": "./dist/types/harness.d.ts",
|
|
23
23
|
"import": "./dist/harness.js"
|
|
24
24
|
},
|
|
25
|
+
"./harness-improvements": {
|
|
26
|
+
"types": "./dist/types/harness-improvements.d.ts",
|
|
27
|
+
"import": "./dist/harness-improvements.js"
|
|
28
|
+
},
|
|
29
|
+
"./harness-workspaces": {
|
|
30
|
+
"types": "./dist/types/harness-workspaces.d.ts",
|
|
31
|
+
"import": "./dist/harness-workspaces.js"
|
|
32
|
+
},
|
|
25
33
|
"./tasksets": {
|
|
26
34
|
"types": "./dist/types/tasksets.d.ts",
|
|
27
35
|
"import": "./dist/tasksets.js"
|