@openpond/evals 0.8.1 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +47 -0
  2. package/dist/deterministic-graders.js +110 -0
  3. package/dist/graders.js +9 -51
  4. package/dist/index.js +1 -0
  5. package/dist/metric-policy.js +50 -0
  6. package/dist/metrics-node.js +16 -0
  7. package/dist/metrics.js +105 -0
  8. package/dist/runs.js +12 -0
  9. package/dist/tasksets.js +5 -0
  10. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +14 -0
  11. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  12. package/dist/types/conformance.d.ts +28 -0
  13. package/dist/types/conformance.d.ts.map +1 -1
  14. package/dist/types/deterministic-graders.d.ts +23 -0
  15. package/dist/types/deterministic-graders.d.ts.map +1 -0
  16. package/dist/types/evidence/conformance.d.ts +4 -4
  17. package/dist/types/evidence/contracts.d.ts +8 -8
  18. package/dist/types/graders.d.ts +1 -0
  19. package/dist/types/graders.d.ts.map +1 -1
  20. package/dist/types/index.d.ts +1 -0
  21. package/dist/types/index.d.ts.map +1 -1
  22. package/dist/types/javascript-isolate.d.ts +1 -1
  23. package/dist/types/javascript-isolate.d.ts.map +1 -1
  24. package/dist/types/learning/admission.d.ts +3 -3
  25. package/dist/types/learning/contracts.d.ts +3 -3
  26. package/dist/types/learning/operations.d.ts +12 -12
  27. package/dist/types/learning/repository.d.ts +26 -4
  28. package/dist/types/learning/repository.d.ts.map +1 -1
  29. package/dist/types/learning/transport.d.ts +33 -11
  30. package/dist/types/learning/transport.d.ts.map +1 -1
  31. package/dist/types/metric-policy.d.ts +112 -0
  32. package/dist/types/metric-policy.d.ts.map +1 -0
  33. package/dist/types/metrics-node.d.ts +8 -0
  34. package/dist/types/metrics-node.d.ts.map +1 -0
  35. package/dist/types/metrics.d.ts +37 -0
  36. package/dist/types/metrics.d.ts.map +1 -0
  37. package/dist/types/preferences.d.ts +3 -3
  38. package/dist/types/rewards.d.ts +9 -9
  39. package/dist/types/runs.d.ts +88 -0
  40. package/dist/types/runs.d.ts.map +1 -1
  41. package/dist/types/tasksets.d.ts +44 -0
  42. package/dist/types/tasksets.d.ts.map +1 -1
  43. package/dist/types/telemetry-analysis.d.ts +1 -1
  44. package/dist/types/telemetry-bundle.d.ts +4 -4
  45. package/dist/types/telemetry.d.ts +3 -3
  46. package/package.json +9 -1
  47. package/schemas/learning/v1/package.schema.json +108 -0
package/dist/types/rewards.d.ts CHANGED
@@ -36,8 +36,10 @@ export declare const RewardImplementationSchema: z.ZodUnion<readonly [z.ZodObjec
36
36
  }, z.core.$strict>>;
37
37
  temperature: z.ZodOptional<z.ZodNumber>;
38
38
  }, z.core.$strict>, z.ZodObject<{
39
- kind: z.ZodLiteral<"custom_verifier">;
39
+ exportName: z.ZodOptional<z.ZodString>;
40
+ timeoutMs: z.ZodNumber;
40
41
  networkPolicy: z.ZodLiteral<"none">;
42
+ kind: z.ZodLiteral<"custom_verifier">;
41
43
  verifierRef: z.ZodObject<{
42
44
  id: z.ZodString;
43
45
  path: z.ZodString;
@@ -50,8 +52,6 @@ export declare const RewardImplementationSchema: z.ZodUnion<readonly [z.ZodObjec
50
52
  host_private: "host_private";
51
53
  }>;
52
54
  }, z.core.$strict>;
53
- exportName: z.ZodOptional<z.ZodString>;
54
- timeoutMs: z.ZodNumber;
55
55
  }, z.core.$strict>, z.ZodObject<{
56
56
  kind: z.ZodLiteral<"human">;
57
57
  rubricRef: z.ZodObject<{
@@ -127,8 +127,10 @@ export declare const RewardReleaseContentSchema: z.ZodObject<{
127
127
  }, z.core.$strict>>;
128
128
  temperature: z.ZodOptional<z.ZodNumber>;
129
129
  }, z.core.$strict>, z.ZodObject<{
130
- kind: z.ZodLiteral<"custom_verifier">;
130
+ exportName: z.ZodOptional<z.ZodString>;
131
+ timeoutMs: z.ZodNumber;
131
132
  networkPolicy: z.ZodLiteral<"none">;
133
+ kind: z.ZodLiteral<"custom_verifier">;
132
134
  verifierRef: z.ZodObject<{
133
135
  id: z.ZodString;
134
136
  path: z.ZodString;
@@ -141,8 +143,6 @@ export declare const RewardReleaseContentSchema: z.ZodObject<{
141
143
  host_private: "host_private";
142
144
  }>;
143
145
  }, z.core.$strict>;
144
- exportName: z.ZodOptional<z.ZodString>;
145
- timeoutMs: z.ZodNumber;
146
146
  }, z.core.$strict>, z.ZodObject<{
147
147
  kind: z.ZodLiteral<"human">;
148
148
  rubricRef: z.ZodObject<{
@@ -247,8 +247,10 @@ export declare const RewardReleaseSchema: z.ZodObject<{
247
247
  }, z.core.$strict>>;
248
248
  temperature: z.ZodOptional<z.ZodNumber>;
249
249
  }, z.core.$strict>, z.ZodObject<{
250
- kind: z.ZodLiteral<"custom_verifier">;
250
+ exportName: z.ZodOptional<z.ZodString>;
251
+ timeoutMs: z.ZodNumber;
251
252
  networkPolicy: z.ZodLiteral<"none">;
253
+ kind: z.ZodLiteral<"custom_verifier">;
252
254
  verifierRef: z.ZodObject<{
253
255
  id: z.ZodString;
254
256
  path: z.ZodString;
@@ -261,8 +263,6 @@ export declare const RewardReleaseSchema: z.ZodObject<{
261
263
  host_private: "host_private";
262
264
  }>;
263
265
  }, z.core.$strict>;
264
- exportName: z.ZodOptional<z.ZodString>;
265
- timeoutMs: z.ZodNumber;
266
266
  }, z.core.$strict>, z.ZodObject<{
267
267
  kind: z.ZodLiteral<"human">;
268
268
  rubricRef: z.ZodObject<{
package/dist/types/runs.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { z } from "zod";
2
+ import { type TasksetMetricResult } from "./metric-policy.js";
2
3
  export declare const RuntimeTargetBindingSchema: z.ZodObject<{
3
4
  adapterId: z.ZodString;
4
5
  placement: z.ZodEnum<{
@@ -221,6 +222,49 @@ export declare const EvaluationResultContentSchema: z.ZodObject<{
221
222
  timeout: "timeout";
222
223
  cancelled: "cancelled";
223
224
  }>, z.ZodNumber>;
225
+ authoredMetric: z.ZodOptional<z.ZodObject<{
226
+ schemaVersion: z.ZodLiteral<"openpond.tasksetMetricResult.v1">;
227
+ runManifest: z.ZodObject<{
228
+ id: z.ZodString;
229
+ contentHash: z.ZodString;
230
+ }, z.core.$strict>;
231
+ tasksetRelease: z.ZodObject<{
232
+ id: z.ZodString;
233
+ contentHash: z.ZodString;
234
+ }, z.core.$strict>;
235
+ policy: z.ZodObject<{
236
+ schemaVersion: z.ZodLiteral<"openpond.tasksetMetricPolicy.v1">;
237
+ primaryMetric: z.ZodString;
238
+ aggregation: z.ZodEnum<{
239
+ mean_score: "mean_score";
240
+ pass_rate: "pass_rate";
241
+ weighted_mean: "weighted_mean";
242
+ custom: "custom";
243
+ }>;
244
+ missingReward: z.ZodEnum<{
245
+ zero: "zero";
246
+ exclude: "exclude";
247
+ }>;
248
+ taskWeights: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
249
+ customAggregator: z.ZodNullable<z.ZodObject<{
250
+ module: z.ZodString;
251
+ exportName: z.ZodString;
252
+ contentHash: z.ZodString;
253
+ timeoutMs: z.ZodNumber;
254
+ networkPolicy: z.ZodLiteral<"none">;
255
+ }, z.core.$strict>>;
256
+ }, z.core.$strict>;
257
+ policyHash: z.ZodString;
258
+ receiptRefs: z.ZodArray<z.ZodObject<{
259
+ id: z.ZodString;
260
+ contentHash: z.ZodString;
261
+ }, z.core.$strict>>;
262
+ includedCount: z.ZodNumber;
263
+ missingRewardCount: z.ZodNumber;
264
+ excludedCount: z.ZodNumber;
265
+ value: z.ZodNullable<z.ZodNumber>;
266
+ contentHash: z.ZodString;
267
+ }, z.core.$strict>>;
224
268
  metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
225
269
  }, z.core.$strict>;
226
270
  export declare const EvaluationResultSchema: z.ZodObject<{
@@ -262,6 +306,49 @@ export declare const EvaluationResultSchema: z.ZodObject<{
262
306
  timeout: "timeout";
263
307
  cancelled: "cancelled";
264
308
  }>, z.ZodNumber>;
309
+ authoredMetric: z.ZodOptional<z.ZodObject<{
310
+ schemaVersion: z.ZodLiteral<"openpond.tasksetMetricResult.v1">;
311
+ runManifest: z.ZodObject<{
312
+ id: z.ZodString;
313
+ contentHash: z.ZodString;
314
+ }, z.core.$strict>;
315
+ tasksetRelease: z.ZodObject<{
316
+ id: z.ZodString;
317
+ contentHash: z.ZodString;
318
+ }, z.core.$strict>;
319
+ policy: z.ZodObject<{
320
+ schemaVersion: z.ZodLiteral<"openpond.tasksetMetricPolicy.v1">;
321
+ primaryMetric: z.ZodString;
322
+ aggregation: z.ZodEnum<{
323
+ mean_score: "mean_score";
324
+ pass_rate: "pass_rate";
325
+ weighted_mean: "weighted_mean";
326
+ custom: "custom";
327
+ }>;
328
+ missingReward: z.ZodEnum<{
329
+ zero: "zero";
330
+ exclude: "exclude";
331
+ }>;
332
+ taskWeights: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
333
+ customAggregator: z.ZodNullable<z.ZodObject<{
334
+ module: z.ZodString;
335
+ exportName: z.ZodString;
336
+ contentHash: z.ZodString;
337
+ timeoutMs: z.ZodNumber;
338
+ networkPolicy: z.ZodLiteral<"none">;
339
+ }, z.core.$strict>>;
340
+ }, z.core.$strict>;
341
+ policyHash: z.ZodString;
342
+ receiptRefs: z.ZodArray<z.ZodObject<{
343
+ id: z.ZodString;
344
+ contentHash: z.ZodString;
345
+ }, z.core.$strict>>;
346
+ includedCount: z.ZodNumber;
347
+ missingRewardCount: z.ZodNumber;
348
+ excludedCount: z.ZodNumber;
349
+ value: z.ZodNullable<z.ZodNumber>;
350
+ contentHash: z.ZodString;
351
+ }, z.core.$strict>>;
265
352
  metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
266
353
  contentHash: z.ZodString;
267
354
  }, z.core.$strict>;
@@ -316,6 +403,7 @@ export declare function aggregateEvaluationReceipts(input: {
316
403
  manifest: RunManifest;
317
404
  receipts: AttemptReceipt[];
318
405
  metadata?: Record<string, unknown>;
406
+ authoredMetric?: TasksetMetricResult;
319
407
  }): EvaluationResult;
320
408
  export declare function createHarnessCompatibilityReceipt(input: z.input<typeof HarnessCompatibilityReceiptContentSchema>): z.infer<typeof HarnessCompatibilityReceiptSchema>;
321
409
  export declare function assertComparableRunManifests(base: RunManifest, candidate: RunManifest, compatibility?: z.input<typeof HarnessCompatibilityReceiptSchema>): void;
package/dist/types/runs.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"runs.d.ts","sourceRoot":"","sources":["../../../../src/runs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAexB,eAAO,MAAM,0BAA0B;;;;;;;;;;kBAK5B,CAAC;AAEZ,eAAO,MAAM,eAAe;;;;;kBAKjB,CAAC;AAEZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAW1B,CAAC;AACZ,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA+E,CAAC;AAE9G,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkB7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAc/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AAExH,eAAO,MAAM,wCAAwC;;;;;;;;;;;;;;;;;;;;kBAW1C,CAAC;AACZ,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;kBAEnC,CAAC;AAEZ,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,GAAG,WAAW,CAG9F;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,GAAG,cAAc,CAGvG;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAKrE;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,WAAW,CAAC;IACtB,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,gBAAgB,CAgCnB;AAED,wBAAgB,iCAAiC,CAC/C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,wCAAwC,CAAC,GAC9D,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,CAGnD;AAED,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,WAAW,EACjB,SAAS,EAAE,WAAW,EACtB,aAAa,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,GAChE,IAAI,CAiBN;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CASnF;AAED,YAAY,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClD,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,2BAA2B,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,CAAC"}
1
+ {"version":3,"file":"runs.d.ts","sourceRoot":"","sources":["../../../../src/runs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAcxB,OAAO,EAAwD,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEpH,eAAO,MAAM,0BAA0B;;;;;;;;;;kBAK5B,CAAC;AAEZ,eAAO,MAAM,eAAe;;;;;kBAKjB,CAAC;AAEZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAW1B,CAAC;AACZ,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA+E,CAAC;AAE9G,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkB7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAe/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AAExH,eAAO,MAAM,wCAAwC;;;;;;;;;;;;;;;;;;;;kBAW1C,CAAC;AACZ,eAAO,MAAM,iCAAiC;;;;;;;;;;;;;;;;;;;;;kBAEnC,CAAC;AAEZ,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,GAAG,WAAW,CAG9F;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,GAAG,cAAc,CAGvG;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAKrE;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,WAAW,CAAC;IACtB,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,cAAc,CAAC,EAAE,mBAAmB,CAAC;CACtC,GAAG,gBAAgB,CA0CnB;AAED,wBAAgB,iCAAiC,CAC/C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,wCAAwC,CAAC,GAC9D,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,CAGnD;AAED,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,WAAW,EACjB,SAAS,EAAE,WAAW,EACtB,aAAa,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,GAChE,IAAI,CAiBN;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CASnF;AAED,YAAY,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClD,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,2BAA2B,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iCAAiC,CAAC,CAAC"}
package/dist/types/tasksets.d.ts CHANGED
@@ -486,6 +486,28 @@ export declare const TasksetReleaseContentSchema: z.ZodObject<{
486
486
  id: z.ZodString;
487
487
  contentHash: z.ZodString;
488
488
  }, z.core.$strict>>;
489
+ metrics: z.ZodOptional<z.ZodObject<{
490
+ schemaVersion: z.ZodLiteral<"openpond.tasksetMetricPolicy.v1">;
491
+ primaryMetric: z.ZodString;
492
+ aggregation: z.ZodEnum<{
493
+ mean_score: "mean_score";
494
+ pass_rate: "pass_rate";
495
+ weighted_mean: "weighted_mean";
496
+ custom: "custom";
497
+ }>;
498
+ missingReward: z.ZodEnum<{
499
+ zero: "zero";
500
+ exclude: "exclude";
501
+ }>;
502
+ taskWeights: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
503
+ customAggregator: z.ZodNullable<z.ZodObject<{
504
+ module: z.ZodString;
505
+ exportName: z.ZodString;
506
+ contentHash: z.ZodString;
507
+ timeoutMs: z.ZodNumber;
508
+ networkPolicy: z.ZodLiteral<"none">;
509
+ }, z.core.$strict>>;
510
+ }, z.core.$strict>>;
489
511
  metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
490
512
  }, z.core.$strict>;
491
513
  export declare const TasksetReleaseSchema: z.ZodObject<{
@@ -689,6 +711,28 @@ export declare const TasksetReleaseSchema: z.ZodObject<{
689
711
  id: z.ZodString;
690
712
  contentHash: z.ZodString;
691
713
  }, z.core.$strict>>;
714
+ metrics: z.ZodOptional<z.ZodObject<{
715
+ schemaVersion: z.ZodLiteral<"openpond.tasksetMetricPolicy.v1">;
716
+ primaryMetric: z.ZodString;
717
+ aggregation: z.ZodEnum<{
718
+ mean_score: "mean_score";
719
+ pass_rate: "pass_rate";
720
+ weighted_mean: "weighted_mean";
721
+ custom: "custom";
722
+ }>;
723
+ missingReward: z.ZodEnum<{
724
+ zero: "zero";
725
+ exclude: "exclude";
726
+ }>;
727
+ taskWeights: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
728
+ customAggregator: z.ZodNullable<z.ZodObject<{
729
+ module: z.ZodString;
730
+ exportName: z.ZodString;
731
+ contentHash: z.ZodString;
732
+ timeoutMs: z.ZodNumber;
733
+ networkPolicy: z.ZodLiteral<"none">;
734
+ }, z.core.$strict>>;
735
+ }, z.core.$strict>>;
692
736
  metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
693
737
  contentHash: z.ZodString;
694
738
  }, z.core.$strict>;
package/dist/types/tasksets.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"tasksets.d.ts","sourceRoot":"","sources":["../../../../src/tasksets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAEL,uBAAuB,EAOxB,MAAM,mBAAmB,CAAC;AAE3B,eAAO,MAAM,eAAe;;;;;EAAyD,CAAC;AACtF,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;kBAM9B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;kBAKtB,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;kBAS3B,CAAC;AAWZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;kBAG/B,CAAC;AACZ,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM5B,CAAC;AACZ,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;kBAMhC,CAAC;AACZ,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;kBAIvB,CAAC;AACZ,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAK3B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAWlB,CAAC;AAEZ,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAa7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,SAAS,GAAG,OAAO,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG;IACtD,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,cAAc,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,sBAAsB,EAAE,CAAC;CAClC,CAsDA;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,CAKlE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,UAAU,GAAG;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9C,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC,CAAC;IAC7D,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB,CAQA;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,cAAc,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,EAAE,CAIpG;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AACpF,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAElF,OAAO,EACL,2BAA2B,EAC3B,qBAAqB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,eAAe,GACrB,MAAM,mBAAmB,CAAC"}
1
+ {"version":3,"file":"tasksets.d.ts","sourceRoot":"","sources":["../../../../src/tasksets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAEL,uBAAuB,EAOxB,MAAM,mBAAmB,CAAC;AAG3B,eAAO,MAAM,eAAe;;;;;EAAyD,CAAC;AACtF,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;kBAM9B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;kBAKtB,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;kBAS3B,CAAC;AAWZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;kBAG/B,CAAC;AACZ,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM5B,CAAC;AACZ,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;kBAMhC,CAAC;AACZ,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;kBAIvB,CAAC;AACZ,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAK3B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAWlB,CAAC;AAEZ,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAc7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,SAAS,GAAG,OAAO,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG;IACtD,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,cAAc,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,sBAAsB,EAAE,CAAC;CAClC,CAyDA;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI,CAKlE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,UAAU,GAAG;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9C,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC,CAAC;IAC7D,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB,CAQA;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,cAAc,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,EAAE,CAIpG;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AACpF,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAElF,OAAO,EACL,2BAA2B,EAC3B,qBAAqB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,eAAe,GACrB,MAAM,mBAAmB,CAAC"}
package/dist/types/telemetry-analysis.d.ts CHANGED
@@ -18,8 +18,8 @@ export declare const EvidenceReferenceSchema: z.ZodObject<{
18
18
  kind: z.ZodEnum<{
19
19
  artifact: "artifact";
20
20
  attempt: "attempt";
21
- trace: "trace";
22
21
  grader: "grader";
22
+ trace: "trace";
23
23
  rollout: "rollout";
24
24
  checkpoint: "checkpoint";
25
25
  }>;
package/dist/types/telemetry-bundle.d.ts CHANGED
@@ -54,8 +54,8 @@ export declare const TelemetryExportBundleContentSchema: z.ZodObject<{
54
54
  source: z.ZodEnum<{
55
55
  environment: "environment";
56
56
  evaluation: "evaluation";
57
- runtime: "runtime";
58
57
  grader: "grader";
58
+ runtime: "runtime";
59
59
  optimizer: "optimizer";
60
60
  control_plane: "control_plane";
61
61
  }>;
@@ -122,8 +122,8 @@ export declare const TelemetryExportBundleContentSchema: z.ZodObject<{
122
122
  kind: z.ZodEnum<{
123
123
  artifact: "artifact";
124
124
  attempt: "attempt";
125
- trace: "trace";
126
125
  grader: "grader";
126
+ trace: "trace";
127
127
  rollout: "rollout";
128
128
  checkpoint: "checkpoint";
129
129
  }>;
@@ -202,8 +202,8 @@ export declare const TelemetryExportBundleSchema: z.ZodObject<{
202
202
  source: z.ZodEnum<{
203
203
  environment: "environment";
204
204
  evaluation: "evaluation";
205
- runtime: "runtime";
206
205
  grader: "grader";
206
+ runtime: "runtime";
207
207
  optimizer: "optimizer";
208
208
  control_plane: "control_plane";
209
209
  }>;
@@ -270,8 +270,8 @@ export declare const TelemetryExportBundleSchema: z.ZodObject<{
270
270
  kind: z.ZodEnum<{
271
271
  artifact: "artifact";
272
272
  attempt: "attempt";
273
- trace: "trace";
274
273
  grader: "grader";
274
+ trace: "trace";
275
275
  rollout: "rollout";
276
276
  checkpoint: "checkpoint";
277
277
  }>;
package/dist/types/telemetry.d.ts CHANGED
@@ -10,8 +10,8 @@ export declare const TelemetryVisibilitySchema: z.ZodEnum<{
10
10
  export declare const TelemetrySourceSchema: z.ZodEnum<{
11
11
  environment: "environment";
12
12
  evaluation: "evaluation";
13
- runtime: "runtime";
14
13
  grader: "grader";
14
+ runtime: "runtime";
15
15
  optimizer: "optimizer";
16
16
  control_plane: "control_plane";
17
17
  }>;
@@ -50,8 +50,8 @@ export declare const RunTelemetryEventSchema: z.ZodObject<{
50
50
  source: z.ZodEnum<{
51
51
  environment: "environment";
52
52
  evaluation: "evaluation";
53
- runtime: "runtime";
54
53
  grader: "grader";
54
+ runtime: "runtime";
55
55
  optimizer: "optimizer";
56
56
  control_plane: "control_plane";
57
57
  }>;
@@ -192,8 +192,8 @@ export declare const RunTelemetryBatchSchema: z.ZodObject<{
192
192
  source: z.ZodEnum<{
193
193
  environment: "environment";
194
194
  evaluation: "evaluation";
195
- runtime: "runtime";
196
195
  grader: "grader";
196
+ runtime: "runtime";
197
197
  optimizer: "optimizer";
198
198
  control_plane: "control_plane";
199
199
  }>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@openpond/evals",
3
- "version": "0.8.1",
3
+ "version": "0.9.1",
4
4
  "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -19,6 +19,14 @@
19
19
  "LICENSE"
20
20
  ],
21
21
  "exports": {
22
+ "./metrics/node": {
23
+ "types": "./dist/types/metrics-node.d.ts",
24
+ "import": "./dist/metrics-node.js"
25
+ },
26
+ "./metrics": {
27
+ "types": "./dist/types/metrics.d.ts",
28
+ "import": "./dist/metrics.js"
29
+ },
22
30
  "./sql-execution/node": {
23
31
  "types": "./dist/types/sql-execution-node.d.ts",
24
32
  "import": "./dist/sql-execution-node.js"
package/schemas/learning/v1/package.schema.json CHANGED
@@ -181,6 +181,37 @@
181
181
  ],
182
182
  "additionalProperties": false
183
183
  },
184
+ "metrics": {
185
+ "type": "object",
186
+ "properties": {
187
+ "schemaVersion": {
188
+ "$ref": "#/$defs/__schema16"
189
+ },
190
+ "primaryMetric": {
191
+ "$ref": "#/$defs/__schema17"
192
+ },
193
+ "aggregation": {
194
+ "$ref": "#/$defs/__schema18"
195
+ },
196
+ "missingReward": {
197
+ "$ref": "#/$defs/__schema19"
198
+ },
199
+ "taskWeights": {
200
+ "$ref": "#/$defs/__schema20"
201
+ },
202
+ "customAggregator": {
203
+ "$ref": "#/$defs/__schema21"
204
+ }
205
+ },
206
+ "required": [
207
+ "schemaVersion",
208
+ "primaryMetric",
209
+ "aggregation",
210
+ "missingReward",
211
+ "customAggregator"
212
+ ],
213
+ "additionalProperties": false
214
+ },
184
215
  "metadata": {
185
216
  "$ref": "#/$defs/__schema9"
186
217
  },
@@ -769,6 +800,83 @@
769
800
  "__schema15": {
770
801
  "default": false,
771
802
  "type": "boolean"
803
+ },
804
+ "__schema16": {
805
+ "type": "string",
806
+ "const": "openpond.tasksetMetricPolicy.v1"
807
+ },
808
+ "__schema17": {
809
+ "type": "string",
810
+ "minLength": 1,
811
+ "maxLength": 240
812
+ },
813
+ "__schema18": {
814
+ "type": "string",
815
+ "enum": [
816
+ "mean_score",
817
+ "pass_rate",
818
+ "weighted_mean",
819
+ "custom"
820
+ ]
821
+ },
822
+ "__schema19": {
823
+ "type": "string",
824
+ "enum": [
825
+ "zero",
826
+ "exclude"
827
+ ]
828
+ },
829
+ "__schema20": {
830
+ "type": "object",
831
+ "propertyNames": {
832
+ "$ref": "#/$defs/__schema17"
833
+ },
834
+ "additionalProperties": {
835
+ "type": "number",
836
+ "exclusiveMinimum": 0,
837
+ "maximum": 1000000
838
+ }
839
+ },
840
+ "__schema21": {
841
+ "anyOf": [
842
+ {
843
+ "type": "object",
844
+ "properties": {
845
+ "module": {
846
+ "type": "string",
847
+ "minLength": 1,
848
+ "maxLength": 1000
849
+ },
850
+ "exportName": {
851
+ "type": "string",
852
+ "pattern": "^[A-Za-z_$][A-Za-z0-9_$]*$"
853
+ },
854
+ "contentHash": {
855
+ "$ref": "#/$defs/__schema2"
856
+ },
857
+ "timeoutMs": {
858
+ "type": "integer",
859
+ "exclusiveMinimum": 0,
860
+ "maximum": 300000
861
+ },
862
+ "networkPolicy": {
863
+ "type": "string",
864
+ "const": "none"
865
+ }
866
+ },
867
+ "required": [
868
+ "module",
869
+ "exportName",
870
+ "contentHash",
871
+ "timeoutMs",
872
+ "networkPolicy"
873
+ ],
874
+ "additionalProperties": false
875
+ },
876
+ {
877
+ "type": "null"
878
+ }
879
+ ]
772
880
  }
773
881
  },
774
882
  "$id": "urn:openpond:learning:v1:package",