@openpond/evals 0.4.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CONTRACT.md +27 -1
  2. package/README.md +31 -2
  3. package/conformance/telemetry/v1/invalid-batch.json +17 -0
  4. package/conformance/telemetry/v1/valid-batch.json +58 -0
  5. package/dist/artifact-verification.js +172 -0
  6. package/dist/builtin-benchmarks/harness-refiner.js +469 -24
  7. package/dist/execution-contracts.js +153 -0
  8. package/dist/execution-receipts.js +185 -0
  9. package/dist/graders.js +8 -2
  10. package/dist/index.js +10 -0
  11. package/dist/learned-preference.js +334 -0
  12. package/dist/preferences.js +803 -0
  13. package/dist/rollouts.js +241 -0
  14. package/dist/tasksets.js +24 -0
  15. package/dist/telemetry/index.js +4 -0
  16. package/dist/telemetry-analysis.js +183 -0
  17. package/dist/telemetry-bundle.js +60 -0
  18. package/dist/telemetry-catalog.js +50 -0
  19. package/dist/telemetry.js +112 -0
  20. package/dist/types/artifact-verification.d.ts +31 -0
  21. package/dist/types/artifact-verification.d.ts.map +1 -0
  22. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +22 -0
  23. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  24. package/dist/types/conformance.d.ts +44 -0
  25. package/dist/types/conformance.d.ts.map +1 -1
  26. package/dist/types/evidence/conformance.d.ts +24 -24
  27. package/dist/types/evidence/contracts.d.ts +42 -42
  28. package/dist/types/execution-contracts.d.ts +806 -0
  29. package/dist/types/execution-contracts.d.ts.map +1 -0
  30. package/dist/types/execution-receipts.d.ts +55 -0
  31. package/dist/types/execution-receipts.d.ts.map +1 -0
  32. package/dist/types/graders.d.ts.map +1 -1
  33. package/dist/types/index.d.ts +10 -0
  34. package/dist/types/index.d.ts.map +1 -1
  35. package/dist/types/learned-preference.d.ts +282 -0
  36. package/dist/types/learned-preference.d.ts.map +1 -0
  37. package/dist/types/preferences.d.ts +583 -0
  38. package/dist/types/preferences.d.ts.map +1 -0
  39. package/dist/types/review-conformance.d.ts +20 -15
  40. package/dist/types/review-conformance.d.ts.map +1 -1
  41. package/dist/types/rollouts.d.ts +221 -0
  42. package/dist/types/rollouts.d.ts.map +1 -0
  43. package/dist/types/tasksets.d.ts +89 -0
  44. package/dist/types/tasksets.d.ts.map +1 -1
  45. package/dist/types/telemetry/index.d.ts +5 -0
  46. package/dist/types/telemetry/index.d.ts.map +1 -0
  47. package/dist/types/telemetry-analysis.d.ts +116 -0
  48. package/dist/types/telemetry-analysis.d.ts.map +1 -0
  49. package/dist/types/telemetry-bundle.d.ts +309 -0
  50. package/dist/types/telemetry-bundle.d.ts.map +1 -0
  51. package/dist/types/telemetry-catalog.d.ts +269 -0
  52. package/dist/types/telemetry-catalog.d.ts.map +1 -0
  53. package/dist/types/telemetry.d.ts +266 -0
  54. package/dist/types/telemetry.d.ts.map +1 -0
  55. package/package.json +20 -2
  56. package/schemas/telemetry/v1/evidence-completeness.schema.json +82 -0
  57. package/schemas/telemetry/v1/evidence-reference.schema.json +41 -0
  58. package/schemas/telemetry/v1/metric-definition.schema.json +96 -0
  59. package/schemas/telemetry/v1/metric-observation.schema.json +182 -0
  60. package/schemas/telemetry/v1/run-metric-summary.schema.json +98 -0
  61. package/schemas/telemetry/v1/run-telemetry-batch.schema.json +405 -0
  62. package/schemas/telemetry/v1/run-telemetry-event.schema.json +201 -0
  63. package/schemas/telemetry/v1/telemetry-cohort.schema.json +100 -0
  64. package/schemas/telemetry/v1/telemetry-export-bundle.schema.json +655 -0
package/dist/types/evidence/contracts.d.ts CHANGED
@@ -13,9 +13,9 @@ export declare const EvidenceArtifactRefSchema: z.ZodObject<{
13
13
  id: z.ZodString;
14
14
  }, z.core.$strict>;
15
15
  export declare const WorkProcessStepKindSchema: z.ZodEnum<{
16
- approval: "approval";
17
16
  validation: "validation";
18
17
  artifact: "artifact";
18
+ approval: "approval";
19
19
  tool: "tool";
20
20
  state_transition: "state_transition";
21
21
  question: "question";
@@ -50,10 +50,10 @@ export declare const WorkProcessActionSchema: z.ZodEnum<{
50
50
  cleanup_failed: "cleanup_failed";
51
51
  }>;
52
52
  export declare const WorkProcessStepStatusSchema: z.ZodEnum<{
53
- cancelled: "cancelled";
54
53
  failed: "failed";
55
- started: "started";
56
54
  completed: "completed";
55
+ cancelled: "cancelled";
56
+ started: "started";
57
57
  }>;
58
58
  export declare const WorkToolCategorySchema: z.ZodEnum<{
59
59
  agent: "agent";
@@ -73,10 +73,10 @@ export declare const WorkValidationKindSchema: z.ZodEnum<{
73
73
  other: "other";
74
74
  }>;
75
75
  export declare const WorkTransitionStateSchema: z.ZodEnum<{
76
- timeout: "timeout";
77
- cancelled: "cancelled";
78
76
  failed: "failed";
79
77
  completed: "completed";
78
+ cancelled: "cancelled";
79
+ timeout: "timeout";
80
80
  running: "running";
81
81
  }>;
82
82
  export declare const WorkInterventionOutcomeSchema: z.ZodEnum<{
@@ -87,12 +87,12 @@ export declare const WorkInterventionOutcomeSchema: z.ZodEnum<{
87
87
  dismissed: "dismissed";
88
88
  }>;
89
89
  export declare const WorkProcessErrorClassSchema: z.ZodEnum<{
90
- unknown: "unknown";
91
- timeout: "timeout";
92
- cancelled: "cancelled";
93
90
  validation: "validation";
94
91
  policy: "policy";
92
+ unknown: "unknown";
95
93
  environment: "environment";
94
+ cancelled: "cancelled";
95
+ timeout: "timeout";
96
96
  infrastructure: "infrastructure";
97
97
  }>;
98
98
  export declare const WorkTraceIncompleteReasonSchema: z.ZodEnum<{
@@ -111,9 +111,9 @@ export declare const WorkProcessStepSchema: z.ZodObject<{
111
111
  environment: "environment";
112
112
  }>;
113
113
  kind: z.ZodEnum<{
114
- approval: "approval";
115
114
  validation: "validation";
116
115
  artifact: "artifact";
116
+ approval: "approval";
117
117
  tool: "tool";
118
118
  state_transition: "state_transition";
119
119
  question: "question";
@@ -144,10 +144,10 @@ export declare const WorkProcessStepSchema: z.ZodObject<{
144
144
  cleanup_failed: "cleanup_failed";
145
145
  }>;
146
146
  status: z.ZodEnum<{
147
- cancelled: "cancelled";
148
147
  failed: "failed";
149
- started: "started";
150
148
  completed: "completed";
149
+ cancelled: "cancelled";
150
+ started: "started";
151
151
  }>;
152
152
  inputHash: z.ZodNullable<z.ZodString>;
153
153
  outputHash: z.ZodNullable<z.ZodString>;
@@ -178,10 +178,10 @@ export declare const WorkProcessStepSchema: z.ZodObject<{
178
178
  other: "other";
179
179
  }>>;
180
180
  transitionState: z.ZodNullable<z.ZodEnum<{
181
- timeout: "timeout";
182
- cancelled: "cancelled";
183
181
  failed: "failed";
184
182
  completed: "completed";
183
+ cancelled: "cancelled";
184
+ timeout: "timeout";
185
185
  running: "running";
186
186
  }>>;
187
187
  interventionOutcome: z.ZodNullable<z.ZodEnum<{
@@ -197,12 +197,12 @@ export declare const WorkProcessStepSchema: z.ZodObject<{
197
197
  cpuTimeMs: z.ZodNullable<z.ZodNumber>;
198
198
  memoryPeakBytes: z.ZodNullable<z.ZodNumber>;
199
199
  errorClass: z.ZodNullable<z.ZodEnum<{
200
- unknown: "unknown";
201
- timeout: "timeout";
202
- cancelled: "cancelled";
203
200
  validation: "validation";
204
201
  policy: "policy";
202
+ unknown: "unknown";
205
203
  environment: "environment";
204
+ cancelled: "cancelled";
205
+ timeout: "timeout";
206
206
  infrastructure: "infrastructure";
207
207
  }>>;
208
208
  }, z.core.$strict>;
@@ -229,9 +229,9 @@ export declare const WorkProcessTraceContentSchema: z.ZodObject<{
229
229
  environment: "environment";
230
230
  }>;
231
231
  kind: z.ZodEnum<{
232
- approval: "approval";
233
232
  validation: "validation";
234
233
  artifact: "artifact";
234
+ approval: "approval";
235
235
  tool: "tool";
236
236
  state_transition: "state_transition";
237
237
  question: "question";
@@ -262,10 +262,10 @@ export declare const WorkProcessTraceContentSchema: z.ZodObject<{
262
262
  cleanup_failed: "cleanup_failed";
263
263
  }>;
264
264
  status: z.ZodEnum<{
265
- cancelled: "cancelled";
266
265
  failed: "failed";
267
- started: "started";
268
266
  completed: "completed";
267
+ cancelled: "cancelled";
268
+ started: "started";
269
269
  }>;
270
270
  inputHash: z.ZodNullable<z.ZodString>;
271
271
  outputHash: z.ZodNullable<z.ZodString>;
@@ -296,10 +296,10 @@ export declare const WorkProcessTraceContentSchema: z.ZodObject<{
296
296
  other: "other";
297
297
  }>>;
298
298
  transitionState: z.ZodNullable<z.ZodEnum<{
299
- timeout: "timeout";
300
- cancelled: "cancelled";
301
299
  failed: "failed";
302
300
  completed: "completed";
301
+ cancelled: "cancelled";
302
+ timeout: "timeout";
303
303
  running: "running";
304
304
  }>>;
305
305
  interventionOutcome: z.ZodNullable<z.ZodEnum<{
@@ -315,12 +315,12 @@ export declare const WorkProcessTraceContentSchema: z.ZodObject<{
315
315
  cpuTimeMs: z.ZodNullable<z.ZodNumber>;
316
316
  memoryPeakBytes: z.ZodNullable<z.ZodNumber>;
317
317
  errorClass: z.ZodNullable<z.ZodEnum<{
318
- unknown: "unknown";
319
- timeout: "timeout";
320
- cancelled: "cancelled";
321
318
  validation: "validation";
322
319
  policy: "policy";
320
+ unknown: "unknown";
323
321
  environment: "environment";
322
+ cancelled: "cancelled";
323
+ timeout: "timeout";
324
324
  infrastructure: "infrastructure";
325
325
  }>>;
326
326
  }, z.core.$strict>;
@@ -348,9 +348,9 @@ export declare const WorkProcessTraceSchema: z.ZodObject<{
348
348
  environment: "environment";
349
349
  }>;
350
350
  kind: z.ZodEnum<{
351
- approval: "approval";
352
351
  validation: "validation";
353
352
  artifact: "artifact";
353
+ approval: "approval";
354
354
  tool: "tool";
355
355
  state_transition: "state_transition";
356
356
  question: "question";
@@ -381,10 +381,10 @@ export declare const WorkProcessTraceSchema: z.ZodObject<{
381
381
  cleanup_failed: "cleanup_failed";
382
382
  }>;
383
383
  status: z.ZodEnum<{
384
- cancelled: "cancelled";
385
384
  failed: "failed";
386
- started: "started";
387
385
  completed: "completed";
386
+ cancelled: "cancelled";
387
+ started: "started";
388
388
  }>;
389
389
  inputHash: z.ZodNullable<z.ZodString>;
390
390
  outputHash: z.ZodNullable<z.ZodString>;
@@ -415,10 +415,10 @@ export declare const WorkProcessTraceSchema: z.ZodObject<{
415
415
  other: "other";
416
416
  }>>;
417
417
  transitionState: z.ZodNullable<z.ZodEnum<{
418
- timeout: "timeout";
419
- cancelled: "cancelled";
420
418
  failed: "failed";
421
419
  completed: "completed";
420
+ cancelled: "cancelled";
421
+ timeout: "timeout";
422
422
  running: "running";
423
423
  }>>;
424
424
  interventionOutcome: z.ZodNullable<z.ZodEnum<{
@@ -434,12 +434,12 @@ export declare const WorkProcessTraceSchema: z.ZodObject<{
434
434
  cpuTimeMs: z.ZodNullable<z.ZodNumber>;
435
435
  memoryPeakBytes: z.ZodNullable<z.ZodNumber>;
436
436
  errorClass: z.ZodNullable<z.ZodEnum<{
437
- unknown: "unknown";
438
- timeout: "timeout";
439
- cancelled: "cancelled";
440
437
  validation: "validation";
441
438
  policy: "policy";
439
+ unknown: "unknown";
442
440
  environment: "environment";
441
+ cancelled: "cancelled";
442
+ timeout: "timeout";
443
443
  infrastructure: "infrastructure";
444
444
  }>>;
445
445
  }, z.core.$strict>;
@@ -450,26 +450,26 @@ export declare const WorkFailureClassSchema: z.ZodEnum<{
450
450
  unknown: "unknown";
451
451
  policy_failure: "policy_failure";
452
452
  environment_failure: "environment_failure";
453
+ cancelled: "cancelled";
453
454
  infrastructure_failure: "infrastructure_failure";
454
455
  timeout: "timeout";
455
- cancelled: "cancelled";
456
456
  validation_failure: "validation_failure";
457
457
  model_failure: "model_failure";
458
458
  }>;
459
459
  export declare const WorkTerminalSchema: z.ZodObject<{
460
460
  status: z.ZodEnum<{
461
- timeout: "timeout";
462
- cancelled: "cancelled";
463
461
  failed: "failed";
464
462
  completed: "completed";
463
+ cancelled: "cancelled";
464
+ timeout: "timeout";
465
465
  }>;
466
466
  failureClass: z.ZodNullable<z.ZodEnum<{
467
467
  unknown: "unknown";
468
468
  policy_failure: "policy_failure";
469
469
  environment_failure: "environment_failure";
470
+ cancelled: "cancelled";
470
471
  infrastructure_failure: "infrastructure_failure";
471
472
  timeout: "timeout";
472
- cancelled: "cancelled";
473
473
  validation_failure: "validation_failure";
474
474
  model_failure: "model_failure";
475
475
  }>>;
@@ -514,18 +514,18 @@ export declare const WorkEvidenceReceiptContentSchema: z.ZodObject<{
514
514
  inputHash: z.ZodString;
515
515
  terminal: z.ZodObject<{
516
516
  status: z.ZodEnum<{
517
- timeout: "timeout";
518
- cancelled: "cancelled";
519
517
  failed: "failed";
520
518
  completed: "completed";
519
+ cancelled: "cancelled";
520
+ timeout: "timeout";
521
521
  }>;
522
522
  failureClass: z.ZodNullable<z.ZodEnum<{
523
523
  unknown: "unknown";
524
524
  policy_failure: "policy_failure";
525
525
  environment_failure: "environment_failure";
526
+ cancelled: "cancelled";
526
527
  infrastructure_failure: "infrastructure_failure";
527
528
  timeout: "timeout";
528
- cancelled: "cancelled";
529
529
  validation_failure: "validation_failure";
530
530
  model_failure: "model_failure";
531
531
  }>>;
@@ -644,18 +644,18 @@ export declare const WorkEvidenceReceiptSchema: z.ZodObject<{
644
644
  inputHash: z.ZodString;
645
645
  terminal: z.ZodObject<{
646
646
  status: z.ZodEnum<{
647
- timeout: "timeout";
648
- cancelled: "cancelled";
649
647
  failed: "failed";
650
648
  completed: "completed";
649
+ cancelled: "cancelled";
650
+ timeout: "timeout";
651
651
  }>;
652
652
  failureClass: z.ZodNullable<z.ZodEnum<{
653
653
  unknown: "unknown";
654
654
  policy_failure: "policy_failure";
655
655
  environment_failure: "environment_failure";
656
+ cancelled: "cancelled";
656
657
  infrastructure_failure: "infrastructure_failure";
657
658
  timeout: "timeout";
658
- cancelled: "cancelled";
659
659
  validation_failure: "validation_failure";
660
660
  model_failure: "model_failure";
661
661
  }>>;