@openpond/evals 0.4.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CONTRACT.md +27 -1
  2. package/README.md +31 -2
  3. package/conformance/telemetry/v1/invalid-batch.json +17 -0
  4. package/conformance/telemetry/v1/valid-batch.json +58 -0
  5. package/dist/artifact-verification.js +172 -0
  6. package/dist/builtin-benchmarks/harness-refiner.js +469 -24
  7. package/dist/execution-contracts.js +153 -0
  8. package/dist/execution-receipts.js +185 -0
  9. package/dist/graders.js +8 -2
  10. package/dist/index.js +10 -0
  11. package/dist/learned-preference.js +334 -0
  12. package/dist/preferences.js +803 -0
  13. package/dist/rollouts.js +241 -0
  14. package/dist/tasksets.js +24 -0
  15. package/dist/telemetry/index.js +4 -0
  16. package/dist/telemetry-analysis.js +183 -0
  17. package/dist/telemetry-bundle.js +60 -0
  18. package/dist/telemetry-catalog.js +50 -0
  19. package/dist/telemetry.js +112 -0
  20. package/dist/types/artifact-verification.d.ts +31 -0
  21. package/dist/types/artifact-verification.d.ts.map +1 -0
  22. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +22 -0
  23. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  24. package/dist/types/conformance.d.ts +44 -0
  25. package/dist/types/conformance.d.ts.map +1 -1
  26. package/dist/types/evidence/conformance.d.ts +24 -24
  27. package/dist/types/evidence/contracts.d.ts +42 -42
  28. package/dist/types/execution-contracts.d.ts +806 -0
  29. package/dist/types/execution-contracts.d.ts.map +1 -0
  30. package/dist/types/execution-receipts.d.ts +55 -0
  31. package/dist/types/execution-receipts.d.ts.map +1 -0
  32. package/dist/types/graders.d.ts.map +1 -1
  33. package/dist/types/index.d.ts +10 -0
  34. package/dist/types/index.d.ts.map +1 -1
  35. package/dist/types/learned-preference.d.ts +282 -0
  36. package/dist/types/learned-preference.d.ts.map +1 -0
  37. package/dist/types/preferences.d.ts +583 -0
  38. package/dist/types/preferences.d.ts.map +1 -0
  39. package/dist/types/review-conformance.d.ts +20 -15
  40. package/dist/types/review-conformance.d.ts.map +1 -1
  41. package/dist/types/rollouts.d.ts +221 -0
  42. package/dist/types/rollouts.d.ts.map +1 -0
  43. package/dist/types/tasksets.d.ts +89 -0
  44. package/dist/types/tasksets.d.ts.map +1 -1
  45. package/dist/types/telemetry/index.d.ts +5 -0
  46. package/dist/types/telemetry/index.d.ts.map +1 -0
  47. package/dist/types/telemetry-analysis.d.ts +116 -0
  48. package/dist/types/telemetry-analysis.d.ts.map +1 -0
  49. package/dist/types/telemetry-bundle.d.ts +309 -0
  50. package/dist/types/telemetry-bundle.d.ts.map +1 -0
  51. package/dist/types/telemetry-catalog.d.ts +269 -0
  52. package/dist/types/telemetry-catalog.d.ts.map +1 -0
  53. package/dist/types/telemetry.d.ts +266 -0
  54. package/dist/types/telemetry.d.ts.map +1 -0
  55. package/package.json +20 -2
  56. package/schemas/telemetry/v1/evidence-completeness.schema.json +82 -0
  57. package/schemas/telemetry/v1/evidence-reference.schema.json +41 -0
  58. package/schemas/telemetry/v1/metric-definition.schema.json +96 -0
  59. package/schemas/telemetry/v1/metric-observation.schema.json +182 -0
  60. package/schemas/telemetry/v1/run-metric-summary.schema.json +98 -0
  61. package/schemas/telemetry/v1/run-telemetry-batch.schema.json +405 -0
  62. package/schemas/telemetry/v1/run-telemetry-event.schema.json +201 -0
  63. package/schemas/telemetry/v1/telemetry-cohort.schema.json +100 -0
  64. package/schemas/telemetry/v1/telemetry-export-bundle.schema.json +655 -0
package/schemas/telemetry/v1/telemetry-export-bundle.schema.json ADDED
@@ -0,0 +1,655 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "type": "object",
4
+ "properties": {
5
+ "schemaVersion": {
6
+ "type": "string",
7
+ "const": "openpond.telemetryExportBundle.v1"
8
+ },
9
+ "id": {
10
+ "type": "string",
11
+ "minLength": 1,
12
+ "maxLength": 240
13
+ },
14
+ "runId": {
15
+ "type": "string",
16
+ "minLength": 1,
17
+ "maxLength": 240
18
+ },
19
+ "exportedAt": {
20
+ "type": "string",
21
+ "format": "date-time",
22
+ "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"
23
+ },
24
+ "definitions": {
25
+ "minItems": 1,
26
+ "type": "array",
27
+ "items": {
28
+ "type": "object",
29
+ "properties": {
30
+ "schemaVersion": {
31
+ "type": "string",
32
+ "const": "openpond.metricDefinition.v1"
33
+ },
34
+ "id": {
35
+ "type": "string",
36
+ "minLength": 1,
37
+ "maxLength": 240
38
+ },
39
+ "displayName": {
40
+ "type": "string",
41
+ "minLength": 1,
42
+ "maxLength": 200
43
+ },
44
+ "description": {
45
+ "type": "string",
46
+ "minLength": 1,
47
+ "maxLength": 2000
48
+ },
49
+ "valueType": {
50
+ "type": "string",
51
+ "enum": [
52
+ "gauge",
53
+ "counter",
54
+ "distribution"
55
+ ]
56
+ },
57
+ "unit": {
58
+ "type": "string",
59
+ "enum": [
60
+ "ratio",
61
+ "count",
62
+ "seconds",
63
+ "milliseconds",
64
+ "tokens",
65
+ "bytes",
66
+ "usd",
67
+ "scalar"
68
+ ]
69
+ },
70
+ "direction": {
71
+ "type": "string",
72
+ "enum": [
73
+ "higher",
74
+ "lower",
75
+ "neutral"
76
+ ]
77
+ },
78
+ "aggregation": {
79
+ "type": "string",
80
+ "enum": [
81
+ "last",
82
+ "sum",
83
+ "mean",
84
+ "min",
85
+ "max",
86
+ "p50",
87
+ "p95"
88
+ ]
89
+ },
90
+ "visibility": {
91
+ "type": "string",
92
+ "enum": [
93
+ "policy_visible",
94
+ "team_visible",
95
+ "host_private"
96
+ ]
97
+ },
98
+ "boundedDimensions": {
99
+ "maxItems": 32,
100
+ "type": "array",
101
+ "items": {
102
+ "type": "string",
103
+ "minLength": 1,
104
+ "maxLength": 100
105
+ }
106
+ }
107
+ },
108
+ "required": [
109
+ "schemaVersion",
110
+ "id",
111
+ "displayName",
112
+ "description",
113
+ "valueType",
114
+ "unit",
115
+ "direction",
116
+ "aggregation",
117
+ "visibility",
118
+ "boundedDimensions"
119
+ ],
120
+ "additionalProperties": false
121
+ }
122
+ },
123
+ "events": {
124
+ "maxItems": 1000000,
125
+ "type": "array",
126
+ "items": {
127
+ "type": "object",
128
+ "properties": {
129
+ "schemaVersion": {
130
+ "type": "string",
131
+ "const": "openpond.runTelemetryEvent.v1"
132
+ },
133
+ "eventId": {
134
+ "type": "string",
135
+ "minLength": 1,
136
+ "maxLength": 240
137
+ },
138
+ "sequence": {
139
+ "type": "integer",
140
+ "minimum": 0,
141
+ "maximum": 9007199254740991
142
+ },
143
+ "occurredAt": {
144
+ "type": "string",
145
+ "format": "date-time",
146
+ "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"
147
+ },
148
+ "source": {
149
+ "type": "string",
150
+ "enum": [
151
+ "runtime",
152
+ "environment",
153
+ "grader",
154
+ "optimizer",
155
+ "control_plane",
156
+ "evaluation"
157
+ ]
158
+ },
159
+ "type": {
160
+ "type": "string",
161
+ "enum": [
162
+ "run_started",
163
+ "run_state_changed",
164
+ "rollout_group_started",
165
+ "attempt_completed",
166
+ "grader_completed",
167
+ "reward_composed",
168
+ "optimizer_step_completed",
169
+ "checkpoint_committed",
170
+ "evaluation_completed",
171
+ "run_completed",
172
+ "run_failed",
173
+ "cleanup_completed"
174
+ ]
175
+ },
176
+ "visibility": {
177
+ "type": "string",
178
+ "enum": [
179
+ "policy_visible",
180
+ "team_visible",
181
+ "host_private"
182
+ ]
183
+ },
184
+ "lineage": {
185
+ "type": "object",
186
+ "properties": {
187
+ "modelProjectId": {
188
+ "type": "string",
189
+ "minLength": 1,
190
+ "maxLength": 240
191
+ },
192
+ "runId": {
193
+ "type": "string",
194
+ "minLength": 1,
195
+ "maxLength": 240
196
+ },
197
+ "modelVersionId": {
198
+ "anyOf": [
199
+ {
200
+ "type": "string",
201
+ "minLength": 1,
202
+ "maxLength": 240
203
+ },
204
+ {
205
+ "type": "null"
206
+ }
207
+ ]
208
+ },
209
+ "harnessReleaseHash": {
210
+ "type": "string",
211
+ "pattern": "^[a-f0-9]{64}$"
212
+ },
213
+ "tasksetReleaseHash": {
214
+ "type": "string",
215
+ "pattern": "^[a-f0-9]{64}$"
216
+ },
217
+ "environmentReleaseHash": {
218
+ "anyOf": [
219
+ {
220
+ "type": "string",
221
+ "pattern": "^[a-f0-9]{64}$"
222
+ },
223
+ {
224
+ "type": "null"
225
+ }
226
+ ]
227
+ },
228
+ "checkpointId": {
229
+ "anyOf": [
230
+ {
231
+ "type": "string",
232
+ "minLength": 1,
233
+ "maxLength": 240
234
+ },
235
+ {
236
+ "type": "null"
237
+ }
238
+ ]
239
+ },
240
+ "step": {
241
+ "anyOf": [
242
+ {
243
+ "type": "integer",
244
+ "minimum": 0,
245
+ "maximum": 9007199254740991
246
+ },
247
+ {
248
+ "type": "null"
249
+ }
250
+ ]
251
+ },
252
+ "rolloutGroupId": {
253
+ "anyOf": [
254
+ {
255
+ "type": "string",
256
+ "minLength": 1,
257
+ "maxLength": 240
258
+ },
259
+ {
260
+ "type": "null"
261
+ }
262
+ ]
263
+ },
264
+ "attemptId": {
265
+ "anyOf": [
266
+ {
267
+ "type": "string",
268
+ "minLength": 1,
269
+ "maxLength": 240
270
+ },
271
+ {
272
+ "type": "null"
273
+ }
274
+ ]
275
+ },
276
+ "scenarioId": {
277
+ "anyOf": [
278
+ {
279
+ "type": "string",
280
+ "minLength": 1,
281
+ "maxLength": 240
282
+ },
283
+ {
284
+ "type": "null"
285
+ }
286
+ ]
287
+ }
288
+ },
289
+ "required": [
290
+ "modelProjectId",
291
+ "runId",
292
+ "modelVersionId",
293
+ "harnessReleaseHash",
294
+ "tasksetReleaseHash",
295
+ "environmentReleaseHash",
296
+ "checkpointId",
297
+ "step",
298
+ "rolloutGroupId",
299
+ "attemptId",
300
+ "scenarioId"
301
+ ],
302
+ "additionalProperties": false
303
+ },
304
+ "attributes": {
305
+ "default": {},
306
+ "type": "object",
307
+ "propertyNames": {
308
+ "type": "string"
309
+ },
310
+ "additionalProperties": {}
311
+ }
312
+ },
313
+ "required": [
314
+ "schemaVersion",
315
+ "eventId",
316
+ "sequence",
317
+ "occurredAt",
318
+ "source",
319
+ "type",
320
+ "visibility",
321
+ "lineage",
322
+ "attributes"
323
+ ],
324
+ "additionalProperties": false
325
+ }
326
+ },
327
+ "observations": {
328
+ "maxItems": 10000000,
329
+ "type": "array",
330
+ "items": {
331
+ "type": "object",
332
+ "properties": {
333
+ "schemaVersion": {
334
+ "type": "string",
335
+ "const": "openpond.metricObservation.v1"
336
+ },
337
+ "observationId": {
338
+ "type": "string",
339
+ "minLength": 1,
340
+ "maxLength": 240
341
+ },
342
+ "metricId": {
343
+ "type": "string",
344
+ "minLength": 1,
345
+ "maxLength": 240
346
+ },
347
+ "eventId": {
348
+ "type": "string",
349
+ "minLength": 1,
350
+ "maxLength": 240
351
+ },
352
+ "sequence": {
353
+ "type": "integer",
354
+ "minimum": 0,
355
+ "maximum": 9007199254740991
356
+ },
357
+ "observedAt": {
358
+ "type": "string",
359
+ "format": "date-time",
360
+ "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"
361
+ },
362
+ "value": {
363
+ "type": "number"
364
+ },
365
+ "lineage": {
366
+ "type": "object",
367
+ "properties": {
368
+ "modelProjectId": {
369
+ "type": "string",
370
+ "minLength": 1,
371
+ "maxLength": 240
372
+ },
373
+ "runId": {
374
+ "type": "string",
375
+ "minLength": 1,
376
+ "maxLength": 240
377
+ },
378
+ "modelVersionId": {
379
+ "anyOf": [
380
+ {
381
+ "type": "string",
382
+ "minLength": 1,
383
+ "maxLength": 240
384
+ },
385
+ {
386
+ "type": "null"
387
+ }
388
+ ]
389
+ },
390
+ "harnessReleaseHash": {
391
+ "type": "string",
392
+ "pattern": "^[a-f0-9]{64}$"
393
+ },
394
+ "tasksetReleaseHash": {
395
+ "type": "string",
396
+ "pattern": "^[a-f0-9]{64}$"
397
+ },
398
+ "environmentReleaseHash": {
399
+ "anyOf": [
400
+ {
401
+ "type": "string",
402
+ "pattern": "^[a-f0-9]{64}$"
403
+ },
404
+ {
405
+ "type": "null"
406
+ }
407
+ ]
408
+ },
409
+ "checkpointId": {
410
+ "anyOf": [
411
+ {
412
+ "type": "string",
413
+ "minLength": 1,
414
+ "maxLength": 240
415
+ },
416
+ {
417
+ "type": "null"
418
+ }
419
+ ]
420
+ },
421
+ "step": {
422
+ "anyOf": [
423
+ {
424
+ "type": "integer",
425
+ "minimum": 0,
426
+ "maximum": 9007199254740991
427
+ },
428
+ {
429
+ "type": "null"
430
+ }
431
+ ]
432
+ },
433
+ "rolloutGroupId": {
434
+ "anyOf": [
435
+ {
436
+ "type": "string",
437
+ "minLength": 1,
438
+ "maxLength": 240
439
+ },
440
+ {
441
+ "type": "null"
442
+ }
443
+ ]
444
+ },
445
+ "attemptId": {
446
+ "anyOf": [
447
+ {
448
+ "type": "string",
449
+ "minLength": 1,
450
+ "maxLength": 240
451
+ },
452
+ {
453
+ "type": "null"
454
+ }
455
+ ]
456
+ },
457
+ "scenarioId": {
458
+ "anyOf": [
459
+ {
460
+ "type": "string",
461
+ "minLength": 1,
462
+ "maxLength": 240
463
+ },
464
+ {
465
+ "type": "null"
466
+ }
467
+ ]
468
+ }
469
+ },
470
+ "required": [
471
+ "modelProjectId",
472
+ "runId",
473
+ "modelVersionId",
474
+ "harnessReleaseHash",
475
+ "tasksetReleaseHash",
476
+ "environmentReleaseHash",
477
+ "checkpointId",
478
+ "step",
479
+ "rolloutGroupId",
480
+ "attemptId",
481
+ "scenarioId"
482
+ ],
483
+ "additionalProperties": false
484
+ },
485
+ "dimensions": {
486
+ "type": "object",
487
+ "propertyNames": {
488
+ "type": "string",
489
+ "minLength": 1,
490
+ "maxLength": 100
491
+ },
492
+ "additionalProperties": {
493
+ "type": "string",
494
+ "maxLength": 500
495
+ }
496
+ }
497
+ },
498
+ "required": [
499
+ "schemaVersion",
500
+ "observationId",
501
+ "metricId",
502
+ "eventId",
503
+ "sequence",
504
+ "observedAt",
505
+ "value",
506
+ "lineage",
507
+ "dimensions"
508
+ ],
509
+ "additionalProperties": false
510
+ }
511
+ },
512
+ "evidenceRefs": {
513
+ "maxItems": 1000000,
514
+ "type": "array",
515
+ "items": {
516
+ "type": "object",
517
+ "properties": {
518
+ "id": {
519
+ "type": "string",
520
+ "minLength": 1,
521
+ "maxLength": 240
522
+ },
523
+ "contentHash": {
524
+ "type": "string",
525
+ "pattern": "^[a-f0-9]{64}$"
526
+ },
527
+ "kind": {
528
+ "type": "string",
529
+ "enum": [
530
+ "rollout",
531
+ "attempt",
532
+ "trace",
533
+ "grader",
534
+ "checkpoint",
535
+ "artifact"
536
+ ]
537
+ },
538
+ "visibility": {
539
+ "type": "string",
540
+ "enum": [
541
+ "policy_visible",
542
+ "team_visible",
543
+ "host_private"
544
+ ]
545
+ }
546
+ },
547
+ "required": [
548
+ "id",
549
+ "contentHash",
550
+ "kind",
551
+ "visibility"
552
+ ],
553
+ "additionalProperties": false
554
+ }
555
+ },
556
+ "completeness": {
557
+ "type": "object",
558
+ "properties": {
559
+ "schemaVersion": {
560
+ "type": "string",
561
+ "const": "openpond.telemetryEvidenceCompleteness.v1"
562
+ },
563
+ "runId": {
564
+ "type": "string",
565
+ "minLength": 1,
566
+ "maxLength": 240
567
+ },
568
+ "status": {
569
+ "type": "string",
570
+ "enum": [
571
+ "complete",
572
+ "partial",
573
+ "missing"
574
+ ]
575
+ },
576
+ "expectedEventTypes": {
577
+ "maxItems": 100,
578
+ "type": "array",
579
+ "items": {
580
+ "type": "string",
581
+ "minLength": 1,
582
+ "maxLength": 100
583
+ }
584
+ },
585
+ "observedEventTypes": {
586
+ "maxItems": 100,
587
+ "type": "array",
588
+ "items": {
589
+ "type": "string",
590
+ "minLength": 1,
591
+ "maxLength": 100
592
+ }
593
+ },
594
+ "missingEventTypes": {
595
+ "maxItems": 100,
596
+ "type": "array",
597
+ "items": {
598
+ "type": "string",
599
+ "minLength": 1,
600
+ "maxLength": 100
601
+ }
602
+ },
603
+ "lastSequence": {
604
+ "anyOf": [
605
+ {
606
+ "type": "integer",
607
+ "minimum": 0,
608
+ "maximum": 9007199254740991
609
+ },
610
+ {
611
+ "type": "null"
612
+ }
613
+ ]
614
+ },
615
+ "sequenceGaps": {
616
+ "maxItems": 10000,
617
+ "type": "array",
618
+ "items": {
619
+ "type": "integer",
620
+ "minimum": 0,
621
+ "maximum": 9007199254740991
622
+ }
623
+ }
624
+ },
625
+ "required": [
626
+ "schemaVersion",
627
+ "runId",
628
+ "status",
629
+ "expectedEventTypes",
630
+ "observedEventTypes",
631
+ "missingEventTypes",
632
+ "lastSequence",
633
+ "sequenceGaps"
634
+ ],
635
+ "additionalProperties": false
636
+ },
637
+ "contentHash": {
638
+ "type": "string",
639
+ "pattern": "^[a-f0-9]{64}$"
640
+ }
641
+ },
642
+ "required": [
643
+ "schemaVersion",
644
+ "id",
645
+ "runId",
646
+ "exportedAt",
647
+ "definitions",
648
+ "events",
649
+ "observations",
650
+ "evidenceRefs",
651
+ "completeness",
652
+ "contentHash"
653
+ ],
654
+ "additionalProperties": false
655
+ }