@openpond/evals 0.11.16 → 0.11.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/dist/connected-evidence/contracts.js +72 -0
  2. package/dist/connected-evidence/imports.js +71 -0
  3. package/dist/connected-evidence/index.js +5 -0
  4. package/dist/connected-evidence/normalize.js +97 -0
  5. package/dist/connected-evidence/python-correctness.js +73 -0
  6. package/dist/connected-evidence/sources/claude-code.js +71 -0
  7. package/dist/connected-evidence/sources/codex.js +52 -0
  8. package/dist/connected-evidence/sources/history.js +86 -0
  9. package/dist/connected-evidence/sources/usage.js +15 -0
  10. package/dist/connected-evidence/usage-summary.js +52 -0
  11. package/dist/experiments.js +16 -4
  12. package/dist/graders.js +9 -6
  13. package/dist/human-review/comparison.js +161 -0
  14. package/dist/human-review/contracts.js +110 -0
  15. package/dist/human-review/execution.js +61 -0
  16. package/dist/human-review/handoff.js +18 -0
  17. package/dist/human-review/index.js +9 -0
  18. package/dist/human-review/local-publication.js +18 -0
  19. package/dist/human-review/repository.js +1 -0
  20. package/dist/human-review/results.js +18 -0
  21. package/dist/human-review/service.js +268 -0
  22. package/dist/human-review/task-proposals.js +39 -0
  23. package/dist/index.js +1 -0
  24. package/dist/javascript-verifier-contract.js +2 -2
  25. package/dist/learning/authoring.js +2 -1
  26. package/dist/learning/contracts.js +353 -81
  27. package/dist/learning/index.js +1 -0
  28. package/dist/learning/reward-authoring.js +5 -2
  29. package/dist/learning/training-handoff.js +32 -0
  30. package/dist/metrics-node.js +2 -0
  31. package/dist/profile-evaluation-target.js +7 -0
  32. package/dist/profile-evaluations.js +7 -10
  33. package/dist/profile-external-evaluation.js +35 -0
  34. package/dist/profile-source-candidate.js +5 -0
  35. package/dist/rewards.js +5 -3
  36. package/dist/taskset-run-contract.js +10 -0
  37. package/dist/tasksets.js +2 -0
  38. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +75 -1
  39. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  40. package/dist/types/conformance.d.ts +150 -2
  41. package/dist/types/conformance.d.ts.map +1 -1
  42. package/dist/types/connected-evidence/contracts.d.ts +317 -0
  43. package/dist/types/connected-evidence/contracts.d.ts.map +1 -0
  44. package/dist/types/connected-evidence/imports.d.ts +19 -0
  45. package/dist/types/connected-evidence/imports.d.ts.map +1 -0
  46. package/dist/types/connected-evidence/index.d.ts +6 -0
  47. package/dist/types/connected-evidence/index.d.ts.map +1 -0
  48. package/dist/types/connected-evidence/normalize.d.ts +25 -0
  49. package/dist/types/connected-evidence/normalize.d.ts.map +1 -0
  50. package/dist/types/connected-evidence/python-correctness.d.ts +196 -0
  51. package/dist/types/connected-evidence/python-correctness.d.ts.map +1 -0
  52. package/dist/types/connected-evidence/sources/claude-code.d.ts +74 -0
  53. package/dist/types/connected-evidence/sources/claude-code.d.ts.map +1 -0
  54. package/dist/types/connected-evidence/sources/codex.d.ts +74 -0
  55. package/dist/types/connected-evidence/sources/codex.d.ts.map +1 -0
  56. package/dist/types/connected-evidence/sources/history.d.ts +146 -0
  57. package/dist/types/connected-evidence/sources/history.d.ts.map +1 -0
  58. package/dist/types/connected-evidence/sources/usage.d.ts +3 -0
  59. package/dist/types/connected-evidence/sources/usage.d.ts.map +1 -0
  60. package/dist/types/connected-evidence/usage-summary.d.ts +162 -0
  61. package/dist/types/connected-evidence/usage-summary.d.ts.map +1 -0
  62. package/dist/types/evidence/conformance.d.ts +28 -28
  63. package/dist/types/evidence/contracts.d.ts +27 -27
  64. package/dist/types/execution-contracts.d.ts +159 -5
  65. package/dist/types/execution-contracts.d.ts.map +1 -1
  66. package/dist/types/experiments.d.ts +476 -7
  67. package/dist/types/experiments.d.ts.map +1 -1
  68. package/dist/types/graders.d.ts +21 -1
  69. package/dist/types/graders.d.ts.map +1 -1
  70. package/dist/types/human-review/comparison.d.ts +1959 -0
  71. package/dist/types/human-review/comparison.d.ts.map +1 -0
  72. package/dist/types/human-review/contracts.d.ts +2719 -0
  73. package/dist/types/human-review/contracts.d.ts.map +1 -0
  74. package/dist/types/human-review/execution.d.ts +298 -0
  75. package/dist/types/human-review/execution.d.ts.map +1 -0
  76. package/dist/types/human-review/handoff.d.ts +124 -0
  77. package/dist/types/human-review/handoff.d.ts.map +1 -0
  78. package/dist/types/human-review/index.d.ts +10 -0
  79. package/dist/types/human-review/index.d.ts.map +1 -0
  80. package/dist/types/human-review/local-publication.d.ts +165 -0
  81. package/dist/types/human-review/local-publication.d.ts.map +1 -0
  82. package/dist/types/human-review/repository.d.ts +43 -0
  83. package/dist/types/human-review/repository.d.ts.map +1 -0
  84. package/dist/types/human-review/results.d.ts +118 -0
  85. package/dist/types/human-review/results.d.ts.map +1 -0
  86. package/dist/types/human-review/service.d.ts +878 -0
  87. package/dist/types/human-review/service.d.ts.map +1 -0
  88. package/dist/types/human-review/task-proposals.d.ts +70 -0
  89. package/dist/types/human-review/task-proposals.d.ts.map +1 -0
  90. package/dist/types/index.d.ts +1 -0
  91. package/dist/types/index.d.ts.map +1 -1
  92. package/dist/types/javascript-environment-contract.d.ts +1 -1
  93. package/dist/types/javascript-verifier-contract.d.ts +1 -1
  94. package/dist/types/javascript-verifier-contract.d.ts.map +1 -1
  95. package/dist/types/learned-preference.d.ts +2 -2
  96. package/dist/types/learning/acceptance-group.d.ts +13 -13
  97. package/dist/types/learning/acceptance-plan.d.ts +1 -1
  98. package/dist/types/learning/admission.d.ts +78 -1
  99. package/dist/types/learning/admission.d.ts.map +1 -1
  100. package/dist/types/learning/authoring-service.d.ts +2 -0
  101. package/dist/types/learning/authoring-service.d.ts.map +1 -1
  102. package/dist/types/learning/authoring.d.ts +20 -0
  103. package/dist/types/learning/authoring.d.ts.map +1 -1
  104. package/dist/types/learning/batch-service.d.ts +76 -2
  105. package/dist/types/learning/batch-service.d.ts.map +1 -1
  106. package/dist/types/learning/contracts.d.ts +759 -24
  107. package/dist/types/learning/contracts.d.ts.map +1 -1
  108. package/dist/types/learning/index.d.ts +1 -0
  109. package/dist/types/learning/index.d.ts.map +1 -1
  110. package/dist/types/learning/iteration-candidate-decision.d.ts +2 -2
  111. package/dist/types/learning/iteration-dispatch-contracts.d.ts +2 -2
  112. package/dist/types/learning/operations.d.ts +1118 -8
  113. package/dist/types/learning/operations.d.ts.map +1 -1
  114. package/dist/types/learning/repository.d.ts +402 -15
  115. package/dist/types/learning/repository.d.ts.map +1 -1
  116. package/dist/types/learning/reward-authoring.d.ts.map +1 -1
  117. package/dist/types/learning/reward-calibration.d.ts +74 -0
  118. package/dist/types/learning/reward-calibration.d.ts.map +1 -1
  119. package/dist/types/learning/reward-checks.d.ts +14 -2
  120. package/dist/types/learning/reward-checks.d.ts.map +1 -1
  121. package/dist/types/learning/training-handoff.d.ts +201 -0
  122. package/dist/types/learning/training-handoff.d.ts.map +1 -0
  123. package/dist/types/learning/transport.d.ts +963 -21
  124. package/dist/types/learning/transport.d.ts.map +1 -1
  125. package/dist/types/metrics-node.d.ts +3 -1
  126. package/dist/types/metrics-node.d.ts.map +1 -1
  127. package/dist/types/model-improvement-qualification.d.ts +3 -3
  128. package/dist/types/native-harness-experiment-attempt.d.ts +2 -2
  129. package/dist/types/profile-evaluation-comparisons.d.ts +230 -0
  130. package/dist/types/profile-evaluation-comparisons.d.ts.map +1 -1
  131. package/dist/types/profile-evaluation-target.d.ts +14 -0
  132. package/dist/types/profile-evaluation-target.d.ts.map +1 -0
  133. package/dist/types/profile-evaluations.d.ts +117 -14
  134. package/dist/types/profile-evaluations.d.ts.map +1 -1
  135. package/dist/types/profile-external-evaluation.d.ts +404 -0
  136. package/dist/types/profile-external-evaluation.d.ts.map +1 -0
  137. package/dist/types/profile-source-candidate.d.ts +15 -0
  138. package/dist/types/profile-source-candidate.d.ts.map +1 -0
  139. package/dist/types/review-conformance.d.ts +7 -7
  140. package/dist/types/rewards.d.ts +253 -0
  141. package/dist/types/rewards.d.ts.map +1 -1
  142. package/dist/types/rollouts.d.ts +3 -3
  143. package/dist/types/taskset-run-contract.d.ts +230 -0
  144. package/dist/types/taskset-run-contract.d.ts.map +1 -1
  145. package/dist/types/tasksets.d.ts +311 -3
  146. package/dist/types/tasksets.d.ts.map +1 -1
  147. package/dist/types/telemetry-analysis.d.ts +2 -2
  148. package/dist/types/telemetry-bundle.d.ts +6 -6
  149. package/dist/types/telemetry.d.ts +3 -3
  150. package/package.json +13 -1
  151. package/schemas/learning/v1/command-request.schema.json +1745 -409
  152. package/schemas/learning/v1/decision.schema.json +11 -0
  153. package/schemas/learning/v1/draft.schema.json +11 -0
  154. package/schemas/learning/v1/grade.schema.json +11 -0
  155. package/schemas/learning/v1/package.schema.json +532 -12
  156. package/schemas/learning/v1/policy.schema.json +836 -28
  157. package/schemas/learning/v1/reward.schema.json +520 -0
  158. package/schemas/learning/v1/reward_check.schema.json +9 -0
@@ -170,6 +170,30 @@
170
170
  "type": "string",
171
171
  "minLength": 1,
172
172
  "maxLength": 500
173
+ },
174
+ "form": {
175
+ "type": "object",
176
+ "properties": {
177
+ "schemaVersion": {
178
+ "$ref": "#/$defs/__schema3"
179
+ },
180
+ "mode": {
181
+ "$ref": "#/$defs/__schema4"
182
+ },
183
+ "instructions": {
184
+ "$ref": "#/$defs/__schema5"
185
+ },
186
+ "criteria": {
187
+ "$ref": "#/$defs/__schema6"
188
+ }
189
+ },
190
+ "required": [
191
+ "schemaVersion",
192
+ "mode",
193
+ "instructions",
194
+ "criteria"
195
+ ],
196
+ "additionalProperties": false
173
197
  }
174
198
  },
175
199
  "required": [
@@ -331,6 +355,502 @@
331
355
  "__schema2": {
332
356
  "type": "string",
333
357
  "pattern": "^[a-f0-9]{64}$"
358
+ },
359
+ "__schema3": {
360
+ "type": "string",
361
+ "const": "openpond.humanForm.v1"
362
+ },
363
+ "__schema4": {
364
+ "type": "string",
365
+ "enum": [
366
+ "individual",
367
+ "pairwise"
368
+ ]
369
+ },
370
+ "__schema5": {
371
+ "type": "string",
372
+ "maxLength": 20000
373
+ },
374
+ "__schema6": {
375
+ "minItems": 1,
376
+ "maxItems": 100,
377
+ "type": "array",
378
+ "items": {
379
+ "$ref": "#/$defs/__schema7"
380
+ }
381
+ },
382
+ "__schema7": {
383
+ "oneOf": [
384
+ {
385
+ "type": "object",
386
+ "properties": {
387
+ "id": {
388
+ "$ref": "#/$defs/__schema0"
389
+ },
390
+ "label": {
391
+ "$ref": "#/$defs/__schema8"
392
+ },
393
+ "instructions": {
394
+ "$ref": "#/$defs/__schema9"
395
+ },
396
+ "required": {
397
+ "$ref": "#/$defs/__schema10"
398
+ },
399
+ "allowAbstain": {
400
+ "$ref": "#/$defs/__schema11"
401
+ },
402
+ "requireRationale": {
403
+ "$ref": "#/$defs/__schema12"
404
+ },
405
+ "anchors": {
406
+ "$ref": "#/$defs/__schema13"
407
+ },
408
+ "kind": {
409
+ "$ref": "#/$defs/__schema15"
410
+ },
411
+ "minimum": {
412
+ "$ref": "#/$defs/__schema16"
413
+ },
414
+ "maximum": {
415
+ "$ref": "#/$defs/__schema17"
416
+ },
417
+ "step": {
418
+ "$ref": "#/$defs/__schema18"
419
+ }
420
+ },
421
+ "required": [
422
+ "id",
423
+ "label",
424
+ "instructions",
425
+ "required",
426
+ "allowAbstain",
427
+ "kind",
428
+ "minimum",
429
+ "maximum",
430
+ "step"
431
+ ],
432
+ "additionalProperties": false
433
+ },
434
+ {
435
+ "type": "object",
436
+ "properties": {
437
+ "id": {
438
+ "$ref": "#/$defs/__schema0"
439
+ },
440
+ "label": {
441
+ "$ref": "#/$defs/__schema19"
442
+ },
443
+ "instructions": {
444
+ "$ref": "#/$defs/__schema20"
445
+ },
446
+ "required": {
447
+ "$ref": "#/$defs/__schema21"
448
+ },
449
+ "allowAbstain": {
450
+ "$ref": "#/$defs/__schema22"
451
+ },
452
+ "requireRationale": {
453
+ "$ref": "#/$defs/__schema23"
454
+ },
455
+ "anchors": {
456
+ "$ref": "#/$defs/__schema24"
457
+ },
458
+ "kind": {
459
+ "$ref": "#/$defs/__schema26"
460
+ },
461
+ "options": {
462
+ "$ref": "#/$defs/__schema27"
463
+ }
464
+ },
465
+ "required": [
466
+ "id",
467
+ "label",
468
+ "instructions",
469
+ "required",
470
+ "allowAbstain",
471
+ "kind",
472
+ "options"
473
+ ],
474
+ "additionalProperties": false
475
+ },
476
+ {
477
+ "type": "object",
478
+ "properties": {
479
+ "id": {
480
+ "$ref": "#/$defs/__schema0"
481
+ },
482
+ "label": {
483
+ "type": "string",
484
+ "minLength": 1,
485
+ "maxLength": 500
486
+ },
487
+ "instructions": {
488
+ "type": "string",
489
+ "maxLength": 20000
490
+ },
491
+ "required": {
492
+ "type": "boolean"
493
+ },
494
+ "allowAbstain": {
495
+ "type": "boolean"
496
+ },
497
+ "requireRationale": {
498
+ "type": "boolean"
499
+ },
500
+ "anchors": {
501
+ "maxItems": 100,
502
+ "type": "array",
503
+ "items": {
504
+ "$ref": "#/$defs/__schema29"
505
+ }
506
+ },
507
+ "kind": {
508
+ "type": "string",
509
+ "const": "boolean"
510
+ }
511
+ },
512
+ "required": [
513
+ "id",
514
+ "label",
515
+ "instructions",
516
+ "required",
517
+ "allowAbstain",
518
+ "kind"
519
+ ],
520
+ "additionalProperties": false
521
+ },
522
+ {
523
+ "type": "object",
524
+ "properties": {
525
+ "id": {
526
+ "$ref": "#/$defs/__schema0"
527
+ },
528
+ "label": {
529
+ "type": "string",
530
+ "minLength": 1,
531
+ "maxLength": 500
532
+ },
533
+ "instructions": {
534
+ "type": "string",
535
+ "maxLength": 20000
536
+ },
537
+ "required": {
538
+ "type": "boolean"
539
+ },
540
+ "allowAbstain": {
541
+ "type": "boolean"
542
+ },
543
+ "requireRationale": {
544
+ "type": "boolean"
545
+ },
546
+ "anchors": {
547
+ "maxItems": 100,
548
+ "type": "array",
549
+ "items": {
550
+ "$ref": "#/$defs/__schema30"
551
+ }
552
+ },
553
+ "kind": {
554
+ "type": "string",
555
+ "const": "text"
556
+ },
557
+ "minimumLength": {
558
+ "type": "integer",
559
+ "minimum": 0,
560
+ "maximum": 20000
561
+ },
562
+ "maximumLength": {
563
+ "type": "integer",
564
+ "exclusiveMinimum": 0,
565
+ "maximum": 20000
566
+ }
567
+ },
568
+ "required": [
569
+ "id",
570
+ "label",
571
+ "instructions",
572
+ "required",
573
+ "allowAbstain",
574
+ "kind",
575
+ "maximumLength"
576
+ ],
577
+ "additionalProperties": false
578
+ },
579
+ {
580
+ "type": "object",
581
+ "properties": {
582
+ "id": {
583
+ "$ref": "#/$defs/__schema0"
584
+ },
585
+ "label": {
586
+ "type": "string",
587
+ "minLength": 1,
588
+ "maxLength": 500
589
+ },
590
+ "instructions": {
591
+ "type": "string",
592
+ "maxLength": 20000
593
+ },
594
+ "required": {
595
+ "type": "boolean"
596
+ },
597
+ "allowAbstain": {
598
+ "type": "boolean"
599
+ },
600
+ "requireRationale": {
601
+ "type": "boolean"
602
+ },
603
+ "anchors": {
604
+ "maxItems": 100,
605
+ "type": "array",
606
+ "items": {
607
+ "$ref": "#/$defs/__schema31"
608
+ }
609
+ },
610
+ "kind": {
611
+ "type": "string",
612
+ "const": "preference"
613
+ },
614
+ "requireStrength": {
615
+ "type": "boolean"
616
+ },
617
+ "allowTie": {
618
+ "type": "boolean"
619
+ }
620
+ },
621
+ "required": [
622
+ "id",
623
+ "label",
624
+ "instructions",
625
+ "required",
626
+ "allowAbstain",
627
+ "kind",
628
+ "allowTie"
629
+ ],
630
+ "additionalProperties": false
631
+ }
632
+ ]
633
+ },
634
+ "__schema8": {
635
+ "type": "string",
636
+ "minLength": 1,
637
+ "maxLength": 500
638
+ },
639
+ "__schema9": {
640
+ "type": "string",
641
+ "maxLength": 20000
642
+ },
643
+ "__schema10": {
644
+ "type": "boolean"
645
+ },
646
+ "__schema11": {
647
+ "type": "boolean"
648
+ },
649
+ "__schema12": {
650
+ "type": "boolean"
651
+ },
652
+ "__schema13": {
653
+ "maxItems": 100,
654
+ "type": "array",
655
+ "items": {
656
+ "$ref": "#/$defs/__schema14"
657
+ }
658
+ },
659
+ "__schema14": {
660
+ "type": "object",
661
+ "properties": {
662
+ "value": {
663
+ "anyOf": [
664
+ {
665
+ "type": "number"
666
+ },
667
+ {
668
+ "type": "string"
669
+ },
670
+ {
671
+ "type": "boolean"
672
+ }
673
+ ]
674
+ },
675
+ "description": {
676
+ "type": "string",
677
+ "minLength": 1,
678
+ "maxLength": 2000
679
+ }
680
+ },
681
+ "required": [
682
+ "value",
683
+ "description"
684
+ ],
685
+ "additionalProperties": false
686
+ },
687
+ "__schema15": {
688
+ "type": "string",
689
+ "const": "score"
690
+ },
691
+ "__schema16": {
692
+ "type": "number"
693
+ },
694
+ "__schema17": {
695
+ "type": "number"
696
+ },
697
+ "__schema18": {
698
+ "type": "number",
699
+ "exclusiveMinimum": 0
700
+ },
701
+ "__schema19": {
702
+ "type": "string",
703
+ "minLength": 1,
704
+ "maxLength": 500
705
+ },
706
+ "__schema20": {
707
+ "type": "string",
708
+ "maxLength": 20000
709
+ },
710
+ "__schema21": {
711
+ "type": "boolean"
712
+ },
713
+ "__schema22": {
714
+ "type": "boolean"
715
+ },
716
+ "__schema23": {
717
+ "type": "boolean"
718
+ },
719
+ "__schema24": {
720
+ "maxItems": 100,
721
+ "type": "array",
722
+ "items": {
723
+ "$ref": "#/$defs/__schema25"
724
+ }
725
+ },
726
+ "__schema25": {
727
+ "type": "object",
728
+ "properties": {
729
+ "value": {
730
+ "anyOf": [
731
+ {
732
+ "type": "number"
733
+ },
734
+ {
735
+ "type": "string"
736
+ },
737
+ {
738
+ "type": "boolean"
739
+ }
740
+ ]
741
+ },
742
+ "description": {
743
+ "type": "string",
744
+ "minLength": 1,
745
+ "maxLength": 2000
746
+ }
747
+ },
748
+ "required": [
749
+ "value",
750
+ "description"
751
+ ],
752
+ "additionalProperties": false
753
+ },
754
+ "__schema26": {
755
+ "type": "string",
756
+ "const": "category"
757
+ },
758
+ "__schema27": {
759
+ "minItems": 2,
760
+ "maxItems": 100,
761
+ "type": "array",
762
+ "items": {
763
+ "$ref": "#/$defs/__schema28"
764
+ }
765
+ },
766
+ "__schema28": {
767
+ "type": "string",
768
+ "minLength": 1,
769
+ "maxLength": 500
770
+ },
771
+ "__schema29": {
772
+ "type": "object",
773
+ "properties": {
774
+ "value": {
775
+ "anyOf": [
776
+ {
777
+ "type": "number"
778
+ },
779
+ {
780
+ "type": "string"
781
+ },
782
+ {
783
+ "type": "boolean"
784
+ }
785
+ ]
786
+ },
787
+ "description": {
788
+ "type": "string",
789
+ "minLength": 1,
790
+ "maxLength": 2000
791
+ }
792
+ },
793
+ "required": [
794
+ "value",
795
+ "description"
796
+ ],
797
+ "additionalProperties": false
798
+ },
799
+ "__schema30": {
800
+ "type": "object",
801
+ "properties": {
802
+ "value": {
803
+ "anyOf": [
804
+ {
805
+ "type": "number"
806
+ },
807
+ {
808
+ "type": "string"
809
+ },
810
+ {
811
+ "type": "boolean"
812
+ }
813
+ ]
814
+ },
815
+ "description": {
816
+ "type": "string",
817
+ "minLength": 1,
818
+ "maxLength": 2000
819
+ }
820
+ },
821
+ "required": [
822
+ "value",
823
+ "description"
824
+ ],
825
+ "additionalProperties": false
826
+ },
827
+ "__schema31": {
828
+ "type": "object",
829
+ "properties": {
830
+ "value": {
831
+ "anyOf": [
832
+ {
833
+ "type": "number"
834
+ },
835
+ {
836
+ "type": "string"
837
+ },
838
+ {
839
+ "type": "boolean"
840
+ }
841
+ ]
842
+ },
843
+ "description": {
844
+ "type": "string",
845
+ "minLength": 1,
846
+ "maxLength": 2000
847
+ }
848
+ },
849
+ "required": [
850
+ "value",
851
+ "description"
852
+ ],
853
+ "additionalProperties": false
334
854
  }
335
855
  },
336
856
  "$id": "urn:openpond:learning:v1:reward",
@@ -321,6 +321,15 @@
321
321
  "minLength": 1,
322
322
  "maxLength": 100
323
323
  },
324
+ "status": {
325
+ "type": "string",
326
+ "enum": [
327
+ "scored",
328
+ "pending",
329
+ "unavailable",
330
+ "failed"
331
+ ]
332
+ },
324
333
  "score": {
325
334
  "anyOf": [
326
335
  {