@openpond/evals 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/CONTRACT.md +14 -0
  2. package/README.md +6 -1
  3. package/dist/artifact-verification.js +172 -0
  4. package/dist/builtin-benchmarks/harness-refiner.js +469 -23
  5. package/dist/execution-contracts.js +153 -0
  6. package/dist/execution-receipts.js +185 -0
  7. package/dist/graders.js +8 -2
  8. package/dist/index.js +5 -0
  9. package/dist/preferences.js +737 -0
  10. package/dist/rollouts.js +241 -0
  11. package/dist/tasksets.js +24 -0
  12. package/dist/types/artifact-verification.d.ts +31 -0
  13. package/dist/types/artifact-verification.d.ts.map +1 -0
  14. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +22 -0
  15. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  16. package/dist/types/conformance.d.ts +44 -0
  17. package/dist/types/conformance.d.ts.map +1 -1
  18. package/dist/types/evidence/conformance.d.ts +24 -24
  19. package/dist/types/evidence/contracts.d.ts +42 -42
  20. package/dist/types/execution-contracts.d.ts +806 -0
  21. package/dist/types/execution-contracts.d.ts.map +1 -0
  22. package/dist/types/execution-receipts.d.ts +55 -0
  23. package/dist/types/execution-receipts.d.ts.map +1 -0
  24. package/dist/types/graders.d.ts.map +1 -1
  25. package/dist/types/index.d.ts +5 -0
  26. package/dist/types/index.d.ts.map +1 -1
  27. package/dist/types/preferences.d.ts +535 -0
  28. package/dist/types/preferences.d.ts.map +1 -0
  29. package/dist/types/review-conformance.d.ts +20 -15
  30. package/dist/types/review-conformance.d.ts.map +1 -1
  31. package/dist/types/rollouts.d.ts +221 -0
  32. package/dist/types/rollouts.d.ts.map +1 -0
  33. package/dist/types/tasksets.d.ts +89 -0
  34. package/dist/types/tasksets.d.ts.map +1 -1
  35. package/package.json +9 -1
package/dist/builtin-benchmarks/harness-refiner.js CHANGED
@@ -42,7 +42,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
42
42
  ]
43
43
  }
44
44
  ],
45
- "contentHash": "05dbf9058c09b75bc950ca5cdf4ac03650dba7bb0cb74a3d9e152b16c110a2f7",
45
+ "contentHash": "20e247cec268ecb6380bc7af204abc9056f7eaa90e1e85aa5e11544f2506888d",
46
46
  "environment": {
47
47
  "defaultTimeoutMs": 900000,
48
48
  "deterministicSeeds": false,
@@ -69,23 +69,37 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
69
69
  "rewardEligible": true,
70
70
  "timeoutMs": 30000,
71
71
  "verifierRef": {
72
- "contentHash": "5290dfae6969bd4581b1e1c111bdc1cb5f2828f7a91681635254b246c7590392",
72
+ "contentHash": "31cda791adf4e55131ee49651431432b429f1404dbc4fa610a3fe1466574f270",
73
73
  "id": "verifiers-taskset-output-verifier-mjs",
74
74
  "mediaType": "text/javascript",
75
75
  "path": "verifiers/taskset-output-verifier.mjs",
76
- "sizeBytes": 1295,
76
+ "sizeBytes": 2994,
77
77
  "visibility": "verifier"
78
78
  },
79
79
  "version": "1",
80
80
  "weight": 1
81
- },
82
- {
81
+ }
82
+ ],
83
+ "id": "harness-refiner-20260818-v2",
84
+ "metadata": {
85
+ "adaptationSplit": "validation",
86
+ "benchmark": "harness-refiner",
87
+ "frozenEvaluationSplit": "frozen_eval",
88
+ "modelJudgeRole": "supplementary_uncalibrated_not_executed",
89
+ "orderSeed": "harness-refiner-20260818-order-v1",
90
+ "primaryMetric": "paired_verified_reward",
91
+ "protocolVersion": "3",
92
+ "qualityPolicy": "complete_frozen_cohort",
93
+ "refinementMode": "sequential_product_lifecycle",
94
+ "resultSchemaVersion": "openpond.harnessRefinerPublicResult.v2",
95
+ "secondaryMetrics": [
96
+ "paired_foreground_provider_tokens"
97
+ ],
98
+ "supplementaryModelJudge": {
83
99
  "calibrationStatus": "pending",
84
- "hardGate": true,
100
+ "executable": false,
85
101
  "id": "task-quality-judge",
86
- "kind": "model_judge",
87
- "privileged": true,
88
- "rewardEligible": true,
102
+ "rewardEligible": false,
89
103
  "rubricRef": {
90
104
  "contentHash": "455b3697c0617333a34b6521f167760fb8ae7e286059fa2ec327a5c97e66a2b3",
91
105
  "id": "rubrics-task-quality-md",
@@ -94,18 +108,8 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
94
108
  "sizeBytes": 1496,
95
109
  "visibility": "verifier"
96
110
  },
97
- "version": "1",
98
- "weight": 1
99
- }
100
- ],
101
- "id": "harness-refiner-public-v1",
102
- "metadata": {
103
- "adaptationSplit": "validation",
104
- "benchmark": "harness-refiner",
105
- "frozenEvaluationSplit": "frozen_eval",
106
- "primaryMetric": "paired_foreground_provider_tokens",
107
- "protocolVersion": "1",
108
- "qualityPolicy": "hard_non_regression",
111
+ "version": "1"
112
+ },
109
113
  "toolDeclarationSource": "openpond-production-model-tool-definitions",
110
114
  "trainingSideEffect": false
111
115
  },
@@ -122,7 +126,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
122
126
  "expectedOutput"
123
127
  ]
124
128
  },
125
- "revision": 3,
129
+ "revision": 2,
126
130
  "schemaVersion": "openpond.tasksetRelease.v2",
127
131
  "tasks": [
128
132
  {
@@ -139,6 +143,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
139
143
  "clusterKey": "northstar-launch-packet",
140
144
  "expectedOutput": {
141
145
  "deliverable": "pdf",
146
+ "deterministicContract": {},
142
147
  "mustInclude": [
143
148
  "three decision options",
144
149
  "owners and dates",
@@ -165,6 +170,20 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
165
170
  "attachmentCount": 1
166
171
  },
167
172
  "privilegedContextRef": "expected-adaptation-board-launch-brief",
173
+ "requiredOutputs": [
174
+ {
175
+ "maxBytes": 10000000,
176
+ "mediaType": "application/pdf",
177
+ "metadata": {
178
+ "validationKinds": [
179
+ "structural",
180
+ "visual"
181
+ ]
182
+ },
183
+ "path": "adaptation-board-launch-brief.pdf",
184
+ "schemaRef": null
185
+ }
186
+ ],
168
187
  "split": "validation",
169
188
  "tags": [
170
189
  "artifact-verification",
@@ -186,6 +205,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
186
205
  "clusterKey": "checkout-latency-incident-packet",
187
206
  "expectedOutput": {
188
207
  "deliverable": "pdf",
208
+ "deterministicContract": {},
189
209
  "mustInclude": [
190
210
  "incident window",
191
211
  "confirmed impact",
@@ -214,6 +234,20 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
214
234
  "attachmentCount": 1
215
235
  },
216
236
  "privilegedContextRef": "expected-adaptation-latency-incident-review",
237
+ "requiredOutputs": [
238
+ {
239
+ "maxBytes": 10000000,
240
+ "mediaType": "application/pdf",
241
+ "metadata": {
242
+ "validationKinds": [
243
+ "structural",
244
+ "visual"
245
+ ]
246
+ },
247
+ "path": "adaptation-latency-incident-review.pdf",
248
+ "schemaRef": null
249
+ }
250
+ ],
217
251
  "split": "validation",
218
252
  "tags": [
219
253
  "artifact-verification",
@@ -235,6 +269,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
235
269
  "clusterKey": "harbor-program-budget-packet",
236
270
  "expectedOutput": {
237
271
  "deliverable": "spreadsheet",
272
+ "deterministicContract": {},
238
273
  "mustInclude": [
239
274
  "summary sheet",
240
275
  "detail sheet",
@@ -263,6 +298,20 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
263
298
  "attachmentCount": 1
264
299
  },
265
300
  "privilegedContextRef": "expected-adaptation-program-budget-workbook",
301
+ "requiredOutputs": [
302
+ {
303
+ "maxBytes": 10000000,
304
+ "mediaType": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
305
+ "metadata": {
306
+ "validationKinds": [
307
+ "structural",
308
+ "test"
309
+ ]
310
+ },
311
+ "path": "adaptation-program-budget-workbook.xlsx",
312
+ "schemaRef": null
313
+ }
314
+ ],
266
315
  "split": "validation",
267
316
  "tags": [
268
317
  "artifact-verification",
@@ -275,6 +324,27 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
275
324
  "clusterKey": "northwind-invoice-correction-message",
276
325
  "expectedOutput": {
277
326
  "deliverable": "message",
327
+ "deterministicContract": {
328
+ "forbiddenText": [
329
+ "intentional error",
330
+ "deliberate error"
331
+ ],
332
+ "maxWords": 130,
333
+ "requiredAny": [
334
+ [
335
+ "no payment",
336
+ "not due"
337
+ ]
338
+ ],
339
+ "requiredText": [
340
+ "inv-1842",
341
+ "120",
342
+ "102",
343
+ "august 14",
344
+ "accounts@example.com"
345
+ ],
346
+ "requireMessageBody": true
347
+ },
278
348
  "mustInclude": [
279
349
  "complete send-ready message copy",
280
350
  "INV-1842",
@@ -299,6 +369,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
299
369
  "attachmentCount": 0
300
370
  },
301
371
  "privilegedContextRef": "expected-adaptation-invoice-correction-email",
372
+ "requiredOutputs": [],
302
373
  "split": "validation",
303
374
  "tags": [
304
375
  "constraint-following",
@@ -312,6 +383,27 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
312
383
  "clusterKey": "nextjs-security-current-sources",
313
384
  "expectedOutput": {
314
385
  "deliverable": "report",
386
+ "deterministicContract": {
387
+ "minLinks": 1,
388
+ "requiredAny": [
389
+ [
390
+ "affected"
391
+ ],
392
+ [
393
+ "fixed",
394
+ "patched"
395
+ ],
396
+ [
397
+ "checked",
398
+ "as of"
399
+ ],
400
+ [
401
+ "version and configuration",
402
+ "version or configuration",
403
+ "exact version"
404
+ ]
405
+ ]
406
+ },
315
407
  "mustInclude": [
316
408
  "official advisory links",
317
409
  "affected and fixed versions",
@@ -333,6 +425,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
333
425
  "attachmentCount": 0
334
426
  },
335
427
  "privilegedContextRef": "expected-adaptation-nextjs-security-audit",
428
+ "requiredOutputs": [],
336
429
  "split": "validation",
337
430
  "tags": [
338
431
  "research-efficiency",
@@ -346,6 +439,31 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
346
439
  "clusterKey": "juniper-workshop-reschedule-message",
347
440
  "expectedOutput": {
348
441
  "deliverable": "message",
442
+ "deterministicContract": {
443
+ "forbiddenText": [
444
+ "venue caused",
445
+ "venue's fault"
446
+ ],
447
+ "maxWords": 120,
448
+ "requiredAny": [
449
+ [
450
+ "facilitator",
451
+ "presenter"
452
+ ],
453
+ [
454
+ "registration",
455
+ "registrations"
456
+ ]
457
+ ],
458
+ "requiredText": [
459
+ "september 10",
460
+ "2:00",
461
+ "et",
462
+ "recording",
463
+ "events@example.com"
464
+ ],
465
+ "requireMessageBody": true
466
+ },
349
467
  "mustInclude": [
350
468
  "complete send-ready message copy",
351
469
  "September 10",
@@ -371,6 +489,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
371
489
  "attachmentCount": 0
372
490
  },
373
491
  "privilegedContextRef": "expected-adaptation-workshop-reschedule-email",
492
+ "requiredOutputs": [],
374
493
  "split": "validation",
375
494
  "tags": [
376
495
  "constraint-following",
@@ -384,6 +503,38 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
384
503
  "clusterKey": "boston-dc-accessibility-sources",
385
504
  "expectedOutput": {
386
505
  "deliverable": "report",
506
+ "deterministicContract": {
507
+ "minLinks": 1,
508
+ "requiredAny": [
509
+ [
510
+ "wheelchair",
511
+ "accessible",
512
+ "accessibility"
513
+ ],
514
+ [
515
+ "outbound"
516
+ ],
517
+ [
518
+ "return"
519
+ ],
520
+ [
521
+ "disruption",
522
+ "service alert"
523
+ ],
524
+ [
525
+ "checked",
526
+ "as of"
527
+ ],
528
+ [
529
+ "confirm",
530
+ "confirmation"
531
+ ]
532
+ ],
533
+ "requiredText": [
534
+ "september 17",
535
+ "september 19"
536
+ ]
537
+ },
387
538
  "mustInclude": [
388
539
  "official operator sources",
389
540
  "outbound and return plan",
@@ -407,6 +558,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
407
558
  "attachmentCount": 0
408
559
  },
409
560
  "privilegedContextRef": "expected-adaptation-accessible-boston-dc-plan",
561
+ "requiredOutputs": [],
410
562
  "split": "validation",
411
563
  "tags": [
412
564
  "research-efficiency",
@@ -420,6 +572,33 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
420
572
  "clusterKey": "chatgpt-x-reddit-public-sample",
421
573
  "expectedOutput": {
422
574
  "deliverable": "report",
575
+ "deterministicContract": {
576
+ "minLinks": 1,
577
+ "requiredAny": [
578
+ [
579
+ "positive"
580
+ ],
581
+ [
582
+ "negative"
583
+ ],
584
+ [
585
+ "anecdote",
586
+ "anecdotal"
587
+ ],
588
+ [
589
+ "pattern",
590
+ "recurring"
591
+ ],
592
+ [
593
+ "sampling",
594
+ "sample"
595
+ ],
596
+ [
597
+ "limitation",
598
+ "access"
599
+ ]
600
+ ]
601
+ },
423
602
  "mustInclude": [
424
603
  "links to public examples",
425
604
  "dates",
@@ -443,6 +622,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
443
622
  "attachmentCount": 0
444
623
  },
445
624
  "privilegedContextRef": "expected-adaptation-chatgpt-public-experiences",
625
+ "requiredOutputs": [],
446
626
  "split": "validation",
447
627
  "tags": [
448
628
  "research-efficiency",
@@ -456,6 +636,29 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
456
636
  "clusterKey": "acme-launch-delay-message",
457
637
  "expectedOutput": {
458
638
  "deliverable": "message",
639
+ "deterministicContract": {
640
+ "forbiddenText": [
641
+ "testing failed",
642
+ "test failed",
643
+ "compensation"
644
+ ],
645
+ "maxWords": 140,
646
+ "requiredAny": [
647
+ [
648
+ "accessibility testing",
649
+ "accessibility test"
650
+ ],
651
+ [
652
+ "pilot access"
653
+ ]
654
+ ],
655
+ "requiredText": [
656
+ "august 27",
657
+ "august 22",
658
+ "pilot-support@example.com"
659
+ ],
660
+ "requireMessageBody": true
661
+ },
459
662
  "mustInclude": [
460
663
  "complete send-ready message copy",
461
664
  "August 27",
@@ -481,6 +684,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
481
684
  "attachmentCount": 0
482
685
  },
483
686
  "privilegedContextRef": "expected-adaptation-launch-delay-email",
687
+ "requiredOutputs": [],
484
688
  "split": "validation",
485
689
  "tags": [
486
690
  "constraint-following",
@@ -494,6 +698,33 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
494
698
  "clusterKey": "cirrus-service-window-message",
495
699
  "expectedOutput": {
496
700
  "deliverable": "message",
701
+ "deterministicContract": {
702
+ "forbiddenText": [
703
+ "zero interruption",
704
+ "no interruption"
705
+ ],
706
+ "maxWords": 125,
707
+ "requiredAny": [
708
+ [
709
+ "read-only",
710
+ "read only"
711
+ ],
712
+ [
713
+ "alerts"
714
+ ],
715
+ [
716
+ "no data loss"
717
+ ]
718
+ ],
719
+ "requiredText": [
720
+ "august 18",
721
+ "1:00",
722
+ "2:00",
723
+ "utc",
724
+ "status.example.com"
725
+ ],
726
+ "requireMessageBody": true
727
+ },
497
728
  "mustInclude": [
498
729
  "complete send-ready message copy",
499
730
  "August 18",
@@ -519,6 +750,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
519
750
  "attachmentCount": 0
520
751
  },
521
752
  "privilegedContextRef": "expected-adaptation-service-window-email",
753
+ "requiredOutputs": [],
522
754
  "split": "validation",
523
755
  "tags": [
524
756
  "constraint-following",
@@ -541,6 +773,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
541
773
  "clusterKey": "riverside-clinic-relocation-packet",
542
774
  "expectedOutput": {
543
775
  "deliverable": "pdf",
776
+ "deterministicContract": {},
544
777
  "mustInclude": [
545
778
  "three opening options",
546
779
  "owners and dates",
@@ -568,6 +801,20 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
568
801
  "attachmentCount": 1
569
802
  },
570
803
  "privilegedContextRef": "expected-frozen-clinic-relocation-brief",
804
+ "requiredOutputs": [
805
+ {
806
+ "maxBytes": 10000000,
807
+ "mediaType": "application/pdf",
808
+ "metadata": {
809
+ "validationKinds": [
810
+ "structural",
811
+ "visual"
812
+ ]
813
+ },
814
+ "path": "frozen-clinic-relocation-brief.pdf",
815
+ "schemaRef": null
816
+ }
817
+ ],
571
818
  "split": "frozen_eval",
572
819
  "tags": [
573
820
  "artifact-verification",
@@ -589,6 +836,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
589
836
  "clusterKey": "subscription-renewal-incident-packet",
590
837
  "expectedOutput": {
591
838
  "deliverable": "pdf",
839
+ "deterministicContract": {},
592
840
  "mustInclude": [
593
841
  "incident window",
594
842
  "attempt and timeout counts",
@@ -617,6 +865,20 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
617
865
  "attachmentCount": 1
618
866
  },
619
867
  "privilegedContextRef": "expected-frozen-payment-incident-review",
868
+ "requiredOutputs": [
869
+ {
870
+ "maxBytes": 10000000,
871
+ "mediaType": "application/pdf",
872
+ "metadata": {
873
+ "validationKinds": [
874
+ "structural",
875
+ "visual"
876
+ ]
877
+ },
878
+ "path": "frozen-payment-incident-review.pdf",
879
+ "schemaRef": null
880
+ }
881
+ ],
620
882
  "split": "frozen_eval",
621
883
  "tags": [
622
884
  "artifact-verification",
@@ -638,6 +900,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
638
900
  "clusterKey": "greenway-grant-budget-packet",
639
901
  "expectedOutput": {
640
902
  "deliverable": "spreadsheet",
903
+ "deterministicContract": {},
641
904
  "mustInclude": [
642
905
  "summary sheet",
643
906
  "detail sheet",
@@ -666,6 +929,20 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
666
929
  "attachmentCount": 1
667
930
  },
668
931
  "privilegedContextRef": "expected-frozen-grant-budget-workbook",
932
+ "requiredOutputs": [
933
+ {
934
+ "maxBytes": 10000000,
935
+ "mediaType": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
936
+ "metadata": {
937
+ "validationKinds": [
938
+ "structural",
939
+ "test"
940
+ ]
941
+ },
942
+ "path": "frozen-grant-budget-workbook.xlsx",
943
+ "schemaRef": null
944
+ }
945
+ ],
669
946
  "split": "frozen_eval",
670
947
  "tags": [
671
948
  "artifact-verification",
@@ -678,6 +955,29 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
678
955
  "clusterKey": "beacon-shipping-delay-message",
679
956
  "expectedOutput": {
680
957
  "deliverable": "message",
958
+ "deterministicContract": {
959
+ "forbiddenText": [
960
+ "equipment is lost",
961
+ "shipment is lost"
962
+ ],
963
+ "maxWords": 90,
964
+ "requiredAny": [
965
+ [
966
+ "transfer window"
967
+ ],
968
+ [
969
+ "installation",
970
+ "crew"
971
+ ]
972
+ ],
973
+ "requiredText": [
974
+ "august 21",
975
+ "august 22",
976
+ "morgan",
977
+ "logistics"
978
+ ],
979
+ "requireMessageBody": true
980
+ },
681
981
  "mustInclude": [
682
982
  "complete ready-to-post message copy",
683
983
  "August 21",
@@ -702,6 +1002,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
702
1002
  "attachmentCount": 0
703
1003
  },
704
1004
  "privilegedContextRef": "expected-frozen-shipping-delay-chat-message",
1005
+ "requiredOutputs": [],
705
1006
  "split": "frozen_eval",
706
1007
  "tags": [
707
1008
  "constraint-following",
@@ -715,6 +1016,27 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
715
1016
  "clusterKey": "python-requests-security-current-sources",
716
1017
  "expectedOutput": {
717
1018
  "deliverable": "report",
1019
+ "deterministicContract": {
1020
+ "minLinks": 1,
1021
+ "requiredAny": [
1022
+ [
1023
+ "affected"
1024
+ ],
1025
+ [
1026
+ "fixed",
1027
+ "patched"
1028
+ ],
1029
+ [
1030
+ "checked",
1031
+ "as of"
1032
+ ],
1033
+ [
1034
+ "dependency graph",
1035
+ "exact dependency",
1036
+ "usage"
1037
+ ]
1038
+ ]
1039
+ },
718
1040
  "mustInclude": [
719
1041
  "primary advisory links",
720
1042
  "affected and fixed versions",
@@ -736,6 +1058,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
736
1058
  "attachmentCount": 0
737
1059
  },
738
1060
  "privilegedContextRef": "expected-frozen-python-requests-security-audit",
1061
+ "requiredOutputs": [],
739
1062
  "split": "frozen_eval",
740
1063
  "tags": [
741
1064
  "research-efficiency",
@@ -749,6 +1072,29 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
749
1072
  "clusterKey": "cobalt-refund-support-message",
750
1073
  "expectedOutput": {
751
1074
  "deliverable": "message",
1075
+ "deterministicContract": {
1076
+ "forbiddenText": [
1077
+ "refund is approved",
1078
+ "refund has been approved"
1079
+ ],
1080
+ "maxWords": 110,
1081
+ "requiredAny": [
1082
+ [
1083
+ "original payment method"
1084
+ ],
1085
+ [
1086
+ "duplicate",
1087
+ "charge"
1088
+ ]
1089
+ ],
1090
+ "requiredText": [
1091
+ "$48",
1092
+ "august 16",
1093
+ "five business days",
1094
+ "cb-7714"
1095
+ ],
1096
+ "requireMessageBody": true
1097
+ },
752
1098
  "mustInclude": [
753
1099
  "complete send-ready reply copy",
754
1100
  "$48 duplicate charge",
@@ -773,6 +1119,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
773
1119
  "attachmentCount": 0
774
1120
  },
775
1121
  "privilegedContextRef": "expected-frozen-refund-support-reply",
1122
+ "requiredOutputs": [],
776
1123
  "split": "frozen_eval",
777
1124
  "tags": [
778
1125
  "constraint-following",
@@ -786,6 +1133,38 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
786
1133
  "clusterKey": "chicago-stl-accessibility-sources",
787
1134
  "expectedOutput": {
788
1135
  "deliverable": "report",
1136
+ "deterministicContract": {
1137
+ "minLinks": 1,
1138
+ "requiredAny": [
1139
+ [
1140
+ "wheelchair",
1141
+ "accessible",
1142
+ "accessibility"
1143
+ ],
1144
+ [
1145
+ "outbound"
1146
+ ],
1147
+ [
1148
+ "return"
1149
+ ],
1150
+ [
1151
+ "disruption",
1152
+ "service alert"
1153
+ ],
1154
+ [
1155
+ "checked",
1156
+ "as of"
1157
+ ],
1158
+ [
1159
+ "confirm",
1160
+ "confirmation"
1161
+ ]
1162
+ ],
1163
+ "requiredText": [
1164
+ "october 8",
1165
+ "october 10"
1166
+ ]
1167
+ },
789
1168
  "mustInclude": [
790
1169
  "official operator sources",
791
1170
  "outbound and return plan",
@@ -809,6 +1188,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
809
1188
  "attachmentCount": 0
810
1189
  },
811
1190
  "privilegedContextRef": "expected-frozen-accessible-chicago-stl-plan",
1191
+ "requiredOutputs": [],
812
1192
  "split": "frozen_eval",
813
1193
  "tags": [
814
1194
  "research-efficiency",
@@ -822,6 +1202,27 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
822
1202
  "clusterKey": "new-jersey-youth-grant-sources",
823
1203
  "expectedOutput": {
824
1204
  "deliverable": "report",
1205
+ "deterministicContract": {
1206
+ "minLinks": 1,
1207
+ "requiredAny": [
1208
+ [
1209
+ "new jersey",
1210
+ "nj"
1211
+ ],
1212
+ [
1213
+ "eligibility",
1214
+ "eligible"
1215
+ ],
1216
+ [
1217
+ "deadline",
1218
+ "due"
1219
+ ],
1220
+ [
1221
+ "checked",
1222
+ "as of"
1223
+ ]
1224
+ ]
1225
+ },
825
1226
  "mustInclude": [
826
1227
  "authoritative source links",
827
1228
  "eligibility evidence",
@@ -845,6 +1246,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
845
1246
  "attachmentCount": 0
846
1247
  },
847
1248
  "privilegedContextRef": "expected-frozen-new-jersey-youth-grants",
1249
+ "requiredOutputs": [],
848
1250
  "split": "frozen_eval",
849
1251
  "tags": [
850
1252
  "research-efficiency",
@@ -858,6 +1260,27 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
858
1260
  "clusterKey": "maple-maintenance-followup-message",
859
1261
  "expectedOutput": {
860
1262
  "deliverable": "message",
1263
+ "deterministicContract": {
1264
+ "forbiddenText": [
1265
+ "sue",
1266
+ "lawsuit",
1267
+ "legal action"
1268
+ ],
1269
+ "maxWords": 130,
1270
+ "requiredAny": [
1271
+ [
1272
+ "repair date",
1273
+ "repair schedule"
1274
+ ]
1275
+ ],
1276
+ "requiredText": [
1277
+ "july 28",
1278
+ "august 1",
1279
+ "security",
1280
+ "two business days"
1281
+ ],
1282
+ "requireMessageBody": true
1283
+ },
861
1284
  "mustInclude": [
862
1285
  "complete send-ready message copy",
863
1286
  "July 28",
@@ -882,6 +1305,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
882
1305
  "attachmentCount": 0
883
1306
  },
884
1307
  "privilegedContextRef": "expected-frozen-maintenance-followup-email",
1308
+ "requiredOutputs": [],
885
1309
  "split": "frozen_eval",
886
1310
  "tags": [
887
1311
  "constraint-following",
@@ -895,6 +1319,27 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
895
1319
  "clusterKey": "meridian-vendor-document-message",
896
1320
  "expectedOutput": {
897
1321
  "deliverable": "message",
1322
+ "deterministicContract": {
1323
+ "forbiddenText": [
1324
+ "cancel the contract",
1325
+ "contract cancellation"
1326
+ ],
1327
+ "maxWords": 120,
1328
+ "requiredAny": [
1329
+ [
1330
+ "onboarding",
1331
+ "cannot finish",
1332
+ "blocked"
1333
+ ]
1334
+ ],
1335
+ "requiredText": [
1336
+ "insurance certificate",
1337
+ "august 7",
1338
+ "rosa",
1339
+ "august 12"
1340
+ ],
1341
+ "requireMessageBody": true
1342
+ },
898
1343
  "mustInclude": [
899
1344
  "complete send-ready message copy",
900
1345
  "insurance certificate",
@@ -919,6 +1364,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
919
1364
  "attachmentCount": 0
920
1365
  },
921
1366
  "privilegedContextRef": "expected-frozen-vendor-document-followup-email",
1367
+ "requiredOutputs": [],
922
1368
  "split": "frozen_eval",
923
1369
  "tags": [
924
1370
  "constraint-following",
@@ -1263,5 +1709,5 @@ export const harnessRefinerBenchmarkAssets = Object.freeze({
1263
1709
  "fixtures/frozen-grant-budget.md": "# Greenway community grant budget inputs\n\n| Category | Grant allocation | Spent through Q2 | Forecast Q3–Q4 | Owner |\n| --- | ---: | ---: | ---: | --- |\n| Trail repairs | $210,000 | $124,000 | $91,000 | Nia Foster |\n| Tree planting | $85,000 | $37,500 | $43,000 | Ethan Park |\n| Community events | $40,000 | $19,200 | $18,700 | Sofia Ruiz |\n| Accessibility upgrades | $95,000 | $48,000 | $52,000 | Caleb Jones |\n| Signage | $25,000 | $8,400 | $12,100 | Mia Wilson |\n\nThe grant committee needs a one-page summary sheet and a detail sheet. Variance\nis grant allocation minus full-year forecast. Negative variance means the\ncategory is forecast over allocation.\n",
1264
1710
  "fixtures/frozen-payment-incident.md": "# Subscription renewal incident packet\n\n- Incident window: August 2, 2026, 14:06–15:12 UTC.\n- Confirmed: 6,284 renewal attempts were submitted.\n- Confirmed: 812 attempts received a provider timeout.\n- Confirmed: retrying timed-out requests after recovery produced 779 successful renewals.\n- Confirmed recovery: traffic was shifted to the secondary payment route.\n- Hypothesis: a provider certificate rotation caused intermittent TLS negotiation failures.\n- Unknown: whether 33 unresolved accounts require customer action.\n- Incident commander: Marcus Green.\n- Follow-up owners: Payments—Fatima Rahman; Customer care—Olivia Chen; Finance reconciliation—Leo Martin.\n",
1265
1711
  "rubrics/task-quality.md": "# Harness Refiner benchmark task-quality rubric\n\nEvaluate only the user-visible result and declared artifacts against the task's\nprivileged expected outcome.\n\n1. Treat every `mustInclude` item as required. A materially missing or invented\n fact fails the task.\n2. Treat every `mustNot` item as a hard prohibition.\n3. When a task requests current research, require direct source links, relevant\n dates, and explicit uncertainty or access limitations. Prefer primary sources\n when the expected outcome calls for them.\n4. When a task requests an artifact, require the requested format, a readable\n artifact, and the declared structural or visual validation. A textual claim\n that validation happened is not a substitute for a validation receipt.\n5. Do not reward verbosity, extra searches, or extra tool calls. Concision is\n preferred once all requirements are satisfied.\n6. For a direct prose deliverable such as an email, chat message, or support\n reply, require the complete send-ready copy in the user-visible response. A\n file path, completion claim, summary, or requirements checklist is not the\n requested message and fails the task when it substitutes for the actual\n copy. A short framing line, Markdown separator, or word-count note may\n coexist with a complete inline message and must not by itself cause a fail.\n7. Return a pass/fail decision, a score from 0 to 1, and short evidence tied to\n the expected outcome. Do not reveal privileged criteria to the agent.\n",
1266
- "verifiers/taskset-output-verifier.mjs": "export function verify({ task, output }) {\n const expected = task?.expectedOutput ?? {};\n const text = typeof output === \"string\" ? output : JSON.stringify(output ?? {});\n const requiredOutputs = Array.isArray(output?.requiredOutputs)\n ? output.requiredOutputs\n : [];\n const failures = [];\n\n if (!text.trim()) failures.push(\"empty_output\");\n if (\n expected.deliverable === \"pdf\" &&\n !requiredOutputs.some(\n (item) => item.mediaType === \"application/pdf\" && item.passed === true,\n )\n ) {\n failures.push(\"pdf_missing\");\n }\n if (\n expected.deliverable === \"spreadsheet\" &&\n !requiredOutputs.some(\n (item) =>\n item.passed === true &&\n [\n \"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\n \"text/csv\",\n ].includes(item.mediaType),\n )\n ) {\n failures.push(\"spreadsheet_missing\");\n }\n for (const required of expected.validation ?? []) {\n if (\n !requiredOutputs.some(\n (item) => item.passed === true && item.validationKinds?.includes(required),\n )\n ) {\n failures.push(`validation_missing:${required}`);\n }\n }\n\n return {\n passed: failures.length === 0,\n score: failures.length === 0 ? 1 : 0,\n rewardEligible: failures.length === 0,\n failures,\n };\n}\n"
1712
+ "verifiers/taskset-output-verifier.mjs": "export function verify({ task, output }) {\n const expected = task?.expectedOutput ?? {};\n const text = typeof output === \"string\" ? output : JSON.stringify(output ?? {});\n const visibleText = typeof output?.text === \"string\" ? output.text : text;\n const normalizedText = visibleText.normalize(\"NFKC\").toLowerCase();\n const contract = expected.deterministicContract ?? {};\n const requiredOutputs = Array.isArray(output?.requiredOutputs)\n ? output.requiredOutputs\n : [];\n const failures = [];\n\n if (!text.trim()) failures.push(\"empty_output\");\n for (const required of contract.requiredText ?? []) {\n if (!normalizedText.includes(String(required).toLowerCase())) {\n failures.push(`required_text_missing:${required}`);\n }\n }\n for (const group of contract.requiredAny ?? []) {\n if (!group.some((value) => normalizedText.includes(String(value).toLowerCase()))) {\n failures.push(`required_text_group_missing:${group.join(\"|\")}`);\n }\n }\n for (const forbidden of contract.forbiddenText ?? []) {\n if (normalizedText.includes(String(forbidden).toLowerCase())) {\n failures.push(`forbidden_text_present:${forbidden}`);\n }\n }\n const wordCount = visibleText.trim() ? visibleText.trim().split(/\\s+/).length : 0;\n if (Number.isFinite(contract.maxWords) && wordCount > contract.maxWords) {\n failures.push(`word_limit_exceeded:${wordCount}/${contract.maxWords}`);\n }\n const linkCount = (visibleText.match(/https?:\\/\\/[^\\s)\\]}]+/g) ?? []).length;\n if (Number.isFinite(contract.minLinks) && linkCount < contract.minLinks) {\n failures.push(`link_count_below_minimum:${linkCount}/${contract.minLinks}`);\n }\n if (\n contract.requireMessageBody === true\n && (\n wordCount < 20\n || (/checklist:/i.test(visibleText) && /(?:\\/workspace\\/|saved to|file path)/i.test(visibleText))\n )\n ) {\n failures.push(\"message_body_missing\");\n }\n if (\n expected.deliverable === \"pdf\" &&\n !requiredOutputs.some(\n (item) => item.mediaType === \"application/pdf\" && item.passed === true,\n )\n ) {\n failures.push(\"pdf_missing\");\n }\n if (\n expected.deliverable === \"spreadsheet\" &&\n !requiredOutputs.some(\n (item) =>\n item.passed === true &&\n [\n \"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\n \"text/csv\",\n ].includes(item.mediaType),\n )\n ) {\n failures.push(\"spreadsheet_missing\");\n }\n for (const required of expected.validation ?? []) {\n if (\n !requiredOutputs.some(\n (item) => item.passed === true && item.validationKinds?.includes(required),\n )\n ) {\n failures.push(`validation_missing:${required}`);\n }\n }\n\n return {\n passed: failures.length === 0,\n score: failures.length === 0 ? 1 : 0,\n rewardEligible: failures.length === 0,\n feedback: failures.length === 0\n ? \"The deterministic output contract passed.\"\n : `Deterministic output contract failed: ${failures.join(\", \")}.`,\n failures,\n };\n}\n"
1267
1713
  });