@openpond/evals 0.4.2 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/CONTRACT.md +14 -0
  2. package/README.md +6 -1
  3. package/dist/artifact-verification.js +172 -0
  4. package/dist/builtin-benchmarks/harness-refiner.js +469 -24
  5. package/dist/execution-contracts.js +153 -0
  6. package/dist/execution-receipts.js +185 -0
  7. package/dist/graders.js +8 -2
  8. package/dist/index.js +5 -0
  9. package/dist/preferences.js +737 -0
  10. package/dist/rollouts.js +241 -0
  11. package/dist/tasksets.js +24 -0
  12. package/dist/types/artifact-verification.d.ts +31 -0
  13. package/dist/types/artifact-verification.d.ts.map +1 -0
  14. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +22 -0
  15. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  16. package/dist/types/conformance.d.ts +44 -0
  17. package/dist/types/conformance.d.ts.map +1 -1
  18. package/dist/types/evidence/conformance.d.ts +24 -24
  19. package/dist/types/evidence/contracts.d.ts +42 -42
  20. package/dist/types/execution-contracts.d.ts +806 -0
  21. package/dist/types/execution-contracts.d.ts.map +1 -0
  22. package/dist/types/execution-receipts.d.ts +55 -0
  23. package/dist/types/execution-receipts.d.ts.map +1 -0
  24. package/dist/types/graders.d.ts.map +1 -1
  25. package/dist/types/index.d.ts +5 -0
  26. package/dist/types/index.d.ts.map +1 -1
  27. package/dist/types/preferences.d.ts +535 -0
  28. package/dist/types/preferences.d.ts.map +1 -0
  29. package/dist/types/review-conformance.d.ts +20 -15
  30. package/dist/types/review-conformance.d.ts.map +1 -1
  31. package/dist/types/rollouts.d.ts +221 -0
  32. package/dist/types/rollouts.d.ts.map +1 -0
  33. package/dist/types/tasksets.d.ts +89 -0
  34. package/dist/types/tasksets.d.ts.map +1 -1
  35. package/package.json +9 -1
package/dist/builtin-benchmarks/harness-refiner.js CHANGED
@@ -42,7 +42,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
42
42
  ]
43
43
  }
44
44
  ],
45
- "contentHash": "4cef91a9c92df39d16f741b4d901dbde6b62e72bd8a48647a4a81c0d517d9634",
45
+ "contentHash": "20e247cec268ecb6380bc7af204abc9056f7eaa90e1e85aa5e11544f2506888d",
46
46
  "environment": {
47
47
  "defaultTimeoutMs": 900000,
48
48
  "deterministicSeeds": false,
@@ -69,23 +69,37 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
69
69
  "rewardEligible": true,
70
70
  "timeoutMs": 30000,
71
71
  "verifierRef": {
72
- "contentHash": "5290dfae6969bd4581b1e1c111bdc1cb5f2828f7a91681635254b246c7590392",
72
+ "contentHash": "31cda791adf4e55131ee49651431432b429f1404dbc4fa610a3fe1466574f270",
73
73
  "id": "verifiers-taskset-output-verifier-mjs",
74
74
  "mediaType": "text/javascript",
75
75
  "path": "verifiers/taskset-output-verifier.mjs",
76
- "sizeBytes": 1295,
76
+ "sizeBytes": 2994,
77
77
  "visibility": "verifier"
78
78
  },
79
79
  "version": "1",
80
80
  "weight": 1
81
- },
82
- {
81
+ }
82
+ ],
83
+ "id": "harness-refiner-20260818-v2",
84
+ "metadata": {
85
+ "adaptationSplit": "validation",
86
+ "benchmark": "harness-refiner",
87
+ "frozenEvaluationSplit": "frozen_eval",
88
+ "modelJudgeRole": "supplementary_uncalibrated_not_executed",
89
+ "orderSeed": "harness-refiner-20260818-order-v1",
90
+ "primaryMetric": "paired_verified_reward",
91
+ "protocolVersion": "3",
92
+ "qualityPolicy": "complete_frozen_cohort",
93
+ "refinementMode": "sequential_product_lifecycle",
94
+ "resultSchemaVersion": "openpond.harnessRefinerPublicResult.v2",
95
+ "secondaryMetrics": [
96
+ "paired_foreground_provider_tokens"
97
+ ],
98
+ "supplementaryModelJudge": {
83
99
  "calibrationStatus": "pending",
84
- "hardGate": true,
100
+ "executable": false,
85
101
  "id": "task-quality-judge",
86
- "kind": "model_judge",
87
- "privileged": true,
88
- "rewardEligible": true,
102
+ "rewardEligible": false,
89
103
  "rubricRef": {
90
104
  "contentHash": "455b3697c0617333a34b6521f167760fb8ae7e286059fa2ec327a5c97e66a2b3",
91
105
  "id": "rubrics-task-quality-md",
@@ -94,19 +108,8 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
94
108
  "sizeBytes": 1496,
95
109
  "visibility": "verifier"
96
110
  },
97
- "version": "1",
98
- "weight": 1
99
- }
100
- ],
101
- "id": "harness-refiner-08112026",
102
- "metadata": {
103
- "adaptationSplit": "validation",
104
- "benchmark": "harness-refiner",
105
- "frozenEvaluationSplit": "frozen_eval",
106
- "primaryMetric": "paired_foreground_provider_tokens",
107
- "protocolVersion": "2",
108
- "qualityPolicy": "hard_non_regression",
109
- "refinementMode": "sequential_product_lifecycle",
111
+ "version": "1"
112
+ },
110
113
  "toolDeclarationSource": "openpond-production-model-tool-definitions",
111
114
  "trainingSideEffect": false
112
115
  },
@@ -123,7 +126,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
123
126
  "expectedOutput"
124
127
  ]
125
128
  },
126
- "revision": 1,
129
+ "revision": 2,
127
130
  "schemaVersion": "openpond.tasksetRelease.v2",
128
131
  "tasks": [
129
132
  {
@@ -140,6 +143,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
140
143
  "clusterKey": "northstar-launch-packet",
141
144
  "expectedOutput": {
142
145
  "deliverable": "pdf",
146
+ "deterministicContract": {},
143
147
  "mustInclude": [
144
148
  "three decision options",
145
149
  "owners and dates",
@@ -166,6 +170,20 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
166
170
  "attachmentCount": 1
167
171
  },
168
172
  "privilegedContextRef": "expected-adaptation-board-launch-brief",
173
+ "requiredOutputs": [
174
+ {
175
+ "maxBytes": 10000000,
176
+ "mediaType": "application/pdf",
177
+ "metadata": {
178
+ "validationKinds": [
179
+ "structural",
180
+ "visual"
181
+ ]
182
+ },
183
+ "path": "adaptation-board-launch-brief.pdf",
184
+ "schemaRef": null
185
+ }
186
+ ],
169
187
  "split": "validation",
170
188
  "tags": [
171
189
  "artifact-verification",
@@ -187,6 +205,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
187
205
  "clusterKey": "checkout-latency-incident-packet",
188
206
  "expectedOutput": {
189
207
  "deliverable": "pdf",
208
+ "deterministicContract": {},
190
209
  "mustInclude": [
191
210
  "incident window",
192
211
  "confirmed impact",
@@ -215,6 +234,20 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
215
234
  "attachmentCount": 1
216
235
  },
217
236
  "privilegedContextRef": "expected-adaptation-latency-incident-review",
237
+ "requiredOutputs": [
238
+ {
239
+ "maxBytes": 10000000,
240
+ "mediaType": "application/pdf",
241
+ "metadata": {
242
+ "validationKinds": [
243
+ "structural",
244
+ "visual"
245
+ ]
246
+ },
247
+ "path": "adaptation-latency-incident-review.pdf",
248
+ "schemaRef": null
249
+ }
250
+ ],
218
251
  "split": "validation",
219
252
  "tags": [
220
253
  "artifact-verification",
@@ -236,6 +269,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
236
269
  "clusterKey": "harbor-program-budget-packet",
237
270
  "expectedOutput": {
238
271
  "deliverable": "spreadsheet",
272
+ "deterministicContract": {},
239
273
  "mustInclude": [
240
274
  "summary sheet",
241
275
  "detail sheet",
@@ -264,6 +298,20 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
264
298
  "attachmentCount": 1
265
299
  },
266
300
  "privilegedContextRef": "expected-adaptation-program-budget-workbook",
301
+ "requiredOutputs": [
302
+ {
303
+ "maxBytes": 10000000,
304
+ "mediaType": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
305
+ "metadata": {
306
+ "validationKinds": [
307
+ "structural",
308
+ "test"
309
+ ]
310
+ },
311
+ "path": "adaptation-program-budget-workbook.xlsx",
312
+ "schemaRef": null
313
+ }
314
+ ],
267
315
  "split": "validation",
268
316
  "tags": [
269
317
  "artifact-verification",
@@ -276,6 +324,27 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
276
324
  "clusterKey": "northwind-invoice-correction-message",
277
325
  "expectedOutput": {
278
326
  "deliverable": "message",
327
+ "deterministicContract": {
328
+ "forbiddenText": [
329
+ "intentional error",
330
+ "deliberate error"
331
+ ],
332
+ "maxWords": 130,
333
+ "requiredAny": [
334
+ [
335
+ "no payment",
336
+ "not due"
337
+ ]
338
+ ],
339
+ "requiredText": [
340
+ "inv-1842",
341
+ "120",
342
+ "102",
343
+ "august 14",
344
+ "accounts@example.com"
345
+ ],
346
+ "requireMessageBody": true
347
+ },
279
348
  "mustInclude": [
280
349
  "complete send-ready message copy",
281
350
  "INV-1842",
@@ -300,6 +369,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
300
369
  "attachmentCount": 0
301
370
  },
302
371
  "privilegedContextRef": "expected-adaptation-invoice-correction-email",
372
+ "requiredOutputs": [],
303
373
  "split": "validation",
304
374
  "tags": [
305
375
  "constraint-following",
@@ -313,6 +383,27 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
313
383
  "clusterKey": "nextjs-security-current-sources",
314
384
  "expectedOutput": {
315
385
  "deliverable": "report",
386
+ "deterministicContract": {
387
+ "minLinks": 1,
388
+ "requiredAny": [
389
+ [
390
+ "affected"
391
+ ],
392
+ [
393
+ "fixed",
394
+ "patched"
395
+ ],
396
+ [
397
+ "checked",
398
+ "as of"
399
+ ],
400
+ [
401
+ "version and configuration",
402
+ "version or configuration",
403
+ "exact version"
404
+ ]
405
+ ]
406
+ },
316
407
  "mustInclude": [
317
408
  "official advisory links",
318
409
  "affected and fixed versions",
@@ -334,6 +425,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
334
425
  "attachmentCount": 0
335
426
  },
336
427
  "privilegedContextRef": "expected-adaptation-nextjs-security-audit",
428
+ "requiredOutputs": [],
337
429
  "split": "validation",
338
430
  "tags": [
339
431
  "research-efficiency",
@@ -347,6 +439,31 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
347
439
  "clusterKey": "juniper-workshop-reschedule-message",
348
440
  "expectedOutput": {
349
441
  "deliverable": "message",
442
+ "deterministicContract": {
443
+ "forbiddenText": [
444
+ "venue caused",
445
+ "venue's fault"
446
+ ],
447
+ "maxWords": 120,
448
+ "requiredAny": [
449
+ [
450
+ "facilitator",
451
+ "presenter"
452
+ ],
453
+ [
454
+ "registration",
455
+ "registrations"
456
+ ]
457
+ ],
458
+ "requiredText": [
459
+ "september 10",
460
+ "2:00",
461
+ "et",
462
+ "recording",
463
+ "events@example.com"
464
+ ],
465
+ "requireMessageBody": true
466
+ },
350
467
  "mustInclude": [
351
468
  "complete send-ready message copy",
352
469
  "September 10",
@@ -372,6 +489,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
372
489
  "attachmentCount": 0
373
490
  },
374
491
  "privilegedContextRef": "expected-adaptation-workshop-reschedule-email",
492
+ "requiredOutputs": [],
375
493
  "split": "validation",
376
494
  "tags": [
377
495
  "constraint-following",
@@ -385,6 +503,38 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
385
503
  "clusterKey": "boston-dc-accessibility-sources",
386
504
  "expectedOutput": {
387
505
  "deliverable": "report",
506
+ "deterministicContract": {
507
+ "minLinks": 1,
508
+ "requiredAny": [
509
+ [
510
+ "wheelchair",
511
+ "accessible",
512
+ "accessibility"
513
+ ],
514
+ [
515
+ "outbound"
516
+ ],
517
+ [
518
+ "return"
519
+ ],
520
+ [
521
+ "disruption",
522
+ "service alert"
523
+ ],
524
+ [
525
+ "checked",
526
+ "as of"
527
+ ],
528
+ [
529
+ "confirm",
530
+ "confirmation"
531
+ ]
532
+ ],
533
+ "requiredText": [
534
+ "september 17",
535
+ "september 19"
536
+ ]
537
+ },
388
538
  "mustInclude": [
389
539
  "official operator sources",
390
540
  "outbound and return plan",
@@ -408,6 +558,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
408
558
  "attachmentCount": 0
409
559
  },
410
560
  "privilegedContextRef": "expected-adaptation-accessible-boston-dc-plan",
561
+ "requiredOutputs": [],
411
562
  "split": "validation",
412
563
  "tags": [
413
564
  "research-efficiency",
@@ -421,6 +572,33 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
421
572
  "clusterKey": "chatgpt-x-reddit-public-sample",
422
573
  "expectedOutput": {
423
574
  "deliverable": "report",
575
+ "deterministicContract": {
576
+ "minLinks": 1,
577
+ "requiredAny": [
578
+ [
579
+ "positive"
580
+ ],
581
+ [
582
+ "negative"
583
+ ],
584
+ [
585
+ "anecdote",
586
+ "anecdotal"
587
+ ],
588
+ [
589
+ "pattern",
590
+ "recurring"
591
+ ],
592
+ [
593
+ "sampling",
594
+ "sample"
595
+ ],
596
+ [
597
+ "limitation",
598
+ "access"
599
+ ]
600
+ ]
601
+ },
424
602
  "mustInclude": [
425
603
  "links to public examples",
426
604
  "dates",
@@ -444,6 +622,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
444
622
  "attachmentCount": 0
445
623
  },
446
624
  "privilegedContextRef": "expected-adaptation-chatgpt-public-experiences",
625
+ "requiredOutputs": [],
447
626
  "split": "validation",
448
627
  "tags": [
449
628
  "research-efficiency",
@@ -457,6 +636,29 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
457
636
  "clusterKey": "acme-launch-delay-message",
458
637
  "expectedOutput": {
459
638
  "deliverable": "message",
639
+ "deterministicContract": {
640
+ "forbiddenText": [
641
+ "testing failed",
642
+ "test failed",
643
+ "compensation"
644
+ ],
645
+ "maxWords": 140,
646
+ "requiredAny": [
647
+ [
648
+ "accessibility testing",
649
+ "accessibility test"
650
+ ],
651
+ [
652
+ "pilot access"
653
+ ]
654
+ ],
655
+ "requiredText": [
656
+ "august 27",
657
+ "august 22",
658
+ "pilot-support@example.com"
659
+ ],
660
+ "requireMessageBody": true
661
+ },
460
662
  "mustInclude": [
461
663
  "complete send-ready message copy",
462
664
  "August 27",
@@ -482,6 +684,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
482
684
  "attachmentCount": 0
483
685
  },
484
686
  "privilegedContextRef": "expected-adaptation-launch-delay-email",
687
+ "requiredOutputs": [],
485
688
  "split": "validation",
486
689
  "tags": [
487
690
  "constraint-following",
@@ -495,6 +698,33 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
495
698
  "clusterKey": "cirrus-service-window-message",
496
699
  "expectedOutput": {
497
700
  "deliverable": "message",
701
+ "deterministicContract": {
702
+ "forbiddenText": [
703
+ "zero interruption",
704
+ "no interruption"
705
+ ],
706
+ "maxWords": 125,
707
+ "requiredAny": [
708
+ [
709
+ "read-only",
710
+ "read only"
711
+ ],
712
+ [
713
+ "alerts"
714
+ ],
715
+ [
716
+ "no data loss"
717
+ ]
718
+ ],
719
+ "requiredText": [
720
+ "august 18",
721
+ "1:00",
722
+ "2:00",
723
+ "utc",
724
+ "status.example.com"
725
+ ],
726
+ "requireMessageBody": true
727
+ },
498
728
  "mustInclude": [
499
729
  "complete send-ready message copy",
500
730
  "August 18",
@@ -520,6 +750,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
520
750
  "attachmentCount": 0
521
751
  },
522
752
  "privilegedContextRef": "expected-adaptation-service-window-email",
753
+ "requiredOutputs": [],
523
754
  "split": "validation",
524
755
  "tags": [
525
756
  "constraint-following",
@@ -542,6 +773,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
542
773
  "clusterKey": "riverside-clinic-relocation-packet",
543
774
  "expectedOutput": {
544
775
  "deliverable": "pdf",
776
+ "deterministicContract": {},
545
777
  "mustInclude": [
546
778
  "three opening options",
547
779
  "owners and dates",
@@ -569,6 +801,20 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
569
801
  "attachmentCount": 1
570
802
  },
571
803
  "privilegedContextRef": "expected-frozen-clinic-relocation-brief",
804
+ "requiredOutputs": [
805
+ {
806
+ "maxBytes": 10000000,
807
+ "mediaType": "application/pdf",
808
+ "metadata": {
809
+ "validationKinds": [
810
+ "structural",
811
+ "visual"
812
+ ]
813
+ },
814
+ "path": "frozen-clinic-relocation-brief.pdf",
815
+ "schemaRef": null
816
+ }
817
+ ],
572
818
  "split": "frozen_eval",
573
819
  "tags": [
574
820
  "artifact-verification",
@@ -590,6 +836,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
590
836
  "clusterKey": "subscription-renewal-incident-packet",
591
837
  "expectedOutput": {
592
838
  "deliverable": "pdf",
839
+ "deterministicContract": {},
593
840
  "mustInclude": [
594
841
  "incident window",
595
842
  "attempt and timeout counts",
@@ -618,6 +865,20 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
618
865
  "attachmentCount": 1
619
866
  },
620
867
  "privilegedContextRef": "expected-frozen-payment-incident-review",
868
+ "requiredOutputs": [
869
+ {
870
+ "maxBytes": 10000000,
871
+ "mediaType": "application/pdf",
872
+ "metadata": {
873
+ "validationKinds": [
874
+ "structural",
875
+ "visual"
876
+ ]
877
+ },
878
+ "path": "frozen-payment-incident-review.pdf",
879
+ "schemaRef": null
880
+ }
881
+ ],
621
882
  "split": "frozen_eval",
622
883
  "tags": [
623
884
  "artifact-verification",
@@ -639,6 +900,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
639
900
  "clusterKey": "greenway-grant-budget-packet",
640
901
  "expectedOutput": {
641
902
  "deliverable": "spreadsheet",
903
+ "deterministicContract": {},
642
904
  "mustInclude": [
643
905
  "summary sheet",
644
906
  "detail sheet",
@@ -667,6 +929,20 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
667
929
  "attachmentCount": 1
668
930
  },
669
931
  "privilegedContextRef": "expected-frozen-grant-budget-workbook",
932
+ "requiredOutputs": [
933
+ {
934
+ "maxBytes": 10000000,
935
+ "mediaType": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
936
+ "metadata": {
937
+ "validationKinds": [
938
+ "structural",
939
+ "test"
940
+ ]
941
+ },
942
+ "path": "frozen-grant-budget-workbook.xlsx",
943
+ "schemaRef": null
944
+ }
945
+ ],
670
946
  "split": "frozen_eval",
671
947
  "tags": [
672
948
  "artifact-verification",
@@ -679,6 +955,29 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
679
955
  "clusterKey": "beacon-shipping-delay-message",
680
956
  "expectedOutput": {
681
957
  "deliverable": "message",
958
+ "deterministicContract": {
959
+ "forbiddenText": [
960
+ "equipment is lost",
961
+ "shipment is lost"
962
+ ],
963
+ "maxWords": 90,
964
+ "requiredAny": [
965
+ [
966
+ "transfer window"
967
+ ],
968
+ [
969
+ "installation",
970
+ "crew"
971
+ ]
972
+ ],
973
+ "requiredText": [
974
+ "august 21",
975
+ "august 22",
976
+ "morgan",
977
+ "logistics"
978
+ ],
979
+ "requireMessageBody": true
980
+ },
682
981
  "mustInclude": [
683
982
  "complete ready-to-post message copy",
684
983
  "August 21",
@@ -703,6 +1002,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
703
1002
  "attachmentCount": 0
704
1003
  },
705
1004
  "privilegedContextRef": "expected-frozen-shipping-delay-chat-message",
1005
+ "requiredOutputs": [],
706
1006
  "split": "frozen_eval",
707
1007
  "tags": [
708
1008
  "constraint-following",
@@ -716,6 +1016,27 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
716
1016
  "clusterKey": "python-requests-security-current-sources",
717
1017
  "expectedOutput": {
718
1018
  "deliverable": "report",
1019
+ "deterministicContract": {
1020
+ "minLinks": 1,
1021
+ "requiredAny": [
1022
+ [
1023
+ "affected"
1024
+ ],
1025
+ [
1026
+ "fixed",
1027
+ "patched"
1028
+ ],
1029
+ [
1030
+ "checked",
1031
+ "as of"
1032
+ ],
1033
+ [
1034
+ "dependency graph",
1035
+ "exact dependency",
1036
+ "usage"
1037
+ ]
1038
+ ]
1039
+ },
719
1040
  "mustInclude": [
720
1041
  "primary advisory links",
721
1042
  "affected and fixed versions",
@@ -737,6 +1058,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
737
1058
  "attachmentCount": 0
738
1059
  },
739
1060
  "privilegedContextRef": "expected-frozen-python-requests-security-audit",
1061
+ "requiredOutputs": [],
740
1062
  "split": "frozen_eval",
741
1063
  "tags": [
742
1064
  "research-efficiency",
@@ -750,6 +1072,29 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
750
1072
  "clusterKey": "cobalt-refund-support-message",
751
1073
  "expectedOutput": {
752
1074
  "deliverable": "message",
1075
+ "deterministicContract": {
1076
+ "forbiddenText": [
1077
+ "refund is approved",
1078
+ "refund has been approved"
1079
+ ],
1080
+ "maxWords": 110,
1081
+ "requiredAny": [
1082
+ [
1083
+ "original payment method"
1084
+ ],
1085
+ [
1086
+ "duplicate",
1087
+ "charge"
1088
+ ]
1089
+ ],
1090
+ "requiredText": [
1091
+ "$48",
1092
+ "august 16",
1093
+ "five business days",
1094
+ "cb-7714"
1095
+ ],
1096
+ "requireMessageBody": true
1097
+ },
753
1098
  "mustInclude": [
754
1099
  "complete send-ready reply copy",
755
1100
  "$48 duplicate charge",
@@ -774,6 +1119,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
774
1119
  "attachmentCount": 0
775
1120
  },
776
1121
  "privilegedContextRef": "expected-frozen-refund-support-reply",
1122
+ "requiredOutputs": [],
777
1123
  "split": "frozen_eval",
778
1124
  "tags": [
779
1125
  "constraint-following",
@@ -787,6 +1133,38 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
787
1133
  "clusterKey": "chicago-stl-accessibility-sources",
788
1134
  "expectedOutput": {
789
1135
  "deliverable": "report",
1136
+ "deterministicContract": {
1137
+ "minLinks": 1,
1138
+ "requiredAny": [
1139
+ [
1140
+ "wheelchair",
1141
+ "accessible",
1142
+ "accessibility"
1143
+ ],
1144
+ [
1145
+ "outbound"
1146
+ ],
1147
+ [
1148
+ "return"
1149
+ ],
1150
+ [
1151
+ "disruption",
1152
+ "service alert"
1153
+ ],
1154
+ [
1155
+ "checked",
1156
+ "as of"
1157
+ ],
1158
+ [
1159
+ "confirm",
1160
+ "confirmation"
1161
+ ]
1162
+ ],
1163
+ "requiredText": [
1164
+ "october 8",
1165
+ "october 10"
1166
+ ]
1167
+ },
790
1168
  "mustInclude": [
791
1169
  "official operator sources",
792
1170
  "outbound and return plan",
@@ -810,6 +1188,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
810
1188
  "attachmentCount": 0
811
1189
  },
812
1190
  "privilegedContextRef": "expected-frozen-accessible-chicago-stl-plan",
1191
+ "requiredOutputs": [],
813
1192
  "split": "frozen_eval",
814
1193
  "tags": [
815
1194
  "research-efficiency",
@@ -823,6 +1202,27 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
823
1202
  "clusterKey": "new-jersey-youth-grant-sources",
824
1203
  "expectedOutput": {
825
1204
  "deliverable": "report",
1205
+ "deterministicContract": {
1206
+ "minLinks": 1,
1207
+ "requiredAny": [
1208
+ [
1209
+ "new jersey",
1210
+ "nj"
1211
+ ],
1212
+ [
1213
+ "eligibility",
1214
+ "eligible"
1215
+ ],
1216
+ [
1217
+ "deadline",
1218
+ "due"
1219
+ ],
1220
+ [
1221
+ "checked",
1222
+ "as of"
1223
+ ]
1224
+ ]
1225
+ },
826
1226
  "mustInclude": [
827
1227
  "authoritative source links",
828
1228
  "eligibility evidence",
@@ -846,6 +1246,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
846
1246
  "attachmentCount": 0
847
1247
  },
848
1248
  "privilegedContextRef": "expected-frozen-new-jersey-youth-grants",
1249
+ "requiredOutputs": [],
849
1250
  "split": "frozen_eval",
850
1251
  "tags": [
851
1252
  "research-efficiency",
@@ -859,6 +1260,27 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
859
1260
  "clusterKey": "maple-maintenance-followup-message",
860
1261
  "expectedOutput": {
861
1262
  "deliverable": "message",
1263
+ "deterministicContract": {
1264
+ "forbiddenText": [
1265
+ "sue",
1266
+ "lawsuit",
1267
+ "legal action"
1268
+ ],
1269
+ "maxWords": 130,
1270
+ "requiredAny": [
1271
+ [
1272
+ "repair date",
1273
+ "repair schedule"
1274
+ ]
1275
+ ],
1276
+ "requiredText": [
1277
+ "july 28",
1278
+ "august 1",
1279
+ "security",
1280
+ "two business days"
1281
+ ],
1282
+ "requireMessageBody": true
1283
+ },
862
1284
  "mustInclude": [
863
1285
  "complete send-ready message copy",
864
1286
  "July 28",
@@ -883,6 +1305,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
883
1305
  "attachmentCount": 0
884
1306
  },
885
1307
  "privilegedContextRef": "expected-frozen-maintenance-followup-email",
1308
+ "requiredOutputs": [],
886
1309
  "split": "frozen_eval",
887
1310
  "tags": [
888
1311
  "constraint-following",
@@ -896,6 +1319,27 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
896
1319
  "clusterKey": "meridian-vendor-document-message",
897
1320
  "expectedOutput": {
898
1321
  "deliverable": "message",
1322
+ "deterministicContract": {
1323
+ "forbiddenText": [
1324
+ "cancel the contract",
1325
+ "contract cancellation"
1326
+ ],
1327
+ "maxWords": 120,
1328
+ "requiredAny": [
1329
+ [
1330
+ "onboarding",
1331
+ "cannot finish",
1332
+ "blocked"
1333
+ ]
1334
+ ],
1335
+ "requiredText": [
1336
+ "insurance certificate",
1337
+ "august 7",
1338
+ "rosa",
1339
+ "august 12"
1340
+ ],
1341
+ "requireMessageBody": true
1342
+ },
899
1343
  "mustInclude": [
900
1344
  "complete send-ready message copy",
901
1345
  "insurance certificate",
@@ -920,6 +1364,7 @@ export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
920
1364
  "attachmentCount": 0
921
1365
  },
922
1366
  "privilegedContextRef": "expected-frozen-vendor-document-followup-email",
1367
+ "requiredOutputs": [],
923
1368
  "split": "frozen_eval",
924
1369
  "tags": [
925
1370
  "constraint-following",
@@ -1264,5 +1709,5 @@ export const harnessRefinerBenchmarkAssets = Object.freeze({
1264
1709
  "fixtures/frozen-grant-budget.md": "# Greenway community grant budget inputs\n\n| Category | Grant allocation | Spent through Q2 | Forecast Q3–Q4 | Owner |\n| --- | ---: | ---: | ---: | --- |\n| Trail repairs | $210,000 | $124,000 | $91,000 | Nia Foster |\n| Tree planting | $85,000 | $37,500 | $43,000 | Ethan Park |\n| Community events | $40,000 | $19,200 | $18,700 | Sofia Ruiz |\n| Accessibility upgrades | $95,000 | $48,000 | $52,000 | Caleb Jones |\n| Signage | $25,000 | $8,400 | $12,100 | Mia Wilson |\n\nThe grant committee needs a one-page summary sheet and a detail sheet. Variance\nis grant allocation minus full-year forecast. Negative variance means the\ncategory is forecast over allocation.\n",
1265
1710
  "fixtures/frozen-payment-incident.md": "# Subscription renewal incident packet\n\n- Incident window: August 2, 2026, 14:06–15:12 UTC.\n- Confirmed: 6,284 renewal attempts were submitted.\n- Confirmed: 812 attempts received a provider timeout.\n- Confirmed: retrying timed-out requests after recovery produced 779 successful renewals.\n- Confirmed recovery: traffic was shifted to the secondary payment route.\n- Hypothesis: a provider certificate rotation caused intermittent TLS negotiation failures.\n- Unknown: whether 33 unresolved accounts require customer action.\n- Incident commander: Marcus Green.\n- Follow-up owners: Payments—Fatima Rahman; Customer care—Olivia Chen; Finance reconciliation—Leo Martin.\n",
1266
1711
  "rubrics/task-quality.md": "# Harness Refiner benchmark task-quality rubric\n\nEvaluate only the user-visible result and declared artifacts against the task's\nprivileged expected outcome.\n\n1. Treat every `mustInclude` item as required. A materially missing or invented\n fact fails the task.\n2. Treat every `mustNot` item as a hard prohibition.\n3. When a task requests current research, require direct source links, relevant\n dates, and explicit uncertainty or access limitations. Prefer primary sources\n when the expected outcome calls for them.\n4. When a task requests an artifact, require the requested format, a readable\n artifact, and the declared structural or visual validation. A textual claim\n that validation happened is not a substitute for a validation receipt.\n5. Do not reward verbosity, extra searches, or extra tool calls. Concision is\n preferred once all requirements are satisfied.\n6. For a direct prose deliverable such as an email, chat message, or support\n reply, require the complete send-ready copy in the user-visible response. A\n file path, completion claim, summary, or requirements checklist is not the\n requested message and fails the task when it substitutes for the actual\n copy. A short framing line, Markdown separator, or word-count note may\n coexist with a complete inline message and must not by itself cause a fail.\n7. Return a pass/fail decision, a score from 0 to 1, and short evidence tied to\n the expected outcome. Do not reveal privileged criteria to the agent.\n",
1267
- "verifiers/taskset-output-verifier.mjs": "export function verify({ task, output }) {\n const expected = task?.expectedOutput ?? {};\n const text = typeof output === \"string\" ? output : JSON.stringify(output ?? {});\n const requiredOutputs = Array.isArray(output?.requiredOutputs)\n ? output.requiredOutputs\n : [];\n const failures = [];\n\n if (!text.trim()) failures.push(\"empty_output\");\n if (\n expected.deliverable === \"pdf\" &&\n !requiredOutputs.some(\n (item) => item.mediaType === \"application/pdf\" && item.passed === true,\n )\n ) {\n failures.push(\"pdf_missing\");\n }\n if (\n expected.deliverable === \"spreadsheet\" &&\n !requiredOutputs.some(\n (item) =>\n item.passed === true &&\n [\n \"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\n \"text/csv\",\n ].includes(item.mediaType),\n )\n ) {\n failures.push(\"spreadsheet_missing\");\n }\n for (const required of expected.validation ?? []) {\n if (\n !requiredOutputs.some(\n (item) => item.passed === true && item.validationKinds?.includes(required),\n )\n ) {\n failures.push(`validation_missing:${required}`);\n }\n }\n\n return {\n passed: failures.length === 0,\n score: failures.length === 0 ? 1 : 0,\n rewardEligible: failures.length === 0,\n failures,\n };\n}\n"
1712
+ "verifiers/taskset-output-verifier.mjs": "export function verify({ task, output }) {\n const expected = task?.expectedOutput ?? {};\n const text = typeof output === \"string\" ? output : JSON.stringify(output ?? {});\n const visibleText = typeof output?.text === \"string\" ? output.text : text;\n const normalizedText = visibleText.normalize(\"NFKC\").toLowerCase();\n const contract = expected.deterministicContract ?? {};\n const requiredOutputs = Array.isArray(output?.requiredOutputs)\n ? output.requiredOutputs\n : [];\n const failures = [];\n\n if (!text.trim()) failures.push(\"empty_output\");\n for (const required of contract.requiredText ?? []) {\n if (!normalizedText.includes(String(required).toLowerCase())) {\n failures.push(`required_text_missing:${required}`);\n }\n }\n for (const group of contract.requiredAny ?? []) {\n if (!group.some((value) => normalizedText.includes(String(value).toLowerCase()))) {\n failures.push(`required_text_group_missing:${group.join(\"|\")}`);\n }\n }\n for (const forbidden of contract.forbiddenText ?? []) {\n if (normalizedText.includes(String(forbidden).toLowerCase())) {\n failures.push(`forbidden_text_present:${forbidden}`);\n }\n }\n const wordCount = visibleText.trim() ? visibleText.trim().split(/\\s+/).length : 0;\n if (Number.isFinite(contract.maxWords) && wordCount > contract.maxWords) {\n failures.push(`word_limit_exceeded:${wordCount}/${contract.maxWords}`);\n }\n const linkCount = (visibleText.match(/https?:\\/\\/[^\\s)\\]}]+/g) ?? []).length;\n if (Number.isFinite(contract.minLinks) && linkCount < contract.minLinks) {\n failures.push(`link_count_below_minimum:${linkCount}/${contract.minLinks}`);\n }\n if (\n contract.requireMessageBody === true\n && (\n wordCount < 20\n || (/checklist:/i.test(visibleText) && /(?:\\/workspace\\/|saved to|file path)/i.test(visibleText))\n )\n ) {\n failures.push(\"message_body_missing\");\n }\n if (\n expected.deliverable === \"pdf\" &&\n !requiredOutputs.some(\n (item) => item.mediaType === \"application/pdf\" && item.passed === true,\n )\n ) {\n failures.push(\"pdf_missing\");\n }\n if (\n expected.deliverable === \"spreadsheet\" &&\n !requiredOutputs.some(\n (item) =>\n item.passed === true &&\n [\n \"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\n \"text/csv\",\n ].includes(item.mediaType),\n )\n ) {\n failures.push(\"spreadsheet_missing\");\n }\n for (const required of expected.validation ?? []) {\n if (\n !requiredOutputs.some(\n (item) => item.passed === true && item.validationKinds?.includes(required),\n )\n ) {\n failures.push(`validation_missing:${required}`);\n }\n }\n\n return {\n passed: failures.length === 0,\n score: failures.length === 0 ? 1 : 0,\n rewardEligible: failures.length === 0,\n feedback: failures.length === 0\n ? \"The deterministic output contract passed.\"\n : `Deterministic output contract failed: ${failures.join(\", \")}.`,\n failures,\n };\n}\n"
1268
1713
  });