@openpond/evals 0.4.0 → 0.4.1

package/dist/builtin-benchmarks/harness-refiner.js

@@ -0,0 +1,1267 @@
+// Generated by benchmarks/harness-refiner/taskset/build.ts. Do not edit.
+import { TasksetReleaseSchema } from "../tasksets.js";
+export const harnessRefinerBenchmarkRelease = TasksetReleaseSchema.parse({
+    "capabilities": [
+        {
+            "id": "filesystem.workspace",
+            "portability": "host_adapter",
+            "required": true,
+            "scopes": [
+                "inputs:read",
+                "work:read-write",
+                "outputs:write"
+            ]
+        },
+        {
+            "id": "network.web-read",
+            "portability": "host_adapter",
+            "required": true,
+            "scopes": [
+                "search",
+                "fetch"
+            ]
+        },
+        {
+            "id": "artifact.pdf",
+            "portability": "host_adapter",
+            "required": true,
+            "scopes": [
+                "create",
+                "render",
+                "inspect"
+            ]
+        },
+        {
+            "id": "artifact.spreadsheet",
+            "portability": "host_adapter",
+            "required": true,
+            "scopes": [
+                "create",
+                "calculate",
+                "inspect"
+            ]
+        }
+    ],
+    "contentHash": "05dbf9058c09b75bc950ca5cdf4ac03650dba7bb0cb74a3d9e152b16c110a2f7",
+    "environment": {
+        "defaultTimeoutMs": 900000,
+        "deterministicSeeds": false,
+        "entrypoint": "openpond-work-v1",
+        "kind": "work",
+        "lifecycle": [
+            "create",
+            "reset",
+            "step",
+            "collect",
+            "destroy"
+        ],
+        "networkPolicy": "declared_read_only",
+        "protocolVersion": "openpond.environment.v1",
+        "stateful": true
+    },
+    "graders": [
+        {
+            "hardGate": true,
+            "id": "task-output-contract",
+            "kind": "custom_verifier",
+            "networkPolicy": "none",
+            "privileged": true,
+            "rewardEligible": true,
+            "timeoutMs": 30000,
+            "verifierRef": {
+                "contentHash": "5290dfae6969bd4581b1e1c111bdc1cb5f2828f7a91681635254b246c7590392",
+                "id": "verifiers-taskset-output-verifier-mjs",
+                "mediaType": "text/javascript",
+                "path": "verifiers/taskset-output-verifier.mjs",
+                "sizeBytes": 1295,
+                "visibility": "verifier"
+            },
+            "version": "1",
+            "weight": 1
+        },
+        {
+            "calibrationStatus": "pending",
+            "hardGate": true,
+            "id": "task-quality-judge",
+            "kind": "model_judge",
+            "privileged": true,
+            "rewardEligible": true,
+            "rubricRef": {
+                "contentHash": "455b3697c0617333a34b6521f167760fb8ae7e286059fa2ec327a5c97e66a2b3",
+                "id": "rubrics-task-quality-md",
+                "mediaType": "text/markdown",
+                "path": "rubrics/task-quality.md",
+                "sizeBytes": 1496,
+                "visibility": "verifier"
+            },
+            "version": "1",
+            "weight": 1
+        }
+    ],
+    "id": "harness-refiner-public-v1",
+    "metadata": {
+        "adaptationSplit": "validation",
+        "benchmark": "harness-refiner",
+        "frozenEvaluationSplit": "frozen_eval",
+        "primaryMetric": "paired_foreground_provider_tokens",
+        "protocolVersion": "1",
+        "qualityPolicy": "hard_non_regression",
+        "toolDeclarationSource": "openpond-production-model-tool-definitions",
+        "trainingSideEffect": false
+    },
+    "policy": {
+        "connectedAppScopes": [],
+        "hiddenGraderRefs": [
+            "rubrics-task-quality-md",
+            "verifiers-taskset-output-verifier-mjs"
+        ],
+        "policyVisibleFields": [
+            "input"
+        ],
+        "privilegedFields": [
+            "expectedOutput"
+        ]
+    },
+    "revision": 3,
+    "schemaVersion": "openpond.tasksetRelease.v2",
+    "tasks": [
+        {
+            "artifactRefs": [
+                {
+                    "contentHash": "8879f8e31a916f165724a3c3e3e57db5baa858cd170ac6799064aa49bc623aa4",
+                    "id": "fixtures-adaptation-board-launch-md",
+                    "mediaType": "text/markdown",
+                    "path": "fixtures/adaptation-board-launch.md",
+                    "sizeBytes": 642,
+                    "visibility": "policy"
+                }
+            ],
+            "clusterKey": "northstar-launch-packet",
+            "expectedOutput": {
+                "deliverable": "pdf",
+                "mustInclude": [
+                    "three decision options",
+                    "owners and dates",
+                    "confirmed facts",
+                    "open legal and finance gates"
+                ],
+                "mustNot": [
+                    "present an open gate as approved",
+                    "invent a recommended decision"
+                ],
+                "validation": [
+                    "structural",
+                    "visual"
+                ]
+            },
+            "id": "adaptation-board-launch-brief",
+            "input": {
+                "attachments": [
+                    "adaptation-board-launch.md"
+                ],
+                "prompt": "Please turn the attached Northstar launch packet into a polished two-page PDF decision brief for the executive meeting. Make the three options, owners, dates, confirmed facts, open gates, and risks easy to scan. The source records no final choice, so do not select, recommend, or imply a preferred option. Visually check the finished PDF before you send it back."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 1
+            },
+            "privilegedContextRef": "expected-adaptation-board-launch-brief",
+            "split": "validation",
+            "tags": [
+                "artifact-verification",
+                "decision-brief",
+                "adaptation"
+            ]
+        },
+        {
+            "artifactRefs": [
+                {
+                    "contentHash": "a3d44bb123a242d78d0f60d4d8ad352e499bef677eaf816a19d07359a172743f",
+                    "id": "fixtures-adaptation-latency-incident-md",
+                    "mediaType": "text/markdown",
+                    "path": "fixtures/adaptation-latency-incident.md",
+                    "sizeBytes": 733,
+                    "visibility": "policy"
+                }
+            ],
+            "clusterKey": "checkout-latency-incident-packet",
+            "expectedOutput": {
+                "deliverable": "pdf",
+                "mustInclude": [
+                    "incident window",
+                    "confirmed impact",
+                    "recovery",
+                    "hypotheses labeled as hypotheses",
+                    "unknowns",
+                    "follow-up owners"
+                ],
+                "mustNot": [
+                    "state either hypothesis as root cause",
+                    "claim unresolved carts were recovered"
+                ],
+                "validation": [
+                    "structural",
+                    "visual"
+                ]
+            },
+            "id": "adaptation-latency-incident-review",
+            "input": {
+                "attachments": [
+                    "adaptation-latency-incident.md"
+                ],
+                "prompt": "Create a concise PDF incident review from the attached checkout latency packet. Clearly separate confirmed observations, recovery actions, hypotheses, unknowns, customer impact, and follow-up owners. Check that the rendered PDF is readable and complete."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 1
+            },
+            "privilegedContextRef": "expected-adaptation-latency-incident-review",
+            "split": "validation",
+            "tags": [
+                "artifact-verification",
+                "incident-review",
+                "adaptation"
+            ]
+        },
+        {
+            "artifactRefs": [
+                {
+                    "contentHash": "63bca96753ae1491a45ba13e1a06cfb48973cfbe4f392e8f181e7b9648fd9db4",
+                    "id": "fixtures-adaptation-program-budget-md",
+                    "mediaType": "text/markdown",
+                    "path": "fixtures/adaptation-program-budget.md",
+                    "sizeBytes": 649,
+                    "visibility": "policy"
+                }
+            ],
+            "clusterKey": "harbor-program-budget-packet",
+            "expectedOutput": {
+                "deliverable": "spreadsheet",
+                "mustInclude": [
+                    "summary sheet",
+                    "detail sheet",
+                    "formula-driven full-year forecast",
+                    "formula-driven variance",
+                    "owner column",
+                    "overrun flag"
+                ],
+                "mustNot": [
+                    "replace formulas with typed totals",
+                    "reverse the variance sign"
+                ],
+                "validation": [
+                    "structural",
+                    "test"
+                ]
+            },
+            "id": "adaptation-program-budget-workbook",
+            "input": {
+                "attachments": [
+                    "adaptation-program-budget.md"
+                ],
+                "prompt": "Build an Excel workbook from the attached Harbor youth program budget. Include a one-page summary and a detail sheet with formulas for full-year forecast and variance, clearly flag forecast overruns, preserve the listed owners, and verify the calculations before returning it."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 1
+            },
+            "privilegedContextRef": "expected-adaptation-program-budget-workbook",
+            "split": "validation",
+            "tags": [
+                "artifact-verification",
+                "spreadsheet",
+                "adaptation"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "northwind-invoice-correction-message",
+            "expectedOutput": {
+                "deliverable": "message",
+                "mustInclude": [
+                    "complete send-ready message copy",
+                    "INV-1842",
+                    "120 seats instead of 102",
+                    "August 14",
+                    "no payment due until corrected",
+                    "accounts@example.com"
+                ],
+                "mustNot": [
+                    "suggest the error was intentional",
+                    "exceed 130 words",
+                    "return only a checklist or file path instead of the message"
+                ],
+                "validation": []
+            },
+            "id": "adaptation-invoice-correction-email",
+            "input": {
+                "attachments": [],
+                "prompt": "Draft a courteous email to Northwind Labs explaining that invoice INV-1842 incorrectly lists 120 seats instead of 102. A corrected invoice will arrive by August 14, no payment is due until it arrives, and billing questions should go to accounts@example.com. Keep it under 130 words and do not suggest the error was intentional."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-adaptation-invoice-correction-email",
+            "split": "validation",
+            "tags": [
+                "constraint-following",
+                "direct-deliverable",
+                "communication",
+                "adaptation"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "nextjs-security-current-sources",
+            "expectedOutput": {
+                "deliverable": "report",
+                "mustInclude": [
+                    "official advisory links",
+                    "affected and fixed versions",
+                    "date checked",
+                    "limits on exposure inference"
+                ],
+                "mustNot": [
+                    "declare exposure without project version and configuration",
+                    "use an uncited vulnerability list"
+                ],
+                "validation": []
+            },
+            "id": "adaptation-nextjs-security-audit",
+            "input": {
+                "attachments": [],
+                "prompt": "Audit the currently supported Next.js release lines for security advisories published in the last twelve months. Use primary sources, explain which versions are affected and fixed, avoid inferring that a project is vulnerable without its exact version and configuration, and give me a concise linked report with the date checked."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-adaptation-nextjs-security-audit",
+            "split": "validation",
+            "tags": [
+                "research-efficiency",
+                "primary-sources",
+                "security",
+                "adaptation"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "juniper-workshop-reschedule-message",
+            "expectedOutput": {
+                "deliverable": "message",
+                "mustInclude": [
+                    "complete send-ready message copy",
+                    "September 10",
+                    "2:00 p.m. ET",
+                    "facilitator unavailable",
+                    "registrations carry over",
+                    "recording",
+                    "events@example.com"
+                ],
+                "mustNot": [
+                    "blame the venue",
+                    "exceed 120 words",
+                    "return only a checklist or file path instead of the message"
+                ],
+                "validation": []
+            },
+            "id": "adaptation-workshop-reschedule-email",
+            "input": {
+                "attachments": [],
+                "prompt": "Draft a warm email to Juniper workshop registrants explaining that the September 3 session is moving to September 10 at 2:00 p.m. ET because the facilitator is unavailable. Existing registrations carry over, a recording will be shared, and questions should go to events@example.com. Keep it under 120 words and do not imply the venue caused the change."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-adaptation-workshop-reschedule-email",
+            "split": "validation",
+            "tags": [
+                "constraint-following",
+                "direct-deliverable",
+                "communication",
+                "adaptation"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "boston-dc-accessibility-sources",
+            "expectedOutput": {
+                "deliverable": "report",
+                "mustInclude": [
+                    "official operator sources",
+                    "outbound and return plan",
+                    "accessibility details",
+                    "disruption check",
+                    "time checked",
+                    "items needing confirmation"
+                ],
+                "mustNot": [
+                    "guarantee availability not confirmed by booking",
+                    "hide access limitations"
+                ],
+                "validation": []
+            },
+            "id": "adaptation-accessible-boston-dc-plan",
+            "input": {
+                "attachments": [],
+                "prompt": "Plan a wheelchair-accessible train trip from Boston to Washington, DC for September 17, 2026, returning September 19. Verify accessibility and disruption information from official sources, distinguish facts from anything that still needs confirmation, include direct links and the time checked, and keep the answer practical and concise."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-adaptation-accessible-boston-dc-plan",
+            "split": "validation",
+            "tags": [
+                "research-efficiency",
+                "current-information",
+                "travel",
+                "adaptation"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "chatgpt-x-reddit-public-sample",
+            "expectedOutput": {
+                "deliverable": "report",
+                "mustInclude": [
+                    "links to public examples",
+                    "dates",
+                    "positive experiences",
+                    "negative experiences",
+                    "pattern versus anecdote",
+                    "sampling limitations"
+                ],
+                "mustNot": [
+                    "post or interact",
+                    "represent a convenience sample as representative sentiment"
+                ],
+                "validation": []
+            },
+            "id": "adaptation-chatgpt-public-experiences",
+            "input": {
+                "attachments": [],
+                "prompt": "Research what people have publicly said about using ChatGPT on X and Reddit during the last thirty days. Give me a concise, linked summary of recurring positive and negative experiences, separate patterns from anecdotes, include dates, and explain any platform-access or sampling limitations. Do not post or interact with anyone."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-adaptation-chatgpt-public-experiences",
+            "split": "validation",
+            "tags": [
+                "research-efficiency",
+                "current-information",
+                "social-research",
+                "adaptation"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "acme-launch-delay-message",
+            "expectedOutput": {
+                "deliverable": "message",
+                "mustInclude": [
+                    "complete send-ready message copy",
+                    "August 27",
+                    "accessibility testing incomplete",
+                    "August 22 expectation",
+                    "pilot access remains",
+                    "support email"
+                ],
+                "mustNot": [
+                    "say testing failed",
+                    "exceed 140 words",
+                    "invent compensation",
+                    "return only a checklist or file path instead of the message"
+                ],
+                "validation": []
+            },
+            "id": "adaptation-launch-delay-email",
+            "input": {
+                "attachments": [],
+                "prompt": "Draft a calm email to the Acme pilot customers explaining that the August 20 launch is moving to August 27 because final accessibility testing is not complete. Testing is expected to finish August 22, existing pilot access remains available, and questions should go to pilot-support@example.com. Keep it under 140 words and do not imply the test has failed."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-adaptation-launch-delay-email",
+            "split": "validation",
+            "tags": [
+                "constraint-following",
+                "direct-deliverable",
+                "communication",
+                "adaptation"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "cirrus-service-window-message",
+            "expectedOutput": {
+                "deliverable": "message",
+                "mustInclude": [
+                    "complete send-ready message copy",
+                    "August 18",
+                    "1:00 to 2:00 UTC",
+                    "dashboard read-only",
+                    "alerts continue",
+                    "no data loss expected",
+                    "status.example.com"
+                ],
+                "mustNot": [
+                    "promise zero interruption",
+                    "exceed 125 words",
+                    "return only a checklist or file path instead of the message"
+                ],
+                "validation": []
+            },
+            "id": "adaptation-service-window-email",
+            "input": {
+                "attachments": [],
+                "prompt": "Draft a calm email to Cirrus customers about planned maintenance on August 18 from 1:00 to 2:00 UTC. The dashboard will be read-only, alerts will continue, no data loss is expected, and updates will appear at status.example.com. Keep it under 125 words and do not promise that there will be zero interruption."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-adaptation-service-window-email",
+            "split": "validation",
+            "tags": [
+                "constraint-following",
+                "direct-deliverable",
+                "communication",
+                "adaptation"
+            ]
+        },
+        {
+            "artifactRefs": [
+                {
+                    "contentHash": "cb3aec594b140129ebd872c427053ba63a303cc36d160448afc9ed1a0289d3ca",
+                    "id": "fixtures-frozen-clinic-relocation-md",
+                    "mediaType": "text/markdown",
+                    "path": "fixtures/frozen-clinic-relocation.md",
+                    "sizeBytes": 652,
+                    "visibility": "policy"
+                }
+            ],
+            "clusterKey": "riverside-clinic-relocation-packet",
+            "expectedOutput": {
+                "deliverable": "pdf",
+                "mustInclude": [
+                    "three opening options",
+                    "owners and dates",
+                    "confirmed facts",
+                    "permit and parking gates",
+                    "delivery risk"
+                ],
+                "mustNot": [
+                    "present either approval as complete",
+                    "invent a chosen opening option"
+                ],
+                "validation": [
+                    "structural",
+                    "visual"
+                ]
+            },
+            "id": "frozen-clinic-relocation-brief",
+            "input": {
+                "attachments": [
+                    "frozen-clinic-relocation.md"
+                ],
+                "prompt": "Please turn the attached Riverside clinic relocation packet into a polished two-page PDF decision brief. Make the opening options, owners, dates, confirmed facts, unresolved approvals, and delivery risk easy to scan, and visually check the finished PDF before returning it."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 1
+            },
+            "privilegedContextRef": "expected-frozen-clinic-relocation-brief",
+            "split": "frozen_eval",
+            "tags": [
+                "artifact-verification",
+                "decision-brief",
+                "frozen-eval"
+            ]
+        },
+        {
+            "artifactRefs": [
+                {
+                    "contentHash": "80aad04ac67682165f098884fd1a86e9fb3ef15d2da3124401308dd23884b612",
+                    "id": "fixtures-frozen-payment-incident-md",
+                    "mediaType": "text/markdown",
+                    "path": "fixtures/frozen-payment-incident.md",
+                    "sizeBytes": 672,
+                    "visibility": "policy"
+                }
+            ],
+            "clusterKey": "subscription-renewal-incident-packet",
+            "expectedOutput": {
+                "deliverable": "pdf",
+                "mustInclude": [
+                    "incident window",
+                    "attempt and timeout counts",
+                    "recovery",
+                    "hypothesis label",
+                    "33 unresolved accounts",
+                    "follow-up owners"
+                ],
+                "mustNot": [
+                    "state the certificate hypothesis as confirmed",
+                    "claim all accounts resolved"
+                ],
+                "validation": [
+                    "structural",
+                    "visual"
+                ]
+            },
+            "id": "frozen-payment-incident-review",
+            "input": {
+                "attachments": [
+                    "frozen-payment-incident.md"
+                ],
+                "prompt": "Create a concise PDF incident review from the attached subscription renewal incident packet. Separate confirmed impact and recovery from the root-cause hypothesis and unresolved customer accounts, preserve the owners, and check the rendered PDF for readability and completeness."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 1
+            },
+            "privilegedContextRef": "expected-frozen-payment-incident-review",
+            "split": "frozen_eval",
+            "tags": [
+                "artifact-verification",
+                "incident-review",
+                "frozen-eval"
+            ]
+        },
+        {
+            "artifactRefs": [
+                {
+                    "contentHash": "c58b740d57b031cfe22d475d30b39a1a49a43820f1840a1d8054731cef58c8e8",
+                    "id": "fixtures-frozen-grant-budget-md",
+                    "mediaType": "text/markdown",
+                    "path": "fixtures/frozen-grant-budget.md",
+                    "sizeBytes": 661,
+                    "visibility": "policy"
+                }
+            ],
+            "clusterKey": "greenway-grant-budget-packet",
+            "expectedOutput": {
+                "deliverable": "spreadsheet",
+                "mustInclude": [
+                    "summary sheet",
+                    "detail sheet",
+                    "formula-driven full-year forecast",
+                    "formula-driven variance",
+                    "owner column",
+                    "overrun flag"
+                ],
+                "mustNot": [
+                    "replace formulas with typed totals",
+                    "reverse the variance sign"
+                ],
+                "validation": [
+                    "structural",
+                    "test"
+                ]
+            },
+            "id": "frozen-grant-budget-workbook",
+            "input": {
+                "attachments": [
+                    "frozen-grant-budget.md"
+                ],
+                "prompt": "Build an Excel workbook from the attached Greenway community grant budget. Include a one-page summary and detail sheet with formulas for full-year forecast and variance, flag forecast overruns, preserve owners, and verify all calculations before returning it."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 1
+            },
+            "privilegedContextRef": "expected-frozen-grant-budget-workbook",
+            "split": "frozen_eval",
+            "tags": [
+                "artifact-verification",
+                "spreadsheet",
+                "frozen-eval"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "beacon-shipping-delay-message",
+            "expectedOutput": {
+                "deliverable": "message",
+                "mustInclude": [
+                    "complete ready-to-post message copy",
+                    "August 21",
+                    "carrier missed transfer window",
+                    "August 22 installation",
+                    "Morgan",
+                    "logistics channel"
+                ],
+                "mustNot": [
+                    "say the equipment is lost",
+                    "exceed 90 words",
+                    "return only a checklist or file path instead of the message"
+                ],
+                "validation": []
+            },
+            "id": "frozen-shipping-delay-chat-message",
+            "input": {
+                "attachments": [],
+                "prompt": "Write a concise team chat message explaining that the Beacon equipment shipment is now expected August 21 instead of August 19 because the carrier missed its transfer window. The installation crew remains booked for August 22, Morgan owns the carrier follow-up, and the team should flag conflicts in the logistics channel. Keep it under 90 words and do not say the equipment is lost."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-frozen-shipping-delay-chat-message",
+            "split": "frozen_eval",
+            "tags": [
+                "constraint-following",
+                "direct-deliverable",
+                "communication",
+                "frozen-eval"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "python-requests-security-current-sources",
+            "expectedOutput": {
+                "deliverable": "report",
+                "mustInclude": [
+                    "primary advisory links",
+                    "affected and fixed versions",
+                    "date checked",
+                    "dependency and usage caveat"
+                ],
+                "mustNot": [
+                    "declare deployment exposure without evidence",
+                    "use an uncited vulnerability list"
+                ],
+                "validation": []
+            },
+            "id": "frozen-python-requests-security-audit",
+            "input": {
+                "attachments": [],
+                "prompt": "Audit the supported Python Requests release lines for security advisories published in the last eighteen months. Use primary sources, identify affected and fixed versions, avoid claiming a specific deployment is exposed without its exact dependency graph and usage, and provide a concise linked report with the date checked."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-frozen-python-requests-security-audit",
+            "split": "frozen_eval",
+            "tags": [
+                "research-efficiency",
+                "primary-sources",
+                "security",
+                "frozen-eval"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "cobalt-refund-support-message",
+            "expectedOutput": {
+                "deliverable": "message",
+                "mustInclude": [
+                    "complete send-ready reply copy",
+                    "$48 duplicate charge",
+                    "August 16",
+                    "original payment method",
+                    "five business days",
+                    "CB-7714"
+                ],
+                "mustNot": [
+                    "state the refund is already approved",
+                    "exceed 110 words",
+                    "return only a checklist or file path instead of the message"
+                ],
+                "validation": []
+            },
+            "id": "frozen-refund-support-reply",
+            "input": {
+                "attachments": [],
+                "prompt": "Write a helpful support reply to a Cobalt customer whose duplicate $48 charge is being reviewed. The review should finish by August 16, any confirmed duplicate will be refunded to the original payment method within five business days, and the case number is CB-7714. Keep it under 110 words and do not state that the refund has already been approved."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-frozen-refund-support-reply",
+            "split": "frozen_eval",
+            "tags": [
+                "constraint-following",
+                "direct-deliverable",
+                "communication",
+                "frozen-eval"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "chicago-stl-accessibility-sources",
+            "expectedOutput": {
+                "deliverable": "report",
+                "mustInclude": [
+                    "official operator sources",
+                    "outbound and return plan",
+                    "accessibility details",
+                    "disruption check",
+                    "time checked",
+                    "items needing confirmation"
+                ],
+                "mustNot": [
+                    "guarantee unconfirmed availability",
+                    "hide access limitations"
+                ],
+                "validation": []
+            },
+            "id": "frozen-accessible-chicago-stl-plan",
+            "input": {
+                "attachments": [],
+                "prompt": "Plan a wheelchair-accessible train trip from Chicago to St. Louis for October 8, 2026, returning October 10. Verify accessibility and disruption information from official sources, distinguish confirmed facts from anything requiring booking confirmation, include direct links and the time checked, and keep it practical."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-frozen-accessible-chicago-stl-plan",
+            "split": "frozen_eval",
+            "tags": [
+                "research-efficiency",
+                "current-information",
+                "travel",
+                "frozen-eval"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "new-jersey-youth-grant-sources",
+            "expectedOutput": {
+                "deliverable": "report",
+                "mustInclude": [
+                    "authoritative source links",
+                    "eligibility evidence",
+                    "deadlines",
+                    "date checked",
+                    "only realistic matches"
+                ],
+                "mustNot": [
+                    "include closed grants",
+                    "include grants without verified nonprofit and program eligibility",
+                    "pad the list"
+                ],
+                "validation": []
+            },
+            "id": "frozen-new-jersey-youth-grants",
+            "input": {
+                "attachments": [],
+                "prompt": "Find currently open grants that a New Jersey nonprofit running after-school programs could realistically apply for, with deadlines between now and December 31, 2026. Use authoritative sources, link each opportunity, verify eligibility and deadline, omit weak matches instead of padding the list, and state when you checked."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-frozen-new-jersey-youth-grants",
+            "split": "frozen_eval",
+            "tags": [
+                "research-efficiency",
+                "current-information",
+                "funding",
+                "frozen-eval"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "maple-maintenance-followup-message",
+            "expectedOutput": {
+                "deliverable": "message",
+                "mustInclude": [
+                    "complete send-ready message copy",
+                    "July 28",
+                    "August 1 inspection",
+                    "security concern",
+                    "repair date within two business days"
+                ],
+                "mustNot": [
+                    "make a legal threat",
+                    "claim facts not supplied",
+                    "exceed 130 words",
+                    "return only a checklist or file path instead of the message"
+                ],
+                "validation": []
+            },
+            "id": "frozen-maintenance-followup-email",
+            "input": {
+                "attachments": [],
+                "prompt": "Draft a firm but courteous email to Maple Property Management following up on a bedroom window that has not closed securely since July 28. Maintenance inspected it August 1 but did not repair it. Ask for a repair date within two business days, mention the security concern, and keep the email under 130 words without making legal threats."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-frozen-maintenance-followup-email",
+            "split": "frozen_eval",
+            "tags": [
+                "constraint-following",
+                "direct-deliverable",
+                "communication",
+                "frozen-eval"
+            ]
+        },
+        {
+            "artifactRefs": [],
+            "clusterKey": "meridian-vendor-document-message",
+            "expectedOutput": {
+                "deliverable": "message",
+                "mustInclude": [
+                    "complete send-ready message copy",
+                    "insurance certificate",
+                    "August 7",
+                    "onboarding blocked",
+                    "Rosa",
+                    "August 12"
+                ],
+                "mustNot": [
+                    "threaten contract cancellation",
+                    "exceed 120 words",
+                    "return only a checklist or file path instead of the message"
+                ],
+                "validation": []
+            },
+            "id": "frozen-vendor-document-followup-email",
+            "input": {
+                "attachments": [],
+                "prompt": "Draft a firm but professional email to Meridian Freight following up on the insurance certificate promised for August 7. It has not arrived, carrier onboarding cannot finish without it, and Rosa needs the document or a confirmed delivery date by August 12. Keep it under 120 words and do not threaten to cancel the contract."
+            },
+            "policyVisibleContext": {
+                "attachmentCount": 0
+            },
+            "privilegedContextRef": "expected-frozen-vendor-document-followup-email",
+            "split": "frozen_eval",
+            "tags": [
+                "constraint-following",
+                "direct-deliverable",
+                "communication",
+                "frozen-eval"
+            ]
+        }
+    ],
+    "tools": [
+        {
+            "description": "Start Work compute only when it is needed, then return live sandbox status and the stable /workspace/inputs, /workspace/work, and /workspace/outputs layout.",
+            "inputSchema": {
+                "additionalProperties": false,
+                "properties": {},
+                "type": "object"
+            },
+            "inputSchemaHash": "042b35e50dd5dd505f0228e829f17a8aeaae50e49d865c4754cf9ba1edba9bd0",
+            "name": "work_environment",
+            "sideEffect": "write",
+            "timeoutMs": 120000
+        },
+        {
+            "description": "List files in the Work scratch, input, or completed-output area. This lazily starts Work compute.",
+            "inputSchema": {
+                "additionalProperties": false,
+                "properties": {
+                    "area": {
+                        "enum": [
+                            "inputs",
+                            "work",
+                            "outputs"
+                        ],
+                        "type": "string"
+                    },
+                    "path": {
+                        "description": "Optional relative path inside the selected area.",
+                        "type": "string"
+                    },
+                    "recursive": {
+                        "type": "boolean"
+                    }
+                },
+                "required": [
+                    "area"
+                ],
+                "type": "object"
+            },
+            "inputSchemaHash": "a3a68e8aeb4f23710aa21fe95d4a361a14c08bcbb61a2a959a60b7b00c82807f",
+            "name": "work_list_files",
+            "sideEffect": "read",
+            "timeoutMs": 120000
+        },
+        {
+            "description": "Read a bounded file from Work inputs, scratch space, or completed-output candidates.",
+            "inputSchema": {
+                "additionalProperties": false,
+                "properties": {
+                    "area": {
+                        "enum": [
+                            "inputs",
+                            "work",
+                            "outputs"
+                        ],
+                        "type": "string"
+                    },
+                    "maxBytes": {
+                        "maximum": 1000000,
+                        "minimum": 1,
+                        "type": "integer"
+                    },
+                    "path": {
+                        "minLength": 1,
+                        "type": "string"
+                    }
+                },
+                "required": [
+                    "area",
+                    "path"
+                ],
+                "type": "object"
+            },
+            "inputSchemaHash": "deb9cd248f2734751286054c381ac9572b248370181970ced6563a73c905a4f0",
+            "name": "work_read_file",
+            "sideEffect": "read",
+            "timeoutMs": 120000
+        },
+        {
+            "description": "Write a UTF-8 Work scratch file or completed-output candidate. Use area=outputs only for a finished result that is ready to validate and save.",
+            "inputSchema": {
+                "additionalProperties": false,
+                "properties": {
+                    "area": {
+                        "enum": [
+                            "work",
+                            "outputs"
+                        ],
+                        "type": "string"
+                    },
+                    "content": {
+                        "type": "string"
+                    },
+                    "path": {
+                        "minLength": 1,
+                        "type": "string"
+                    }
+                },
+                "required": [
+                    "area",
+                    "path",
+                    "content"
+                ],
+                "type": "object"
+            },
+            "inputSchemaHash": "6a19df98686e99a324564006b0b92082f185ac4b430e9285829f781251dd145b",
+            "name": "work_write_file",
+            "sideEffect": "write",
+            "timeoutMs": 120000
+        },
+        {
+            "description": "Edit a Work scratch file or output candidate by exact text replacement after reading it.",
+            "inputSchema": {
+                "additionalProperties": false,
+                "properties": {
+                    "area": {
+                        "enum": [
+                            "work",
+                            "outputs"
+                        ],
+                        "type": "string"
+                    },
+                    "newText": {
+                        "type": "string"
+                    },
+                    "oldText": {
+                        "minLength": 1,
+                        "type": "string"
+                    },
+                    "path": {
+                        "minLength": 1,
+                        "type": "string"
+                    },
+                    "replaceAll": {
+                        "type": "boolean"
+                    }
+                },
+                "required": [
+                    "area",
+                    "path",
+                    "oldText",
+                    "newText"
+                ],
+                "type": "object"
+            },
+            "inputSchemaHash": "f507d9d9abbd8cf1556f6e925c159234295c3fa01800c5556767f96be8b011db",
+            "name": "work_edit_file",
+            "sideEffect": "write",
+            "timeoutMs": 120000
+        },
+        {
+            "description": "Run a bounded shell command from /workspace/work. Write finished deliverables to ../outputs and inspect them; the runtime preserves output files automatically when the turn ends.",
+            "inputSchema": {
+                "additionalProperties": false,
+                "properties": {
+                    "command": {
+                        "minLength": 1,
+                        "type": "string"
+                    },
+                    "timeoutSeconds": {
+                        "maximum": 3600,
+                        "minimum": 1,
+                        "type": "integer"
+                    }
+                },
+                "required": [
+                    "command"
+                ],
+                "type": "object"
+            },
+            "inputSchemaHash": "5ba3a52cddb3f575218bed855020e91eef14092d75922c963a24c6bc97fadeda",
+            "name": "work_exec",
+            "sideEffect": "write",
+            "timeoutMs": 300000
+        },
+        {
+            "description": "Explicitly copy one completed file from /workspace/outputs to durable OpenPond output storage before turn completion. Normal Work turns preserve output files automatically.",
+            "inputSchema": {
+                "additionalProperties": false,
+                "properties": {
+                    "path": {
+                        "description": "Path relative to /workspace/outputs, for example report.md.",
+                        "minLength": 1,
+                        "type": "string"
+                    },
+                    "suggestedName": {
+                        "maxLength": 180,
+                        "minLength": 1,
+                        "type": "string"
+                    },
+                    "validation": {
+                        "items": {
+                            "additionalProperties": false,
+                            "properties": {
+                                "detail": {
+                                    "maxLength": 4000,
+                                    "type": "string"
+                                },
+                                "kind": {
+                                    "enum": [
+                                        "structural",
+                                        "visual",
+                                        "test",
+                                        "user_review"
+                                    ],
+                                    "type": "string"
+                                },
+                                "label": {
+                                    "maxLength": 240,
+                                    "minLength": 1,
+                                    "type": "string"
+                                },
+                                "ref": {
+                                    "maxLength": 4096,
+                                    "type": "string"
+                                },
+                                "status": {
+                                    "enum": [
+                                        "passed",
+                                        "failed",
+                                        "not_run"
+                                    ],
+                                    "type": "string"
+                                }
+                            },
+                            "required": [
+                                "kind",
+                                "status",
+                                "label"
+                            ],
+                            "type": "object"
+                        },
+                        "maxItems": 32,
+                        "type": "array"
+                    }
+                },
+                "required": [
+                    "path"
+                ],
+                "type": "object"
+            },
+            "inputSchemaHash": "ca853f64b300c74441b522db03facbcc9c5bb89c86189dae24182bab39510a83",
+            "name": "work_save_output",
+            "sideEffect": "write",
+            "timeoutMs": 120000
+        },
+        {
+            "description": "Stop Work compute after durable outputs have been saved. Saved OutputRefs remain available.",
+            "inputSchema": {
+                "additionalProperties": false,
+                "properties": {},
+                "type": "object"
+            },
+            "inputSchemaHash": "042b35e50dd5dd505f0228e829f17a8aeaae50e49d865c4754cf9ba1edba9bd0",
+            "name": "work_stop",
+            "sideEffect": "write",
+            "timeoutMs": 120000
+        },
+        {
+            "description": "Search the web for current or external information. The app renders clickable source pills automatically. Cite by source title or source name in prose by default; when the user explicitly requests URLs or linked evidence, use the result URLs in clickable Markdown links.",
+            "inputSchema": {
+                "additionalProperties": false,
+                "properties": {
+                    "domains": {
+                        "description": "Optional domain filters.",
+                        "items": {
+                            "type": "string"
+                        },
+                        "maxItems": 10,
+                        "type": "array"
+                    },
+                    "limit": {
+                        "description": "Maximum number of results to return.",
+                        "maximum": 10,
+                        "minimum": 1,
+                        "type": "integer"
+                    },
+                    "query": {
+                        "description": "Search query.",
+                        "minLength": 1,
+                        "type": "string"
+                    },
+                    "recencyDays": {
+                        "description": "Only return results from this many recent days when supported.",
+                        "minimum": 0,
+                        "type": "integer"
+                    }
+                },
+                "required": [
+                    "query"
+                ],
+                "type": "object"
+            },
+            "inputSchemaHash": "2a138bf9b526a624ac2442669fccb277fac2ba818821b3a2da1dd27d62954056",
+            "name": "web_search",
+            "sideEffect": "read",
+            "timeoutMs": 120000
+        },
+        {
+            "description": "Fetch and extract readable text from a known HTTP(S) URL. Use this when the user provides a URL or a search result has an exact page to inspect; use web_search when discovering unknown pages by query.",
+            "inputSchema": {
+                "additionalProperties": false,
+                "properties": {
+                    "maxBytes": {
+                        "description": "Maximum UTF-8 bytes of extracted text to return. Defaults to 20000.",
+                        "maximum": 100000,
+                        "minimum": 1,
+                        "type": "integer"
+                    },
+                    "url": {
+                        "description": "HTTP or HTTPS URL to fetch.",
+                        "minLength": 1,
+                        "type": "string"
+                    }
+                },
+                "required": [
+                    "url"
+                ],
+                "type": "object"
+            },
+            "inputSchemaHash": "0afe31d8dbe05643e4a19ca0897c606e379f37d741b4f61816ad8ba4d48b61b3",
+            "name": "web_fetch",
+            "sideEffect": "read",
+            "timeoutMs": 120000
+        }
+    ]
+});
+export const harnessRefinerBenchmarkAssets = Object.freeze({
+    "fixtures/adaptation-board-launch.md": "# Northstar launch decision packet\n\n- Decision meeting: August 18, 2026 at 2:00 PM America/New_York.\n- Proposed launch: September 14, 2026.\n- Executive owner: Maya Chen.\n- Engineering owner: Rafael Ortiz.\n- Confirmed: the API load test passed at 2.4x expected peak traffic.\n- Confirmed: support coverage is staffed for launch week.\n- Open gate: Legal has not approved the updated data-processing addendum.\n- Open gate: Finance has not approved the final annual-plan price.\n- Risk: the Android store review may take between three and seven business days.\n- Decision requested: launch on September 14, delay one week, or run a web-only launch.\n",
+    "fixtures/adaptation-latency-incident.md": "# Checkout latency incident packet\n\n- Incident window: August 7, 2026, 09:42–10:31 UTC.\n- Confirmed: p95 checkout latency rose from 780 ms to 4.8 seconds.\n- Confirmed: 3.1% of checkout attempts returned HTTP 504.\n- Confirmed: the database connection pool reached its configured ceiling.\n- Confirmed recovery: increasing the pool ceiling and recycling two workers restored service.\n- Hypothesis: a reporting query introduced in release 2026.08.07 increased lock contention.\n- Hypothesis: a regional network event amplified connection churn.\n- Unknown: whether abandoned carts were later recovered.\n- Incident commander: Priya Shah.\n- Follow-up owners: Database—Noah Williams; Reporting—Elena García; Customer impact—Sam Lee.\n",
+    "fixtures/adaptation-program-budget.md": "# Harbor youth program budget inputs\n\n| Category | Approved budget | Actual through July | Forecast Aug–Dec | Owner |\n| --- | ---: | ---: | ---: | --- |\n| Teaching staff | $180,000 | $101,400 | $78,000 | Jordan Bell |\n| Facility | $72,000 | $42,000 | $30,000 | Casey Morgan |\n| Transportation | $48,000 | $31,800 | $24,500 | Taylor Reed |\n| Meals | $36,000 | $20,700 | $17,500 | Morgan Patel |\n| Supplies | $24,000 | $11,900 | $9,600 | Avery Kim |\n\nThe board wants a one-page summary sheet plus a detail sheet. Variance should be\ncalculated as approved budget minus full-year forecast. Negative variance means\nthe program is forecast over budget.\n",
+    "fixtures/frozen-clinic-relocation.md": "# Riverside clinic relocation packet\n\n- Target opening: October 5, 2026.\n- Executive owner: Dr. Lena Brooks.\n- Facilities owner: Omar Haddad.\n- Confirmed: the lease is executed and construction passed its first inspection.\n- Confirmed: the medical-record network circuit is installed.\n- Open gate: the state pharmacy permit has not been issued.\n- Open gate: the accessible parking redesign needs city approval.\n- Risk: two examination tables have an estimated delivery date of October 2–9.\n- Decision requested: retain October 5, delay to October 12, or open without pharmacy service.\n- Decision meeting: September 22, 2026 at 11:00 America/Chicago.\n",
+    "fixtures/frozen-grant-budget.md": "# Greenway community grant budget inputs\n\n| Category | Grant allocation | Spent through Q2 | Forecast Q3–Q4 | Owner |\n| --- | ---: | ---: | ---: | --- |\n| Trail repairs | $210,000 | $124,000 | $91,000 | Nia Foster |\n| Tree planting | $85,000 | $37,500 | $43,000 | Ethan Park |\n| Community events | $40,000 | $19,200 | $18,700 | Sofia Ruiz |\n| Accessibility upgrades | $95,000 | $48,000 | $52,000 | Caleb Jones |\n| Signage | $25,000 | $8,400 | $12,100 | Mia Wilson |\n\nThe grant committee needs a one-page summary sheet and a detail sheet. Variance\nis grant allocation minus full-year forecast. Negative variance means the\ncategory is forecast over allocation.\n",
+    "fixtures/frozen-payment-incident.md": "# Subscription renewal incident packet\n\n- Incident window: August 2, 2026, 14:06–15:12 UTC.\n- Confirmed: 6,284 renewal attempts were submitted.\n- Confirmed: 812 attempts received a provider timeout.\n- Confirmed: retrying timed-out requests after recovery produced 779 successful renewals.\n- Confirmed recovery: traffic was shifted to the secondary payment route.\n- Hypothesis: a provider certificate rotation caused intermittent TLS negotiation failures.\n- Unknown: whether 33 unresolved accounts require customer action.\n- Incident commander: Marcus Green.\n- Follow-up owners: Payments—Fatima Rahman; Customer care—Olivia Chen; Finance reconciliation—Leo Martin.\n",
+    "rubrics/task-quality.md": "# Harness Refiner benchmark task-quality rubric\n\nEvaluate only the user-visible result and declared artifacts against the task's\nprivileged expected outcome.\n\n1. Treat every `mustInclude` item as required. A materially missing or invented\n   fact fails the task.\n2. Treat every `mustNot` item as a hard prohibition.\n3. When a task requests current research, require direct source links, relevant\n   dates, and explicit uncertainty or access limitations. Prefer primary sources\n   when the expected outcome calls for them.\n4. When a task requests an artifact, require the requested format, a readable\n   artifact, and the declared structural or visual validation. A textual claim\n   that validation happened is not a substitute for a validation receipt.\n5. Do not reward verbosity, extra searches, or extra tool calls. Concision is\n   preferred once all requirements are satisfied.\n6. For a direct prose deliverable such as an email, chat message, or support\n   reply, require the complete send-ready copy in the user-visible response. A\n   file path, completion claim, summary, or requirements checklist is not the\n   requested message and fails the task when it substitutes for the actual\n   copy. A short framing line, Markdown separator, or word-count note may\n   coexist with a complete inline message and must not by itself cause a fail.\n7. Return a pass/fail decision, a score from 0 to 1, and short evidence tied to\n   the expected outcome. Do not reveal privileged criteria to the agent.\n",
+    "verifiers/taskset-output-verifier.mjs": "export function verify({ task, output }) {\n  const expected = task?.expectedOutput ?? {};\n  const text = typeof output === \"string\" ? output : JSON.stringify(output ?? {});\n  const requiredOutputs = Array.isArray(output?.requiredOutputs)\n    ? output.requiredOutputs\n    : [];\n  const failures = [];\n\n  if (!text.trim()) failures.push(\"empty_output\");\n  if (\n    expected.deliverable === \"pdf\" &&\n    !requiredOutputs.some(\n      (item) => item.mediaType === \"application/pdf\" && item.passed === true,\n    )\n  ) {\n    failures.push(\"pdf_missing\");\n  }\n  if (\n    expected.deliverable === \"spreadsheet\" &&\n    !requiredOutputs.some(\n      (item) =>\n        item.passed === true &&\n        [\n          \"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\n          \"text/csv\",\n        ].includes(item.mediaType),\n    )\n  ) {\n    failures.push(\"spreadsheet_missing\");\n  }\n  for (const required of expected.validation ?? []) {\n    if (\n      !requiredOutputs.some(\n        (item) => item.passed === true && item.validationKinds?.includes(required),\n      )\n    ) {\n      failures.push(`validation_missing:${required}`);\n    }\n  }\n\n  return {\n    passed: failures.length === 0,\n    score: failures.length === 0 ? 1 : 0,\n    rewardEligible: failures.length === 0,\n    failures,\n  };\n}\n"
+});