openpond-sdk 0.10.0 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/advanced-refiner-evaluations.js +42 -0
- package/dist/advanced-refiner-evaluations.js.map +7 -0
- package/dist/candidate-evaluations.js +25 -0
- package/dist/candidate-evaluations.js.map +7 -0
- package/dist/{chunk-W5TXWVFS.js → chunk-2JKV5OJO.js} +88 -59
- package/dist/chunk-2JKV5OJO.js.map +7 -0
- package/dist/{chunk-2ZJRG3KL.js → chunk-2NSNBED7.js} +7 -3
- package/dist/chunk-2NSNBED7.js.map +7 -0
- package/dist/{chunk-Y4NGWGTA.js → chunk-4NZW4LFB.js} +268 -264
- package/dist/chunk-4NZW4LFB.js.map +7 -0
- package/dist/{chunk-DSGGM2ME.js → chunk-4SZKRRTO.js} +2 -2
- package/dist/chunk-6FQKRY5K.js +445 -0
- package/dist/chunk-6FQKRY5K.js.map +7 -0
- package/dist/chunk-7Q7I6FAT.js +139 -0
- package/dist/chunk-7Q7I6FAT.js.map +7 -0
- package/dist/chunk-CPEOFPPF.js +632 -0
- package/dist/chunk-CPEOFPPF.js.map +7 -0
- package/dist/chunk-G3FWDZMZ.js +267 -0
- package/dist/chunk-G3FWDZMZ.js.map +7 -0
- package/dist/chunk-GDJRGRHA.js +144 -0
- package/dist/chunk-GDJRGRHA.js.map +7 -0
- package/dist/chunk-HKAR3VPB.js +65 -0
- package/dist/chunk-HKAR3VPB.js.map +7 -0
- package/dist/{chunk-32H2EJRA.js → chunk-JEUHIQWF.js} +10 -7
- package/dist/chunk-JEUHIQWF.js.map +7 -0
- package/dist/chunk-KQWCXW3A.js +294 -0
- package/dist/chunk-KQWCXW3A.js.map +7 -0
- package/dist/chunk-LFH4EKWH.js +1408 -0
- package/dist/chunk-LFH4EKWH.js.map +7 -0
- package/dist/{chunk-DT7FRNIZ.js → chunk-MW2N3BX5.js} +4 -3
- package/dist/{chunk-DT7FRNIZ.js.map → chunk-MW2N3BX5.js.map} +2 -2
- package/dist/{chunk-I3TQTXKB.js → chunk-QVCMFQKA.js} +6 -6
- package/dist/{chunk-I3TQTXKB.js.map → chunk-QVCMFQKA.js.map} +1 -1
- package/dist/{chunk-I2YM3UY4.js → chunk-RIX56CJL.js} +9 -6
- package/dist/chunk-RIX56CJL.js.map +7 -0
- package/dist/{chunk-PJ2VLNJF.js → chunk-SGRYMU4C.js} +2 -2
- package/dist/chunk-SLS3GFPG.js +125 -0
- package/dist/chunk-SLS3GFPG.js.map +7 -0
- package/dist/chunk-SVU4RZ4K.js +70 -0
- package/dist/chunk-SVU4RZ4K.js.map +7 -0
- package/dist/chunk-SXTC2GT6.js +160 -0
- package/dist/chunk-SXTC2GT6.js.map +7 -0
- package/dist/chunk-T4APHDES.js +74 -0
- package/dist/chunk-T4APHDES.js.map +7 -0
- package/dist/chunk-UNMOEI2K.js +587 -0
- package/dist/chunk-UNMOEI2K.js.map +7 -0
- package/dist/chunk-VC5OV5AS.js +174 -0
- package/dist/chunk-VC5OV5AS.js.map +7 -0
- package/dist/{chunk-7B44TQZD.js → chunk-WX22WN33.js} +13 -7
- package/dist/chunk-WX22WN33.js.map +7 -0
- package/dist/{chunk-JCPLAIZL.js → chunk-XCH27NHJ.js} +106 -76
- package/dist/chunk-XCH27NHJ.js.map +7 -0
- package/dist/chunk-XEUOJ5WF.js +31 -0
- package/dist/chunk-XEUOJ5WF.js.map +7 -0
- package/dist/chunk-YCQVVFE5.js +37 -0
- package/dist/chunk-YCQVVFE5.js.map +7 -0
- package/dist/chunk-YTGRET2L.js +604 -0
- package/dist/chunk-YTGRET2L.js.map +7 -0
- package/dist/connected-evidence.js +97 -0
- package/dist/connected-evidence.js.map +7 -0
- package/dist/dataset-commerce.js +773 -0
- package/dist/dataset-commerce.js.map +7 -0
- package/dist/dataset-workspaces.js +45 -80
- package/dist/dataset-workspaces.js.map +4 -4
- package/dist/experiment-evaluation-schedules.js +48 -0
- package/dist/experiment-evaluation-schedules.js.map +7 -0
- package/dist/experiment-improvements.js +52 -0
- package/dist/experiment-improvements.js.map +7 -0
- package/dist/experiments.js +48 -90
- package/dist/experiments.js.map +3 -3
- package/dist/human-review.js +13 -0
- package/dist/human-review.js.map +7 -0
- package/dist/index.js +221 -14
- package/dist/index.js.map +2 -2
- package/dist/learning.js +1 -1
- package/dist/model-starter-attempts.js +2 -2
- package/dist/model-starter-catalog.js +9 -4
- package/dist/model-starter-catalog.js.map +1 -1
- package/dist/model-starters.js +9 -4
- package/dist/model-taskset-runs.js +3 -3
- package/dist/post-training.js +47 -0
- package/dist/post-training.js.map +7 -0
- package/dist/taskset-drafts.js +33 -21
- package/dist/taskset-drafts.js.map +2 -2
- package/dist/taskset-packages.js +28 -558
- package/dist/taskset-packages.js.map +4 -4
- package/dist/training-bundle.js +61 -20
- package/dist/training-bundle.js.map +2 -2
- package/dist/training-projects.js +1 -1
- package/dist/training.js +7 -2
- package/dist/types/packages/sdk/src/advanced-refiner-evaluations.d.ts +2682 -0
- package/dist/types/packages/sdk/src/advanced-refiner-evaluations.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/candidate-evaluation-client.d.ts +1079 -0
- package/dist/types/packages/sdk/src/candidate-evaluation-client.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/candidate-evaluations.d.ts +845 -0
- package/dist/types/packages/sdk/src/candidate-evaluations.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/claude-code-runtime.d.ts +51 -0
- package/dist/types/packages/sdk/src/claude-code-runtime.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/connected-collection-status.d.ts +47 -0
- package/dist/types/packages/sdk/src/connected-collection-status.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/connected-dataset-publication.d.ts +15 -0
- package/dist/types/packages/sdk/src/connected-dataset-publication.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/connected-dataset-source.d.ts +52 -0
- package/dist/types/packages/sdk/src/connected-dataset-source.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/connected-evidence-client.d.ts +2502 -0
- package/dist/types/packages/sdk/src/connected-evidence-client.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/connected-evidence-contracts.d.ts +235 -0
- package/dist/types/packages/sdk/src/connected-evidence-contracts.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/connected-evidence-http.d.ts +6 -0
- package/dist/types/packages/sdk/src/connected-evidence-http.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/connected-evidence.d.ts +8 -0
- package/dist/types/packages/sdk/src/connected-evidence.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/connected-recorded-execution.d.ts +690 -0
- package/dist/types/packages/sdk/src/connected-recorded-execution.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/connected-sync.d.ts +282 -0
- package/dist/types/packages/sdk/src/connected-sync.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/dataset-commerce-contracts.d.ts +230 -0
- package/dist/types/packages/sdk/src/dataset-commerce-contracts.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/dataset-commerce-files.d.ts +114 -0
- package/dist/types/packages/sdk/src/dataset-commerce-files.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/dataset-commerce-responses.d.ts +608 -0
- package/dist/types/packages/sdk/src/dataset-commerce-responses.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/dataset-commerce.d.ts +575 -0
- package/dist/types/packages/sdk/src/dataset-commerce.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/dataset-workspace-client.d.ts +772 -120
- package/dist/types/packages/sdk/src/dataset-workspace-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/dataset-workspace-contracts.d.ts +293 -38
- package/dist/types/packages/sdk/src/dataset-workspace-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/dataset-workspace-operations.d.ts +294 -40
- package/dist/types/packages/sdk/src/dataset-workspace-operations.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/evaluation-coordination-http.d.ts +17 -0
- package/dist/types/packages/sdk/src/evaluation-coordination-http.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/experiment-client.d.ts +3513 -52
- package/dist/types/packages/sdk/src/experiment-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiment-contracts.d.ts +1017 -1
- package/dist/types/packages/sdk/src/experiment-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiment-evaluation-schedules.d.ts +2780 -0
- package/dist/types/packages/sdk/src/experiment-evaluation-schedules.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/experiment-grading-contracts.d.ts +24 -0
- package/dist/types/packages/sdk/src/experiment-grading-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiment-improvement-client.d.ts +5724 -0
- package/dist/types/packages/sdk/src/experiment-improvement-client.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/experiment-improvement-contracts.d.ts +13024 -0
- package/dist/types/packages/sdk/src/experiment-improvement-contracts.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/experiment-improvement-hosted-adoption.d.ts +1361 -0
- package/dist/types/packages/sdk/src/experiment-improvement-hosted-adoption.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/experiment-improvements.d.ts +4 -0
- package/dist/types/packages/sdk/src/experiment-improvements.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/experiment-run-contracts.d.ts +262 -0
- package/dist/types/packages/sdk/src/experiment-run-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiment-run-details.d.ts +1388 -1
- package/dist/types/packages/sdk/src/experiment-run-details.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiment-scoring-contracts.d.ts +5 -2
- package/dist/types/packages/sdk/src/experiment-scoring-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiments.d.ts +3 -0
- package/dist/types/packages/sdk/src/experiments.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/grader-inspection.d.ts +307 -5
- package/dist/types/packages/sdk/src/grader-inspection.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/harness-experiment-policy.d.ts +238 -0
- package/dist/types/packages/sdk/src/harness-experiment-policy.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/human-review-client.d.ts +1267 -0
- package/dist/types/packages/sdk/src/human-review-client.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/human-review.d.ts +3 -0
- package/dist/types/packages/sdk/src/human-review.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/index.d.ts +10 -0
- package/dist/types/packages/sdk/src/index.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/managed-training-preparation.d.ts +181 -0
- package/dist/types/packages/sdk/src/managed-training-preparation.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-batch-review-contracts.d.ts +63 -4
- package/dist/types/packages/sdk/src/model-batch-review-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-batch-review-preparation.d.ts +90 -5
- package/dist/types/packages/sdk/src/model-batch-review-preparation.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-batch-review.d.ts +55 -13
- package/dist/types/packages/sdk/src/model-batch-review.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-learning-policy.d.ts +370 -1
- package/dist/types/packages/sdk/src/model-learning-policy.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-starter-attempts.d.ts +363 -19
- package/dist/types/packages/sdk/src/model-starter-attempts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-starter-catalog.d.ts +234 -12
- package/dist/types/packages/sdk/src/model-starter-catalog.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-starter-execution.d.ts +78 -1
- package/dist/types/packages/sdk/src/model-starter-execution.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-starters.d.ts +466 -13
- package/dist/types/packages/sdk/src/model-starters.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-derivation.d.ts +312 -4
- package/dist/types/packages/sdk/src/model-taskset-derivation.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-resources.d.ts +78 -1
- package/dist/types/packages/sdk/src/model-taskset-resources.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-runs-contracts.d.ts +1282 -4
- package/dist/types/packages/sdk/src/model-taskset-runs-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-runs.d.ts +1285 -5
- package/dist/types/packages/sdk/src/model-taskset-runs.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/post-training.d.ts +3529 -0
- package/dist/types/packages/sdk/src/post-training.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/profile-evaluation-discovery.d.ts +1127 -0
- package/dist/types/packages/sdk/src/profile-evaluation-discovery.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/profile-external-dataset-package.d.ts +1398 -0
- package/dist/types/packages/sdk/src/profile-external-dataset-package.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/taskset-authored-contracts.d.ts +157 -24
- package/dist/types/packages/sdk/src/taskset-authored-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-authored-files.d.ts +111 -18
- package/dist/types/packages/sdk/src/taskset-authored-files.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-authored-validation.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-calibration-fixtures.d.ts +438 -0
- package/dist/types/packages/sdk/src/taskset-calibration-fixtures.d.ts.map +1 -0
- package/dist/types/packages/sdk/src/taskset-draft-api-contracts.d.ts +292 -38
- package/dist/types/packages/sdk/src/taskset-draft-api-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-authoring.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-client.d.ts +661 -103
- package/dist/types/packages/sdk/src/taskset-draft-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-core.d.ts +227 -20
- package/dist/types/packages/sdk/src/taskset-draft-core.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-dataset-artifacts.d.ts +14 -14
- package/dist/types/packages/sdk/src/taskset-draft-dataset-sources.d.ts +17 -17
- package/dist/types/packages/sdk/src/taskset-draft-document.d.ts +147 -20
- package/dist/types/packages/sdk/src/taskset-draft-document.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-package-compiler.d.ts +696 -42
- package/dist/types/packages/sdk/src/taskset-draft-package-compiler.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-source.d.ts +256 -36
- package/dist/types/packages/sdk/src/taskset-draft-source.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-workspace.d.ts +292 -38
- package/dist/types/packages/sdk/src/taskset-draft-workspace.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-drafts.d.ts +2 -0
- package/dist/types/packages/sdk/src/taskset-drafts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-client.d.ts +1494 -26
- package/dist/types/packages/sdk/src/taskset-package-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-contracts.d.ts +746 -12
- package/dist/types/packages/sdk/src/taskset-package-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-grader-projection.d.ts +77 -3
- package/dist/types/packages/sdk/src/taskset-package-grader-projection.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-learning.d.ts +101 -11
- package/dist/types/packages/sdk/src/taskset-package-learning.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-projection.d.ts +459 -39
- package/dist/types/packages/sdk/src/taskset-package-projection.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-acceptance.d.ts +10 -10
- package/dist/types/packages/sdk/src/training-candidate-decisions.d.ts +2 -2
- package/dist/types/packages/sdk/src/training-evaluation-results.d.ts +1 -1
- package/dist/types/packages/sdk/src/training-learning-batch.d.ts +376 -42
- package/dist/types/packages/sdk/src/training-learning-batch.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-preparations.d.ts +3930 -0
- package/dist/types/packages/sdk/src/training-preparations.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-project-client.d.ts +776 -8
- package/dist/types/packages/sdk/src/training-project-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-project-contracts.d.ts +1161 -11
- package/dist/types/packages/sdk/src/training-project-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training.d.ts +2655 -24
- package/dist/types/packages/sdk/src/training.d.ts.map +1 -1
- package/package.json +35 -3
- package/dist/chunk-2ZJRG3KL.js.map +0 -7
- package/dist/chunk-32H2EJRA.js.map +0 -7
- package/dist/chunk-7B44TQZD.js.map +0 -7
- package/dist/chunk-I2YM3UY4.js.map +0 -7
- package/dist/chunk-JA3KFK3U.js +0 -580
- package/dist/chunk-JA3KFK3U.js.map +0 -7
- package/dist/chunk-JCPLAIZL.js.map +0 -7
- package/dist/chunk-LGEF4S5I.js +0 -1229
- package/dist/chunk-LGEF4S5I.js.map +0 -7
- package/dist/chunk-PRRUBH2B.js +0 -314
- package/dist/chunk-PRRUBH2B.js.map +0 -7
- package/dist/chunk-W5TXWVFS.js.map +0 -7
- package/dist/chunk-Y4NGWGTA.js.map +0 -7
- /package/dist/{chunk-DSGGM2ME.js.map → chunk-4SZKRRTO.js.map} +0 -0
- /package/dist/{chunk-PJ2VLNJF.js.map → chunk-SGRYMU4C.js.map} +0 -0
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/taskset-authored-contracts.ts", "../src/taskset-authored-validation.ts"],
|
|
4
|
+
"sourcesContent": ["import { z } from \"zod\";\nimport { VersionedReleaseRefSchema } from \"@openpond/harness\";\nimport { ChatModelRefSchema } from \"@openpond/harness/models\";\nimport { TasksetMetricPolicySchema } from \"@openpond/evals/metrics\";\nimport { DatasetArtifactManifestSchema } from \"./taskset-draft-dataset-artifacts.js\";\nimport {\n TasksetPurposeSchema,\n TasksetBenchmarkBindingSchema,\n TasksetPreferenceComparisonBindingSchema,\n TasksetSourceRefSchema,\n TaskPolicyBoundarySchema,\n TasksetEnvironmentContractSchema,\n TasksetCapabilityManifestSchema,\n TaskDataRecordSchema,\n GraderSpecSchema,\n GraderFixtureSchema,\n LearningSignalInventorySchema,\n type TasksetSourceRef, type TrainingSourceRef,\n} from \"./taskset-draft-core.js\";\n\nconst IdSchema = z.string().trim().min(1).max(240);\nconst TimestampSchema = z.string().trim().min(1);\nconst HashSchema = z.string().trim().min(8).max(256);\nconst MetadataSchema = z.record(z.string(), z.unknown()).default({});\nconst NullableIdSchema = IdSchema.nullable();\n\nexport const TASKSET_WORK_TOOL_NAMES = [\n \"work_capabilities\",\n \"work_environment\",\n \"work_list_files\",\n \"work_read_file\",\n \"work_read_document\",\n \"work_write_file\",\n \"work_write_docx\",\n \"work_edit_file\",\n \"work_delete_file\",\n \"work_exec\",\n \"work_save_output\",\n \"work_stop\",\n] as const;\n\nexport const TasksetStatusSchema = z.enum([\n \"draft\",\n \"awaiting_disclosure_approval\",\n \"awaiting_materialization_approval\",\n \"materializing\",\n \"validating\",\n \"needs_review\",\n \"baselining\",\n \"ready\",\n \"blocked\",\n \"failed\",\n \"archived\",\n]);\n\nexport const DatasetBuildIntentSchema = z.enum([\n \"demonstrations\",\n \"preferences\",\n \"verifiable_reward\",\n \"rubric\",\n \"discovery\",\n]);\n\nconst DatasetEvidenceTextSchema = z.string().trim().max(100_000);\n\nexport const DatasetBuildSpecificationSchema = z.discriminatedUnion(\"kind\", [\n z.object({\n kind: z.literal(\"demonstrations\"),\n behavior: DatasetEvidenceTextSchema,\n examples: z.array(z.object({\n id: IdSchema,\n prompt: DatasetEvidenceTextSchema,\n response: DatasetEvidenceTextSchema,\n })).max(1_000).default([]),\n }),\n z.object({\n kind: z.literal(\"preferences\"),\n preference: DatasetEvidenceTextSchema,\n pairs: z.array(z.object({\n id: IdSchema,\n prompt: DatasetEvidenceTextSchema,\n chosen: DatasetEvidenceTextSchema,\n rejected: DatasetEvidenceTextSchema,\n rationale: DatasetEvidenceTextSchema,\n })).max(1_000).default([]),\n }),\n z.object({\n kind: z.literal(\"verifiable_reward\"),\n task: DatasetEvidenceTextSchema,\n rules: z.array(z.object({\n id: IdSchema,\n points: z.number().finite(),\n condition: DatasetEvidenceTextSchema,\n })).max(1_000).default([]),\n otherwisePoints: z.number().finite().default(0),\n }),\n z.object({\n kind: z.literal(\"rubric\"),\n task: DatasetEvidenceTextSchema,\n criteria: z.array(z.object({\n id: IdSchema,\n label: z.string().trim().max(500),\n description: DatasetEvidenceTextSchema,\n })).max(1_000).default([]),\n positiveExample: DatasetEvidenceTextSchema,\n negativeExample: DatasetEvidenceTextSchema,\n boundaryExample: DatasetEvidenceTextSchema,\n }),\n]);\n\nexport const GeneratedTaskFileSchema = z.object({\n path: z.string().trim().min(1).max(1_000),\n role: z.enum([\"environment\", \"verifier\", \"fixture\"]),\n content: z.string().max(250_000),\n});\n\nexport const TrainingPathRecommendationSchema = z.object({\n primaryMethod: z.enum([\"sft\", \"dpo\", \"grpo\", \"ppo\", \"sdft\", \"opsd\", \"sdpo\"]),\n bootstrap: z.object({\n method: z.literal(\"sft\"),\n purpose: z.literal(\"trajectory_bootstrap\"),\n demonstrationRefs: z.array(IdSchema).min(1).max(100_000),\n limitations: z.array(z.string().trim().min(1).max(5_000)).min(1).max(100),\n }).nullable(),\n});\n\nexport const TrainingMethodReadinessReasonCodeSchema = z.enum([\n \"taskset_not_ready\",\n \"demonstrations_missing\",\n \"preference_pairs_missing\",\n \"preference_pairs_invalid\",\n \"executable_reward_missing\",\n \"reward_not_calibrated\",\n \"reward_model_missing\",\n \"value_model_required\",\n \"frozen_eval_missing\",\n]);\n\nexport const TrainingMethodReadinessSchema = z.object({\n method: z.enum([\"sft\", \"dpo\", \"grpo\", \"ppo\"]),\n status: z.enum([\"recommended\", \"compatible\", \"needs_dataset_work\"]),\n reasonCodes: z.array(TrainingMethodReadinessReasonCodeSchema).default([]),\n reasons: z.array(z.string().trim().min(1).max(5_000)).default([]),\n});\n\nexport const TasksetReadinessFindingSchema = z.object({\n code: IdSchema,\n message: z.string().trim().min(1).max(5_000),\n path: z.string().trim().max(2_000).nullable(),\n});\n\nexport const TasksetReadinessReportSchema = z.object({\n schemaVersion: z.literal(\"openpond.tasksetReadiness.v1\"),\n tasksetId: IdSchema,\n tasksetHash: HashSchema,\n ready: z.boolean(),\n recommendedMethod: z.enum([\"none\", \"retrieval\", \"sft\", \"dpo\", \"grpo\", \"ppo\", \"sdft\", \"opd\", \"opsd\", \"sdpo\"]),\n trainingPath: TrainingPathRecommendationSchema.nullable().default(null),\n methodReadiness: z.array(TrainingMethodReadinessSchema).default([]),\n compatibleDestinationClasses: z.array(\n z.enum([\"export\", \"custom\", \"hosted_managed\"]),\n ),\n blockers: z.array(TasksetReadinessFindingSchema).default([]),\n advisories: z.array(TasksetReadinessFindingSchema).default([]),\n warnings: z.array(z.string().trim().min(1).max(5_000)).default([]),\n generatedAt: TimestampSchema,\n});\n\nexport const AuthoringRepairSchema = z.object({ attempt: z.number().int().positive(), summary: z.string().trim().min(1).max(5_000), createdAt: TimestampSchema });\n\nexport const AuthoringProvenanceSchema = z.object({\n schemaVersion: z.literal(\"openpond.taskAuthoringProvenance.v1\"),\n model: ChatModelRefSchema.nullable(),\n modelConfig: MetadataSchema,\n skillHash: HashSchema,\n promptTemplateVersion: z.string().trim().min(1).max(200),\n buildIntent: DatasetBuildIntentSchema.default(\"demonstrations\"),\n buildSpecification: DatasetBuildSpecificationSchema.nullable().default(null),\n evidenceHashes: z.array(HashSchema).max(100_000),\n tasksetSdkVersion: z.string().trim().min(1).max(100),\n sourceCommit: z.string().trim().min(1).max(256).nullable(),\n repairHistory: z.array(AuthoringRepairSchema).max(1_000),\n createdAt: TimestampSchema,\n});\n\nexport const TasksetSchema = z.object({\n schemaVersion: z.literal(\"openpond.taskset.v1\"),\n id: IdSchema,\n revision: z.number().int().positive().default(1),\n profileId: IdSchema,\n profileRelease: VersionedReleaseRefSchema.nullable().optional(),\n createImproveRunId: NullableIdSchema.default(null),\n name: z.string().trim().min(1).max(500),\n // Portable tasks may carry all instructions in their individual inputs.\n // Preserve an absent package prompt; draft publication validates its objective.\n objective: z.string().trim().max(20_000),\n purpose: TasksetPurposeSchema.default(\"general\"),\n benchmark: TasksetBenchmarkBindingSchema.nullable().default(null),\n preferenceComparison: TasksetPreferenceComparisonBindingSchema.nullable().default(null),\n status: TasksetStatusSchema,\n sourceRefs: z.array(TasksetSourceRefSchema).min(1).max(100_000),\n datasetArtifact: DatasetArtifactManifestSchema.nullable().optional(),\n policy: TaskPolicyBoundarySchema,\n environment: TasksetEnvironmentContractSchema,\n capabilities: TasksetCapabilityManifestSchema,\n metrics: TasksetMetricPolicySchema.optional(),\n tasks: z.array(TaskDataRecordSchema).max(1_000_000),\n graders: z.array(GraderSpecSchema).min(1).max(1_000),\n // Imported releases are inspectable before local calibration. Admission\n // requires real fixtures separately in validateTaskset.\n graderFixtures: z.array(GraderFixtureSchema).max(100_000),\n learningSignals: LearningSignalInventorySchema,\n authoringProvenance: AuthoringProvenanceSchema,\n readiness: TasksetReadinessReportSchema.nullable(),\n contentHash: HashSchema,\n createdAt: TimestampSchema,\n updatedAt: TimestampSchema,\n metadata: MetadataSchema,\n}).superRefine((taskset, context) => {\n if (taskset.purpose === \"benchmark\" && !taskset.benchmark) {\n context.addIssue({\n code: \"custom\",\n message: \"Benchmark Tasksets require an immutable benchmark binding.\",\n path: [\"benchmark\"],\n });\n }\n if (taskset.purpose !== \"benchmark\" && taskset.benchmark) {\n context.addIssue({\n code: \"custom\",\n message: \"Only benchmark Tasksets may carry a benchmark binding.\",\n path: [\"benchmark\"],\n });\n }\n if (taskset.datasetArtifact && taskset.tasks.length > 0) {\n context.addIssue({\n code: \"custom\",\n message:\n \"Artifact-backed Tasksets may not duplicate canonical rows inline.\",\n path: [\"tasks\"],\n });\n }\n if (!taskset.datasetArtifact && taskset.tasks.length === 0) {\n context.addIssue({\n code: \"custom\",\n message: \"A Taskset requires inline tasks or a Dataset artifact manifest.\",\n path: [\"tasks\"],\n });\n }\n});\n\nexport type DatasetBuildIntent = z.infer<typeof DatasetBuildIntentSchema>;\nexport type DatasetBuildSpecification = z.infer<typeof DatasetBuildSpecificationSchema>;\nexport type GeneratedTaskFile = z.infer<typeof GeneratedTaskFileSchema>;\nexport type TrainingPathRecommendation = z.infer<typeof TrainingPathRecommendationSchema>;\nexport type TrainingMethodReadinessReasonCode = z.infer<typeof TrainingMethodReadinessReasonCodeSchema>;\nexport type TrainingMethodReadiness = z.infer<typeof TrainingMethodReadinessSchema>;\nexport type TasksetReadinessReport = z.infer<typeof TasksetReadinessReportSchema>;\nexport type AuthoringProvenance = z.infer<typeof AuthoringProvenanceSchema>;\nexport type AuthoringRepair = z.infer<typeof AuthoringRepairSchema>;\nexport type Taskset = z.infer<typeof TasksetSchema>;\n\nexport function isTrainingSourceRef(\n source: TasksetSourceRef,\n): source is TrainingSourceRef {\n return source.schemaVersion === \"openpond.trainingSource.v1\";\n}\n", "import type { TasksetCapabilityManifest } from \"./taskset-draft-core.js\";\nimport {\n isTrainingSourceRef,\n TASKSET_WORK_TOOL_NAMES,\n TasksetSchema,\n type Taskset,\n} from \"./taskset-authored-contracts.js\";\nimport { contentHash } from \"@openpond/harness\";\n\nexport type TasksetValidationIssue = {\n code: string;\n severity: \"warning\" | \"error\";\n message: string;\n path: string | null;\n};\n\nexport type TasksetValidationReport = {\n valid: boolean;\n taskset: Taskset | null;\n computedHash: string | null;\n issues: TasksetValidationIssue[];\n};\n\nexport function validateTaskset(input: unknown, options: { allowUnscoredOnlineRewardBatch?: boolean } = {}): TasksetValidationReport {\n const parsed = TasksetSchema.safeParse(input);\n if (!parsed.success) {\n return {\n valid: false,\n taskset: null,\n computedHash: null,\n issues: parsed.error.issues.map((issue) => ({\n code: \"schema_invalid\",\n severity: \"error\",\n message: issue.message,\n path: issue.path.join(\".\"),\n })),\n };\n }\n\n const taskset = parsed.data;\n const issues: TasksetValidationIssue[] = [];\n validateSourceConsent(taskset, issues);\n validateDatasetArtifact(taskset, issues);\n validateSplitIsolation(taskset, issues);\n validatePolicyBoundary(taskset, issues);\n validateGraders(taskset, issues);\n validateGraderFixtures(taskset, issues, options);\n validateLearningSignals(taskset, issues);\n validateCapabilities(taskset.capabilities, issues, taskset);\n validateWorkExecution(taskset, issues);\n\n const computedHash = tasksetContentHash(taskset);\n if (taskset.contentHash !== computedHash) {\n issues.push({\n code: \"content_hash_mismatch\",\n severity: \"error\",\n message: `Taskset contentHash is ${taskset.contentHash}, expected ${computedHash}.`,\n path: \"contentHash\",\n });\n }\n\n return {\n valid: !issues.some((issue) => issue.severity === \"error\"),\n taskset,\n computedHash,\n issues,\n };\n}\n\nfunction validateGraderFixtures(taskset: Taskset, issues: TasksetValidationIssue[], options: { allowUnscoredOnlineRewardBatch?: boolean }): void {\n // The hosted learning boundary checks the sealed batch and executable Reward\n // before opting into fresh-rollout GRPO. Human-only review has no automated\n // grader to calibrate; every mixed/automated taskset retains fixture gates.\n const humanReviewOnly = taskset.graders.length > 0 && taskset.graders.every(grader => grader.kind === \"human\");\n if (!humanReviewOnly && !taskset.graderFixtures.length && !(options.allowUnscoredOnlineRewardBatch && taskset.metadata.trainingMethod === \"grpo\" && taskset.metadata.learning)) {\n issues.push({ code: \"grader_fixtures_required\", severity: \"error\", message: \"Taskset admission requires authored grader fixtures.\", path: \"graderFixtures\" });\n }\n const taskIds = new Set(taskset.tasks.map((task) => task.id));\n const required = new Set([\"positive\", \"negative\", \"boundary\", \"adversarial\", \"prompt_injection\", \"infrastructure_failure\"]);\n for (const fixture of taskset.graderFixtures) {\n if (!taskset.datasetArtifact && !taskIds.has(fixture.taskId)) issues.push({ code: \"grader_fixture_task_missing\", severity: \"warning\", message: `Fixture ${fixture.id} references a task outside this Taskset revision (${fixture.taskId}).`, path: `graderFixtures.${fixture.id}.taskId` });\n required.delete(fixture.label);\n if (fixture.label === \"infrastructure_failure\" && !fixture.infrastructureError) issues.push({ code: \"infrastructure_fixture_error_missing\", severity: \"error\", message: `Infrastructure fixture ${fixture.id} must declare an infrastructure error.`, path: `graderFixtures.${fixture.id}.infrastructureError` });\n }\n for (const label of humanReviewOnly ? [] : required) issues.push({ code: \"grader_fixture_missing\", severity: \"warning\", message: `Taskset does not include the optional ${label} grader calibration fixture.`, path: \"graderFixtures\" });\n}\n\nfunction validateDatasetArtifact(\n taskset: Taskset,\n issues: TasksetValidationIssue[],\n): void {\n const artifact = taskset.datasetArtifact;\n if (!artifact) return;\n if (\n artifact.tasksetId !== taskset.id\n || artifact.tasksetRevision !== taskset.revision\n ) {\n issues.push({\n code: \"dataset_artifact_taskset_mismatch\",\n severity: \"error\",\n message: \"Dataset artifact identity does not match its Taskset revision.\",\n path: \"datasetArtifact\",\n });\n }\n const splitTotal = Object.values(artifact.splitCounts)\n .reduce((total, count) => total + count, 0);\n const shardTotal = artifact.shards\n .reduce((total, shard) => total + shard.rowCount, 0);\n if (splitTotal !== artifact.rowCount || shardTotal !== artifact.rowCount) {\n issues.push({\n code: \"dataset_artifact_row_count_mismatch\",\n severity: \"error\",\n message: \"Dataset artifact row, split, and shard counts do not agree.\",\n path: \"datasetArtifact.rowCount\",\n });\n }\n const shardSplitCounts = new Map<string, number>();\n for (const shard of artifact.shards) {\n shardSplitCounts.set(\n shard.split,\n (shardSplitCounts.get(shard.split) ?? 0) + shard.rowCount,\n );\n }\n for (const [split, count] of Object.entries(artifact.splitCounts)) {\n if ((shardSplitCounts.get(split) ?? 0) !== count) {\n issues.push({\n code: \"dataset_artifact_split_count_mismatch\",\n severity: \"error\",\n message: `Dataset artifact ${split} shard counts do not match its manifest.`,\n path: `datasetArtifact.splitCounts.${split}`,\n });\n }\n }\n const { contentHash: _contentHash, ...hashable } = artifact;\n if (contentHash(hashable) !== artifact.contentHash) {\n issues.push({\n code: \"dataset_artifact_hash_mismatch\",\n severity: \"error\",\n message: \"Dataset artifact manifest content hash is invalid.\",\n path: \"datasetArtifact.contentHash\",\n });\n }\n}\n\nexport function computeTasksetHash(taskset: Omit<Taskset, \"contentHash\"> | Taskset): string {\n return tasksetContentHash(taskset);\n}\n\nfunction tasksetContentHash(taskset: Omit<Taskset, \"contentHash\"> | Taskset): string {\n const { contentHash: _contentHash, status: _status, readiness: _readiness, updatedAt: _updatedAt, ...source } = taskset as Taskset;\n return contentHash(source);\n}\n\nexport function validatePortability(capabilities: TasksetCapabilityManifest): TasksetValidationIssue[] {\n const issues: TasksetValidationIssue[] = [];\n validateCapabilities(capabilities, issues, null);\n return issues;\n}\n\nfunction validateSourceConsent(taskset: Taskset, issues: TasksetValidationIssue[]): void {\n for (const [index, source] of taskset.sourceRefs.entries()) {\n if (isTrainingSourceRef(source) && source.consent.status !== \"granted\") {\n issues.push({ code: \"source_consent_missing\", severity: \"error\", message: `Source ${source.id} is not consented.`, path: `sourceRefs.${index}.consent.status` });\n }\n if (source.secretScanStatus !== \"passed\") {\n issues.push({ code: \"source_secret_scan\", severity: \"error\", message: `Source ${source.id} did not pass secret scanning.`, path: `sourceRefs.${index}.secretScanStatus` });\n }\n if (source.piiScanStatus !== \"passed\") {\n issues.push({ code: \"source_pii_scan\", severity: \"error\", message: `Source ${source.id} has unresolved PII policy.`, path: `sourceRefs.${index}.piiScanStatus` });\n }\n if (source.licensingStatus !== \"approved\") {\n issues.push({ code: \"source_license\", severity: \"error\", message: `Source ${source.id} has unresolved licensing policy.`, path: `sourceRefs.${index}.licensingStatus` });\n }\n }\n}\n\nfunction validateSplitIsolation(taskset: Taskset, issues: TasksetValidationIssue[]): void {\n const clusterSplits = new Map<string, Set<string>>();\n for (const task of taskset.tasks) {\n const splits = clusterSplits.get(task.clusterKey) ?? new Set<string>();\n splits.add(task.split);\n clusterSplits.set(task.clusterKey, splits);\n }\n for (const [cluster, splits] of clusterSplits) {\n if (splits.size > 1) {\n issues.push({ code: \"split_cluster_contamination\", severity: \"error\", message: `Source cluster ${cluster} appears in multiple splits: ${[...splits].join(\", \")}.`, path: \"tasks\" });\n }\n }\n const frozenCount = taskset.datasetArtifact\n ? taskset.datasetArtifact.splitCounts.frozen_eval ?? 0\n : taskset.tasks.filter((task) => task.split === \"frozen_eval\").length;\n if (frozenCount === 0) issues.push({ code: \"frozen_eval_missing\", severity: \"warning\", message: \"Add an independent test example before training.\", path: \"tasks\" });\n}\n\nfunction validateWorkExecution(\n taskset: Taskset,\n issues: TasksetValidationIssue[],\n): void {\n if (taskset.environment.kind !== \"work\") return;\n if (taskset.environment.entrypoint !== \"openpond-work-v1\") {\n issues.push({\n code: \"work_entrypoint_invalid\",\n severity: \"error\",\n message: \"Work Tasksets must use the openpond-work-v1 entrypoint.\",\n path: \"environment.entrypoint\",\n });\n }\n if (!taskset.environment.lifecycle.includes(\"reset\") || !taskset.environment.lifecycle.includes(\"cleanup\")) {\n issues.push({\n code: \"work_reset_lifecycle_missing\",\n severity: \"error\",\n message: \"Work attempts require reset and cleanup; filesystem state belongs to one attempt.\",\n path: \"environment.lifecycle\",\n });\n }\n const allowedTools = new Set<string>(TASKSET_WORK_TOOL_NAMES);\n const unknownTools = taskset.environment.toolNames.filter(\n (name) => !allowedTools.has(name),\n );\n if (unknownTools.length) {\n issues.push({\n code: \"work_tool_unknown\",\n severity: \"error\",\n message: `Work Taskset declares unknown tools: ${unknownTools.join(\", \")}.`,\n path: \"environment.toolNames\",\n });\n }\n if (!taskset.environment.toolNames.includes(\"work_save_output\")) {\n issues.push({\n code: \"work_output_tool_missing\",\n severity: \"error\",\n message: \"Work Tasksets must declare work_save_output.\",\n path: \"environment.toolNames\",\n });\n }\n for (const lifecycle of [\"create\", \"reset\", \"step\", \"grade\", \"cleanup\"] as const) {\n if (!taskset.environment.lifecycle.includes(lifecycle)) {\n issues.push({\n code: \"work_lifecycle_incomplete\",\n severity: \"error\",\n message: `Work Tasksets require the ${lifecycle} lifecycle stage.`,\n path: \"environment.lifecycle\",\n });\n }\n }\n\n const sourceById = new Map(\n taskset.sourceRefs.map((source) => [source.id, source]),\n );\n for (const [taskIndex, task] of taskset.tasks.entries()) {\n const assetIds = new Set<string>();\n const fileNames = new Set<string>();\n for (const [assetIndex, asset] of (task.assets ?? []).entries()) {\n const path = `tasks.${taskIndex}.assets.${assetIndex}`;\n if (assetIds.has(asset.id)) {\n issues.push({\n code: \"work_asset_duplicate_id\",\n severity: \"error\",\n message: `Task ${task.id} repeats asset id ${asset.id}.`,\n path: `${path}.id`,\n });\n }\n if (fileNames.has(asset.fileName)) {\n issues.push({\n code: \"work_asset_duplicate_name\",\n severity: \"error\",\n message: `Task ${task.id} repeats staged file name ${asset.fileName}.`,\n path: `${path}.fileName`,\n });\n }\n assetIds.add(asset.id);\n fileNames.add(asset.fileName);\n if (!task.sourceRefs.includes(asset.sourceRefId)) {\n issues.push({\n code: \"work_asset_task_source_missing\",\n severity: \"error\",\n message: `Asset ${asset.id} references source ${asset.sourceRefId} outside task ${task.id}.`,\n path: `${path}.sourceRefId`,\n });\n }\n const source = sourceById.get(asset.sourceRefId);\n if (!source) {\n issues.push({\n code: \"work_asset_source_missing\",\n severity: \"error\",\n message: `Asset ${asset.id} references missing source ${asset.sourceRefId}.`,\n path: `${path}.sourceRefId`,\n });\n } else if (\n \"sourceFileHashes\" in source\n && !source.sourceFileHashes.includes(asset.sha256)\n ) {\n issues.push({\n code: \"work_asset_hash_unregistered\",\n severity: \"error\",\n message: `Asset ${asset.id} hash is not registered by source ${asset.sourceRefId}.`,\n path: `${path}.sha256`,\n });\n }\n if (\n source\n && \"kind\" in source\n && source.kind === \"uploaded_file\"\n && !source.originalFileNames.includes(asset.fileName)\n ) {\n issues.push({\n code: \"work_asset_name_unregistered\",\n severity: \"error\",\n message: `Asset ${asset.id} file name is not registered by source ${asset.sourceRefId}.`,\n path: `${path}.fileName`,\n });\n }\n if (\n source\n && \"kind\" in source\n && source.kind === \"uploaded_file\"\n && !source.mediaTypes.includes(asset.mediaType)\n ) {\n issues.push({\n code: \"work_asset_media_type_unregistered\",\n severity: \"error\",\n message: `Asset ${asset.id} media type is not registered by source ${asset.sourceRefId}.`,\n path: `${path}.mediaType`,\n });\n }\n if (asset.split !== task.split) {\n issues.push({\n code: \"work_asset_split_mismatch\",\n severity: \"error\",\n message: `Asset ${asset.id} is assigned to ${asset.split}, not task split ${task.split}.`,\n path: `${path}.split`,\n });\n }\n }\n\n const outputPaths = new Set<string>();\n for (const [outputIndex, output] of (task.requiredOutputs ?? []).entries()) {\n const normalized = output.path.replaceAll(\"\\\\\", \"/\");\n const path = `tasks.${taskIndex}.requiredOutputs.${outputIndex}.path`;\n if (outputPaths.has(normalized)) {\n issues.push({\n code: \"work_output_duplicate_path\",\n severity: \"error\",\n message: `Task ${task.id} repeats required output path ${normalized}.`,\n path,\n });\n }\n outputPaths.add(normalized);\n }\n if ((task.requiredOutputs ?? []).length === 0) {\n issues.push({\n code: \"work_output_missing\",\n severity: \"error\",\n message: `Work task ${task.id} must declare at least one required output.`,\n path: `tasks.${taskIndex}.requiredOutputs`,\n });\n }\n }\n}\n\nfunction validatePolicyBoundary(taskset: Taskset, issues: TasksetValidationIssue[]): void {\n const visible = new Set(taskset.policy.policyVisibleFields);\n for (const field of taskset.policy.privilegedFields) {\n if (visible.has(field)) issues.push({ code: \"privileged_field_visible\", severity: \"error\", message: `Field ${field} is both policy-visible and privileged.`, path: \"policy\" });\n }\n for (const task of taskset.tasks) {\n if (task.privilegedContextRef && Object.keys(task.policyVisibleContext).includes(task.privilegedContextRef)) {\n issues.push({ code: \"privileged_context_leak\", severity: \"error\", message: `Task ${task.id} exposes its privileged context reference.`, path: `tasks.${task.id}.policyVisibleContext` });\n }\n }\n}\n\nfunction validateGraders(taskset: Taskset, issues: TasksetValidationIssue[]): void {\n const ids = new Set<string>();\n for (const grader of taskset.graders) {\n if (ids.has(grader.id)) issues.push({ code: \"grader_duplicate\", severity: \"error\", message: `Duplicate grader id ${grader.id}.`, path: \"graders\" });\n ids.add(grader.id);\n if (grader.kind === \"model_judge\" && grader.rewardEligible && grader.calibrationStatus !== \"passed\") {\n issues.push({ code: \"judge_calibration_pending\", severity: \"warning\", message: `Model judge ${grader.id} is selected for reward without a passing advisory calibration report.`, path: `graders.${grader.id}` });\n }\n if (grader.kind === \"model_judge\" && grader.calibrationStatus === \"passed\" && typeof grader.metadata.calibrationEvidenceHash !== \"string\") {\n issues.push({ code: \"judge_calibration_evidence_missing\", severity: \"warning\", message: `Model judge ${grader.id} declares calibration without an OpenPond fixture evidence hash.`, path: `graders.${grader.id}.metadata.calibrationEvidenceHash` });\n }\n if (grader.kind === \"human\" && grader.rewardEligible) {\n issues.push({ code: \"human_online_reward\", severity: \"error\", message: `Human grader ${grader.id} cannot be an online optimizer reward.`, path: `graders.${grader.id}` });\n }\n }\n}\n\nfunction validateLearningSignals(taskset: Taskset, issues: TasksetValidationIssue[]): void {\n const taskIds = new Set(taskset.tasks.map((task) => task.id));\n const sourceIds = new Set(taskset.sourceRefs.map((source) => source.id));\n for (const signal of [\n ...taskset.learningSignals.demonstrations,\n ...taskset.learningSignals.preferences,\n ...taskset.learningSignals.corrections,\n ...taskset.learningSignals.feedback,\n ...taskset.learningSignals.rewards,\n ...taskset.learningSignals.labels,\n ]) {\n if (signal.taskId && !taskset.datasetArtifact && !taskIds.has(signal.taskId)) {\n issues.push({ code: \"learning_signal_task_missing\", severity: \"error\", message: `Signal ${signal.id} references missing task ${signal.taskId}.`, path: `learningSignals.${signal.kind}.${signal.id}.taskId` });\n }\n if (signal.sourceRefs.some((sourceId) => !sourceIds.has(sourceId))) {\n issues.push({ code: \"learning_signal_source_missing\", severity: \"error\", message: `Signal ${signal.id} references a source outside the Dataset.`, path: `learningSignals.${signal.kind}.${signal.id}.sourceRefs` });\n }\n }\n for (const preference of taskset.learningSignals.preferences) {\n if (preference.chosen.trim() === preference.rejected.trim()) {\n issues.push({ code: \"preference_pair_identical\", severity: \"error\", message: `Preference ${preference.id} has identical chosen and rejected responses.`, path: `learningSignals.preferences.${preference.id}` });\n }\n }\n for (const reward of taskset.learningSignals.rewards) {\n if (!reward.executable) {\n issues.push({ code: \"reward_not_executable\", severity: \"warning\", message: `Reward ${reward.id} is a reviewed specification but has no executable verifier yet.`, path: `learningSignals.rewards.${reward.id}.executable` });\n }\n }\n}\n\nfunction validateCapabilities(\n capabilities: TasksetCapabilityManifest,\n issues: TasksetValidationIssue[],\n taskset: Taskset | null,\n): void {\n if (!capabilities.exportable && capabilities.portabilityBlockers.length === 0) {\n issues.push({ code: \"portability_reason_missing\", severity: \"error\", message: \"Non-exportable Tasksets must declare a portability blocker.\", path: \"capabilities.portabilityBlockers\" });\n }\n if (capabilities.compatibleMethods.includes(\"grpo\") && !capabilities.rewardKinds.some((kind) => kind === \"exact\" || kind === \"deterministic\" || kind === \"model_judge\")) {\n issues.push({ code: \"grpo_reward_missing\", severity: \"error\", message: \"GRPO compatibility requires a scalar reward kind.\", path: \"capabilities.rewardKinds\" });\n }\n if (taskset?.capabilities.compatibleMethods.includes(\"sft\") && taskset.learningSignals.demonstrations.length === 0) {\n issues.push({ code: \"sft_demonstrations_missing\", severity: \"error\", message: \"SFT compatibility requires approved demonstrations.\", path: \"learningSignals.demonstrations\" });\n }\n if (taskset?.capabilities.compatibleMethods.includes(\"dpo\") && taskset.learningSignals.preferences.length === 0) {\n issues.push({ code: \"dpo_preferences_missing\", severity: \"error\", message: \"DPO compatibility requires chosen and rejected response pairs.\", path: \"learningSignals.preferences\" });\n }\n if (\n taskset\n && (taskset.capabilities.compatibleMethods.includes(\"grpo\") || taskset.capabilities.compatibleMethods.includes(\"ppo\"))\n && !taskset.learningSignals.rewards.some((reward) => reward.executable)\n && !taskset.graders.some(grader => grader.rewardEligible && grader.weight > 0 && grader.kind !== \"human\")\n ) {\n issues.push({ code: \"online_reward_not_executable\", severity: \"error\", message: \"GRPO/PPO compatibility requires an executable scalar reward.\", path: \"learningSignals.rewards\" });\n }\n}\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,SAAS;AAGlB,SAAS,iCAAiC;AAiB1C,IAAM,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG;AACjD,IAAM,kBAAkB,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC;AAC/C,IAAM,aAAa,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG;AACnD,IAAM,iBAAiB,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;AACnE,IAAM,mBAAmB,SAAS,SAAS;AAEpC,IAAM,0BAA0B;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,IAAM,sBAAsB,EAAE,KAAK;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAEM,IAAM,2BAA2B,EAAE,KAAK;AAAA,EAC7C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,4BAA4B,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,GAAO;AAExD,IAAM,kCAAkC,EAAE,mBAAmB,QAAQ;AAAA,EAC1E,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,gBAAgB;AAAA,IAChC,UAAU;AAAA,IACV,UAAU,EAAE,MAAM,EAAE,OAAO;AAAA,MACzB,IAAI;AAAA,MACJ,QAAQ;AAAA,MACR,UAAU;AAAA,IACZ,CAAC,CAAC,EAAE,IAAI,GAAK,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC3B,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,aAAa;AAAA,IAC7B,YAAY;AAAA,IACZ,OAAO,EAAE,MAAM,EAAE,OAAO;AAAA,MACtB,IAAI;AAAA,MACJ,QAAQ;AAAA,MACR,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,WAAW;AAAA,IACb,CAAC,CAAC,EAAE,IAAI,GAAK,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC3B,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,mBAAmB;AAAA,IACnC,MAAM;AAAA,IACN,OAAO,EAAE,MAAM,EAAE,OAAO;AAAA,MACtB,IAAI;AAAA,MACJ,QAAQ,EAAE,OAAO,EAAE,OAAO;AAAA,MAC1B,WAAW;AAAA,IACb,CAAC,CAAC,EAAE,IAAI,GAAK,EAAE,QAAQ,CAAC,CAAC;AAAA,IACzB,iBAAiB,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,EAChD,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,QAAQ;AAAA,IACxB,MAAM;AAAA,IACN,UAAU,EAAE,MAAM,EAAE,OAAO;AAAA,MACzB,IAAI;AAAA,MACJ,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,GAAG;AAAA,MAChC,aAAa;AAAA,IACf,CAAC,CAAC,EAAE,IAAI,GAAK,EAAE,QAAQ,CAAC,CAAC;AAAA,IACzB,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,EACnB,CAAC;AACH,CAAC;AAEM,IAAM,0BAA0B,EAAE,OAAO;AAAA,EAC9C,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAK;AAAA,EACxC,MAAM,EAAE,KAAK,CAAC,eAAe,YAAY,SAAS,CAAC;AAAA,EACnD,SAAS,EAAE,OAAO,EAAE,IAAI,IAAO;AACjC,CAAC;AAEM,IAAM,mCAAmC,EAAE,OAAO;AAAA,EACvD,eAAe,EAAE,KAAK,CAAC,OAAO,OAAO,QAAQ,OAAO,QAAQ,QAAQ,MAAM,CAAC;AAAA,EAC3E,WAAW,EAAE,OAAO;AAAA,IAClB,QAAQ,EAAE,QAAQ,KAAK;AAAA,IACvB,SAAS,EAAE,QAAQ,sBAAsB;AAAA,IACzC,mBAAmB,EAAE,MAAM,QAAQ,EAAE,IAAI,CAAC,EAAE,IAAI,GAAO;AAAA,IACvD,aAAa,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAK,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG;AAAA,EAC1E,CAAC,EAAE,SAAS;AACd,CAAC;AAEM,IAAM,0CAA0C,EAAE,KAAK;AAAA,EAC5D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAEM,IAAM,gCAAgC,EAAE,OAAO;AAAA,EACpD,QAAQ,EAAE,KAAK,CAAC,OAAO,OAAO,QAAQ,KAAK,CAAC;AAAA,EAC5C,QAAQ,EAAE,KAAK,CAAC,eAAe,cAAc,oBAAoB,CAAC;AAAA,EAClE,aAAa,EAAE,MAAM,uCAAuC,EAAE,QAAQ,CAAC,CAAC;AAAA,EACxE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAK,CAAC,EAAE,QAAQ,CAAC,CAAC;AAClE,CAAC;AAEM,IAAM,gCAAgC,EAAE,OAAO;AAAA,EACpD,MAAM;AAAA,EACN,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAK;AAAA,EAC3C,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,GAAK,EAAE,SAAS;AAC9C,CAAC;AAEM,IAAM,+BAA+B,EAAE,OAAO;AAAA,EACnD,eAAe,EAAE,QAAQ,8BAA8B;AAAA,EACvD,WAAW;AAAA,EACX,aAAa;AAAA,EACb,OAAO,EAAE,QAAQ;AAAA,EACjB,mBAAmB,EAAE,KAAK,CAAC,QAAQ,aAAa,OAAO,OAAO,QAAQ,OAAO,QAAQ,OAAO,QAAQ,MAAM,CAAC;AAAA,EAC3G,cAAc,iCAAiC,SAAS,EAAE,QAAQ,IAAI;AAAA,EACtE,iBAAiB,EAAE,MAAM,6BAA6B,EAAE,QAAQ,CAAC,CAAC;AAAA,EAClE,8BAA8B,EAAE;AAAA,IAC9B,EAAE,KAAK,CAAC,UAAU,UAAU,gBAAgB,CAAC;AAAA,EAC/C;AAAA,EACA,UAAU,EAAE,MAAM,6BAA6B,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC3D,YAAY,EAAE,MAAM,6BAA6B,EAAE,QAAQ,CAAC,CAAC;AAAA,EAC7D,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAK,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,EACjE,aAAa;AACf,CAAC;AAEM,IAAM,wBAAwB,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,GAAG,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAK,GAAG,WAAW,gBAAgB,CAAC;AAEzJ,IAAM,4BAA4B,EAAE,OAAO;AAAA,EAChD,eAAe,EAAE,QAAQ,qCAAqC;AAAA,EAC9D,OAAO,mBAAmB,SAAS;AAAA,EACnC,aAAa;AAAA,EACb,WAAW;AAAA,EACX,uBAAuB,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG;AAAA,EACvD,aAAa,yBAAyB,QAAQ,gBAAgB;AAAA,EAC9D,oBAAoB,gCAAgC,SAAS,EAAE,QAAQ,IAAI;AAAA,EAC3E,gBAAgB,EAAE,MAAM,UAAU,EAAE,IAAI,GAAO;AAAA,EAC/C,mBAAmB,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG;AAAA,EACnD,cAAc,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG,EAAE,SAAS;AAAA,EACzD,eAAe,EAAE,MAAM,qBAAqB,EAAE,IAAI,GAAK;AAAA,EACvD,WAAW;AACb,CAAC;AAEM,IAAM,gBAAgB,EAAE,OAAO;AAAA,EACpC,eAAe,EAAE,QAAQ,qBAAqB;AAAA,EAC9C,IAAI;AAAA,EACJ,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC;AAAA,EAC/C,WAAW;AAAA,EACX,gBAAgB,0BAA0B,SAAS,EAAE,SAAS;AAAA,EAC9D,oBAAoB,iBAAiB,QAAQ,IAAI;AAAA,EACjD,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG;AAAA;AAAA;AAAA,EAGtC,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,GAAM;AAAA,EACvC,SAAS,qBAAqB,QAAQ,SAAS;AAAA,EAC/C,WAAW,8BAA8B,SAAS,EAAE,QAAQ,IAAI;AAAA,EAChE,sBAAsB,yCAAyC,SAAS,EAAE,QAAQ,IAAI;AAAA,EACtF,QAAQ;AAAA,EACR,YAAY,EAAE,MAAM,sBAAsB,EAAE,IAAI,CAAC,EAAE,IAAI,GAAO;AAAA,EAC9D,iBAAiB,8BAA8B,SAAS,EAAE,SAAS;AAAA,EACnE,QAAQ;AAAA,EACR,aAAa;AAAA,EACb,cAAc;AAAA,EACd,SAAS,0BAA0B,SAAS;AAAA,EAC5C,OAAO,EAAE,MAAM,oBAAoB,EAAE,IAAI,GAAS;AAAA,EAClD,SAAS,EAAE,MAAM,gBAAgB,EAAE,IAAI,CAAC,EAAE,IAAI,GAAK;AAAA;AAAA;AAAA,EAGnD,gBAAgB,EAAE,MAAM,mBAAmB,EAAE,IAAI,GAAO;AAAA,EACxD,iBAAiB;AAAA,EACjB,qBAAqB;AAAA,EACrB,WAAW,6BAA6B,SAAS;AAAA,EACjD,aAAa;AAAA,EACb,WAAW;AAAA,EACX,WAAW;AAAA,EACX,UAAU;AACZ,CAAC,EAAE,YAAY,CAAC,SAAS,YAAY;AACnC,MAAI,QAAQ,YAAY,eAAe,CAAC,QAAQ,WAAW;AACzD,YAAQ,SAAS;AAAA,MACf,MAAM;AAAA,MACN,SAAS;AAAA,MACT,MAAM,CAAC,WAAW;AAAA,IACpB,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,YAAY,eAAe,QAAQ,WAAW;AACxD,YAAQ,SAAS;AAAA,MACf,MAAM;AAAA,MACN,SAAS;AAAA,MACT,MAAM,CAAC,WAAW;AAAA,IACpB,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,mBAAmB,QAAQ,MAAM,SAAS,GAAG;AACvD,YAAQ,SAAS;AAAA,MACf,MAAM;AAAA,MACN,SACE;AAAA,MACF,MAAM,CAAC,OAAO;AAAA,IAChB,CAAC;AAAA,EACH;AACA,MAAI,CAAC,QAAQ,mBAAmB,QAAQ,MAAM,WAAW,GAAG;AAC1D,YAAQ,SAAS;AAAA,MACf,MAAM;AAAA,MACN,SAAS;AAAA,MACT,MAAM,CAAC,OAAO;AAAA,IAChB,CAAC;AAAA,EACH;AACF,CAAC;AAaM,SAAS,oBACd,QAC6B;AAC7B,SAAO,OAAO,kBAAkB;AAClC;;;AClPO,SAAS,gBAAgB,OAAgB,UAAwD,CAAC,GAA4B;AACnI,QAAM,SAAS,cAAc,UAAU,KAAK;AAC5C,MAAI,CAAC,OAAO,SAAS;AACnB,WAAO;AAAA,MACL,OAAO;AAAA,MACP,SAAS;AAAA,MACT,cAAc;AAAA,MACd,QAAQ,OAAO,MAAM,OAAO,IAAI,CAAC,WAAW;AAAA,QAC1C,MAAM;AAAA,QACN,UAAU;AAAA,QACV,SAAS,MAAM;AAAA,QACf,MAAM,MAAM,KAAK,KAAK,GAAG;AAAA,MAC3B,EAAE;AAAA,IACJ;AAAA,EACF;AAEA,QAAM,UAAU,OAAO;AACvB,QAAM,SAAmC,CAAC;AAC1C,wBAAsB,SAAS,MAAM;AACrC,0BAAwB,SAAS,MAAM;AACvC,yBAAuB,SAAS,MAAM;AACtC,yBAAuB,SAAS,MAAM;AACtC,kBAAgB,SAAS,MAAM;AAC/B,yBAAuB,SAAS,QAAQ,OAAO;AAC/C,0BAAwB,SAAS,MAAM;AACvC,uBAAqB,QAAQ,cAAc,QAAQ,OAAO;AAC1D,wBAAsB,SAAS,MAAM;AAErC,QAAM,eAAe,mBAAmB,OAAO;AAC/C,MAAI,QAAQ,gBAAgB,cAAc;AACxC,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,SAAS,0BAA0B,QAAQ,WAAW,cAAc,YAAY;AAAA,MAChF,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,OAAO,CAAC,OAAO,KAAK,CAAC,UAAU,MAAM,aAAa,OAAO;AAAA,IACzD;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,uBAAuB,SAAkB,QAAkC,SAA6D;AAI/I,QAAM,kBAAkB,QAAQ,QAAQ,SAAS,KAAK,QAAQ,QAAQ,MAAM,YAAU,OAAO,SAAS,OAAO;AAC7G,MAAI,CAAC,mBAAmB,CAAC,QAAQ,eAAe,UAAU,EAAE,QAAQ,kCAAkC,QAAQ,SAAS,mBAAmB,UAAU,QAAQ,SAAS,WAAW;AAC9K,WAAO,KAAK,EAAE,MAAM,4BAA4B,UAAU,SAAS,SAAS,wDAAwD,MAAM,iBAAiB,CAAC;AAAA,EAC9J;AACA,QAAM,UAAU,IAAI,IAAI,QAAQ,MAAM,IAAI,CAAC,SAAS,KAAK,EAAE,CAAC;AAC5D,QAAM,WAAW,oBAAI,IAAI,CAAC,YAAY,YAAY,YAAY,eAAe,oBAAoB,wBAAwB,CAAC;AAC1H,aAAW,WAAW,QAAQ,gBAAgB;AAC5C,QAAI,CAAC,QAAQ,mBAAmB,CAAC,QAAQ,IAAI,QAAQ,MAAM,EAAG,QAAO,KAAK,EAAE,MAAM,+BAA+B,UAAU,WAAW,SAAS,WAAW,QAAQ,EAAE,qDAAqD,QAAQ,MAAM,MAAM,MAAM,kBAAkB,QAAQ,EAAE,UAAU,CAAC;AAC1R,aAAS,OAAO,QAAQ,KAAK;AAC7B,QAAI,QAAQ,UAAU,4BAA4B,CAAC,QAAQ,oBAAqB,QAAO,KAAK,EAAE,MAAM,wCAAwC,UAAU,SAAS,SAAS,0BAA0B,QAAQ,EAAE,0CAA0C,MAAM,kBAAkB,QAAQ,EAAE,uBAAuB,CAAC;AAAA,EAClT;AACA,aAAW,SAAS,kBAAkB,CAAC,IAAI,SAAU,QAAO,KAAK,EAAE,MAAM,0BAA0B,UAAU,WAAW,SAAS,yCAAyC,KAAK,gCAAgC,MAAM,iBAAiB,CAAC;AACzO;AAEA,SAAS,wBACP,SACA,QACM;AACN,QAAM,WAAW,QAAQ;AACzB,MAAI,CAAC,SAAU;AACf,MACE,SAAS,cAAc,QAAQ,MAC5B,SAAS,oBAAoB,QAAQ,UACxC;AACA,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,SAAS;AAAA,MACT,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACA,QAAM,aAAa,OAAO,OAAO,SAAS,WAAW,EAClD,OAAO,CAAC,OAAO,UAAU,QAAQ,OAAO,CAAC;AAC5C,QAAM,aAAa,SAAS,OACzB,OAAO,CAAC,OAAO,UAAU,QAAQ,MAAM,UAAU,CAAC;AACrD,MAAI,eAAe,SAAS,YAAY,eAAe,SAAS,UAAU;AACxE,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,SAAS;AAAA,MACT,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACA,QAAM,mBAAmB,oBAAI,IAAoB;AACjD,aAAW,SAAS,SAAS,QAAQ;AACnC,qBAAiB;AAAA,MACf,MAAM;AAAA,OACL,iBAAiB,IAAI,MAAM,KAAK,KAAK,KAAK,MAAM;AAAA,IACnD;AAAA,EACF;AACA,aAAW,CAAC,OAAO,KAAK,KAAK,OAAO,QAAQ,SAAS,WAAW,GAAG;AACjE,SAAK,iBAAiB,IAAI,KAAK,KAAK,OAAO,OAAO;AAChD,aAAO,KAAK;AAAA,QACV,MAAM;AAAA,QACN,UAAU;AAAA,QACV,SAAS,oBAAoB,KAAK;AAAA,QAClC,MAAM,+BAA+B,KAAK;AAAA,MAC5C,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,EAAE,aAAa,cAAc,GAAG,SAAS,IAAI;AACnD,MAAI,YAAY,QAAQ,MAAM,SAAS,aAAa;AAClD,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,SAAS;AAAA,MACT,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACF;AAEO,SAAS,mBAAmB,SAAyD;AAC1F,SAAO,mBAAmB,OAAO;AACnC;AAEA,SAAS,mBAAmB,SAAyD;AACnF,QAAM,EAAE,aAAa,cAAc,QAAQ,SAAS,WAAW,YAAY,WAAW,YAAY,GAAG,OAAO,IAAI;AAChH,SAAO,YAAY,MAAM;AAC3B;AAEO,SAAS,oBAAoB,cAAmE;AACrG,QAAM,SAAmC,CAAC;AAC1C,uBAAqB,cAAc,QAAQ,IAAI;AAC/C,SAAO;AACT;AAEA,SAAS,sBAAsB,SAAkB,QAAwC;AACvF,aAAW,CAAC,OAAO,MAAM,KAAK,QAAQ,WAAW,QAAQ,GAAG;AAC1D,QAAI,oBAAoB,MAAM,KAAK,OAAO,QAAQ,WAAW,WAAW;AACtE,aAAO,KAAK,EAAE,MAAM,0BAA0B,UAAU,SAAS,SAAS,UAAU,OAAO,EAAE,sBAAsB,MAAM,cAAc,KAAK,kBAAkB,CAAC;AAAA,IACjK;AACA,QAAI,OAAO,qBAAqB,UAAU;AACxC,aAAO,KAAK,EAAE,MAAM,sBAAsB,UAAU,SAAS,SAAS,UAAU,OAAO,EAAE,kCAAkC,MAAM,cAAc,KAAK,oBAAoB,CAAC;AAAA,IAC3K;AACA,QAAI,OAAO,kBAAkB,UAAU;AACrC,aAAO,KAAK,EAAE,MAAM,mBAAmB,UAAU,SAAS,SAAS,UAAU,OAAO,EAAE,+BAA+B,MAAM,cAAc,KAAK,iBAAiB,CAAC;AAAA,IAClK;AACA,QAAI,OAAO,oBAAoB,YAAY;AACzC,aAAO,KAAK,EAAE,MAAM,kBAAkB,UAAU,SAAS,SAAS,UAAU,OAAO,EAAE,qCAAqC,MAAM,cAAc,KAAK,mBAAmB,CAAC;AAAA,IACzK;AAAA,EACF;AACF;AAEA,SAAS,uBAAuB,SAAkB,QAAwC;AACxF,QAAM,gBAAgB,oBAAI,IAAyB;AACnD,aAAW,QAAQ,QAAQ,OAAO;AAChC,UAAM,SAAS,cAAc,IAAI,KAAK,UAAU,KAAK,oBAAI,IAAY;AACrE,WAAO,IAAI,KAAK,KAAK;AACrB,kBAAc,IAAI,KAAK,YAAY,MAAM;AAAA,EAC3C;AACA,aAAW,CAAC,SAAS,MAAM,KAAK,eAAe;AAC7C,QAAI,OAAO,OAAO,GAAG;AACnB,aAAO,KAAK,EAAE,MAAM,+BAA+B,UAAU,SAAS,SAAS,kBAAkB,OAAO,gCAAgC,CAAC,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,KAAK,MAAM,QAAQ,CAAC;AAAA,IACpL;AAAA,EACF;AACA,QAAM,cAAc,QAAQ,kBACxB,QAAQ,gBAAgB,YAAY,eAAe,IACnD,QAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,UAAU,aAAa,EAAE;AACjE,MAAI,gBAAgB,EAAG,QAAO,KAAK,EAAE,MAAM,uBAAuB,UAAU,WAAW,SAAS,oDAAoD,MAAM,QAAQ,CAAC;AACrK;AAEA,SAAS,sBACP,SACA,QACM;AACN,MAAI,QAAQ,YAAY,SAAS,OAAQ;AACzC,MAAI,QAAQ,YAAY,eAAe,oBAAoB;AACzD,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,SAAS;AAAA,MACT,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACA,MAAI,CAAC,QAAQ,YAAY,UAAU,SAAS,OAAO,KAAK,CAAC,QAAQ,YAAY,UAAU,SAAS,SAAS,GAAG;AAC1G,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,SAAS;AAAA,MACT,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACA,QAAM,eAAe,IAAI,IAAY,uBAAuB;AAC5D,QAAM,eAAe,QAAQ,YAAY,UAAU;AAAA,IACjD,CAAC,SAAS,CAAC,aAAa,IAAI,IAAI;AAAA,EAClC;AACA,MAAI,aAAa,QAAQ;AACvB,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,SAAS,wCAAwC,aAAa,KAAK,IAAI,CAAC;AAAA,MACxE,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACA,MAAI,CAAC,QAAQ,YAAY,UAAU,SAAS,kBAAkB,GAAG;AAC/D,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,SAAS;AAAA,MACT,MAAM;AAAA,IACR,CAAC;AAAA,EACH;AACA,aAAW,aAAa,CAAC,UAAU,SAAS,QAAQ,SAAS,SAAS,GAAY;AAChF,QAAI,CAAC,QAAQ,YAAY,UAAU,SAAS,SAAS,GAAG;AACtD,aAAO,KAAK;AAAA,QACV,MAAM;AAAA,QACN,UAAU;AAAA,QACV,SAAS,6BAA6B,SAAS;AAAA,QAC/C,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,aAAa,IAAI;AAAA,IACrB,QAAQ,WAAW,IAAI,CAAC,WAAW,CAAC,OAAO,IAAI,MAAM,CAAC;AAAA,EACxD;AACA,aAAW,CAAC,WAAW,IAAI,KAAK,QAAQ,MAAM,QAAQ,GAAG;AACvD,UAAM,WAAW,oBAAI,IAAY;AACjC,UAAM,YAAY,oBAAI,IAAY;AAClC,eAAW,CAAC,YAAY,KAAK,MAAM,KAAK,UAAU,CAAC,GAAG,QAAQ,GAAG;AAC/D,YAAM,OAAO,SAAS,SAAS,WAAW,UAAU;AACpD,UAAI,SAAS,IAAI,MAAM,EAAE,GAAG;AAC1B,eAAO,KAAK;AAAA,UACV,MAAM;AAAA,UACN,UAAU;AAAA,UACV,SAAS,QAAQ,KAAK,EAAE,qBAAqB,MAAM,EAAE;AAAA,UACrD,MAAM,GAAG,IAAI;AAAA,QACf,CAAC;AAAA,MACH;AACA,UAAI,UAAU,IAAI,MAAM,QAAQ,GAAG;AACjC,eAAO,KAAK;AAAA,UACV,MAAM;AAAA,UACN,UAAU;AAAA,UACV,SAAS,QAAQ,KAAK,EAAE,6BAA6B,MAAM,QAAQ;AAAA,UACnE,MAAM,GAAG,IAAI;AAAA,QACf,CAAC;AAAA,MACH;AACA,eAAS,IAAI,MAAM,EAAE;AACrB,gBAAU,IAAI,MAAM,QAAQ;AAC5B,UAAI,CAAC,KAAK,WAAW,SAAS,MAAM,WAAW,GAAG;AAChD,eAAO,KAAK;AAAA,UACV,MAAM;AAAA,UACN,UAAU;AAAA,UACV,SAAS,SAAS,MAAM,EAAE,sBAAsB,MAAM,WAAW,iBAAiB,KAAK,EAAE;AAAA,UACzF,MAAM,GAAG,IAAI;AAAA,QACf,CAAC;AAAA,MACH;AACA,YAAM,SAAS,WAAW,IAAI,MAAM,WAAW;AAC/C,UAAI,CAAC,QAAQ;AACX,eAAO,KAAK;AAAA,UACV,MAAM;AAAA,UACN,UAAU;AAAA,UACV,SAAS,SAAS,MAAM,EAAE,8BAA8B,MAAM,WAAW;AAAA,UACzE,MAAM,GAAG,IAAI;AAAA,QACf,CAAC;AAAA,MACH,WACE,sBAAsB,UACnB,CAAC,OAAO,iBAAiB,SAAS,MAAM,MAAM,GACjD;AACA,eAAO,KAAK;AAAA,UACV,MAAM;AAAA,UACN,UAAU;AAAA,UACV,SAAS,SAAS,MAAM,EAAE,qCAAqC,MAAM,WAAW;AAAA,UAChF,MAAM,GAAG,IAAI;AAAA,QACf,CAAC;AAAA,MACH;AACA,UACE,UACG,UAAU,UACV,OAAO,SAAS,mBAChB,CAAC,OAAO,kBAAkB,SAAS,MAAM,QAAQ,GACpD;AACA,eAAO,KAAK;AAAA,UACV,MAAM;AAAA,UACN,UAAU;AAAA,UACV,SAAS,SAAS,MAAM,EAAE,0CAA0C,MAAM,WAAW;AAAA,UACrF,MAAM,GAAG,IAAI;AAAA,QACf,CAAC;AAAA,MACH;AACA,UACE,UACG,UAAU,UACV,OAAO,SAAS,mBAChB,CAAC,OAAO,WAAW,SAAS,MAAM,SAAS,GAC9C;AACA,eAAO,KAAK;AAAA,UACV,MAAM;AAAA,UACN,UAAU;AAAA,UACV,SAAS,SAAS,MAAM,EAAE,2CAA2C,MAAM,WAAW;AAAA,UACtF,MAAM,GAAG,IAAI;AAAA,QACf,CAAC;AAAA,MACH;AACA,UAAI,MAAM,UAAU,KAAK,OAAO;AAC9B,eAAO,KAAK;AAAA,UACV,MAAM;AAAA,UACN,UAAU;AAAA,UACV,SAAS,SAAS,MAAM,EAAE,mBAAmB,MAAM,KAAK,oBAAoB,KAAK,KAAK;AAAA,UACtF,MAAM,GAAG,IAAI;AAAA,QACf,CAAC;AAAA,MACH;AAAA,IACF;AAEA,UAAM,cAAc,oBAAI,IAAY;AACpC,eAAW,CAAC,aAAa,MAAM,MAAM,KAAK,mBAAmB,CAAC,GAAG,QAAQ,GAAG;AAC1E,YAAM,aAAa,OAAO,KAAK,WAAW,MAAM,GAAG;AACnD,YAAM,OAAO,SAAS,SAAS,oBAAoB,WAAW;AAC9D,UAAI,YAAY,IAAI,UAAU,GAAG;AAC/B,eAAO,KAAK;AAAA,UACV,MAAM;AAAA,UACN,UAAU;AAAA,UACV,SAAS,QAAQ,KAAK,EAAE,iCAAiC,UAAU;AAAA,UACnE;AAAA,QACF,CAAC;AAAA,MACH;AACA,kBAAY,IAAI,UAAU;AAAA,IAC5B;AACA,SAAK,KAAK,mBAAmB,CAAC,GAAG,WAAW,GAAG;AAC7C,aAAO,KAAK;AAAA,QACV,MAAM;AAAA,QACN,UAAU;AAAA,QACV,SAAS,aAAa,KAAK,EAAE;AAAA,QAC7B,MAAM,SAAS,SAAS;AAAA,MAC1B,CAAC;AAAA,IACH;AAAA,EACF;AACF;AAEA,SAAS,uBAAuB,SAAkB,QAAwC;AACxF,QAAM,UAAU,IAAI,IAAI,QAAQ,OAAO,mBAAmB;AAC1D,aAAW,SAAS,QAAQ,OAAO,kBAAkB;AACnD,QAAI,QAAQ,IAAI,KAAK,EAAG,QAAO,KAAK,EAAE,MAAM,4BAA4B,UAAU,SAAS,SAAS,SAAS,KAAK,2CAA2C,MAAM,SAAS,CAAC;AAAA,EAC/K;AACA,aAAW,QAAQ,QAAQ,OAAO;AAChC,QAAI,KAAK,wBAAwB,OAAO,KAAK,KAAK,oBAAoB,EAAE,SAAS,KAAK,oBAAoB,GAAG;AAC3G,aAAO,KAAK,EAAE,MAAM,2BAA2B,UAAU,SAAS,SAAS,QAAQ,KAAK,EAAE,8CAA8C,MAAM,SAAS,KAAK,EAAE,wBAAwB,CAAC;AAAA,IACzL;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,SAAkB,QAAwC;AACjF,QAAM,MAAM,oBAAI,IAAY;AAC5B,aAAW,UAAU,QAAQ,SAAS;AACpC,QAAI,IAAI,IAAI,OAAO,EAAE,EAAG,QAAO,KAAK,EAAE,MAAM,oBAAoB,UAAU,SAAS,SAAS,uBAAuB,OAAO,EAAE,KAAK,MAAM,UAAU,CAAC;AAClJ,QAAI,IAAI,OAAO,EAAE;AACjB,QAAI,OAAO,SAAS,iBAAiB,OAAO,kBAAkB,OAAO,sBAAsB,UAAU;AACnG,aAAO,KAAK,EAAE,MAAM,6BAA6B,UAAU,WAAW,SAAS,eAAe,OAAO,EAAE,0EAA0E,MAAM,WAAW,OAAO,EAAE,GAAG,CAAC;AAAA,IACjN;AACA,QAAI,OAAO,SAAS,iBAAiB,OAAO,sBAAsB,YAAY,OAAO,OAAO,SAAS,4BAA4B,UAAU;AACzI,aAAO,KAAK,EAAE,MAAM,sCAAsC,UAAU,WAAW,SAAS,eAAe,OAAO,EAAE,oEAAoE,MAAM,WAAW,OAAO,EAAE,oCAAoC,CAAC;AAAA,IACrP;AACA,QAAI,OAAO,SAAS,WAAW,OAAO,gBAAgB;AACpD,aAAO,KAAK,EAAE,MAAM,uBAAuB,UAAU,SAAS,SAAS,gBAAgB,OAAO,EAAE,0CAA0C,MAAM,WAAW,OAAO,EAAE,GAAG,CAAC;AAAA,IAC1K;AAAA,EACF;AACF;AAEA,SAAS,wBAAwB,SAAkB,QAAwC;AACzF,QAAM,UAAU,IAAI,IAAI,QAAQ,MAAM,IAAI,CAAC,SAAS,KAAK,EAAE,CAAC;AAC5D,QAAM,YAAY,IAAI,IAAI,QAAQ,WAAW,IAAI,CAAC,WAAW,OAAO,EAAE,CAAC;AACvE,aAAW,UAAU;AAAA,IACnB,GAAG,QAAQ,gBAAgB;AAAA,IAC3B,GAAG,QAAQ,gBAAgB;AAAA,IAC3B,GAAG,QAAQ,gBAAgB;AAAA,IAC3B,GAAG,QAAQ,gBAAgB;AAAA,IAC3B,GAAG,QAAQ,gBAAgB;AAAA,IAC3B,GAAG,QAAQ,gBAAgB;AAAA,EAC7B,GAAG;AACD,QAAI,OAAO,UAAU,CAAC,QAAQ,mBAAmB,CAAC,QAAQ,IAAI,OAAO,MAAM,GAAG;AAC5E,aAAO,KAAK,EAAE,MAAM,gCAAgC,UAAU,SAAS,SAAS,UAAU,OAAO,EAAE,4BAA4B,OAAO,MAAM,KAAK,MAAM,mBAAmB,OAAO,IAAI,IAAI,OAAO,EAAE,UAAU,CAAC;AAAA,IAC/M;AACA,QAAI,OAAO,WAAW,KAAK,CAAC,aAAa,CAAC,UAAU,IAAI,QAAQ,CAAC,GAAG;AAClE,aAAO,KAAK,EAAE,MAAM,kCAAkC,UAAU,SAAS,SAAS,UAAU,OAAO,EAAE,6CAA6C,MAAM,mBAAmB,OAAO,IAAI,IAAI,OAAO,EAAE,cAAc,CAAC;AAAA,IACpN;AAAA,EACF;AACA,aAAW,cAAc,QAAQ,gBAAgB,aAAa;AAC5D,QAAI,WAAW,OAAO,KAAK,MAAM,WAAW,SAAS,KAAK,GAAG;AAC3D,aAAO,KAAK,EAAE,MAAM,6BAA6B,UAAU,SAAS,SAAS,cAAc,WAAW,EAAE,iDAAiD,MAAM,+BAA+B,WAAW,EAAE,GAAG,CAAC;AAAA,IACjN;AAAA,EACF;AACA,aAAW,UAAU,QAAQ,gBAAgB,SAAS;AACpD,QAAI,CAAC,OAAO,YAAY;AACtB,aAAO,KAAK,EAAE,MAAM,yBAAyB,UAAU,WAAW,SAAS,UAAU,OAAO,EAAE,oEAAoE,MAAM,2BAA2B,OAAO,EAAE,cAAc,CAAC;AAAA,IAC7N;AAAA,EACF;AACF;AAEA,SAAS,qBACP,cACA,QACA,SACM;AACN,MAAI,CAAC,aAAa,cAAc,aAAa,oBAAoB,WAAW,GAAG;AAC7E,WAAO,KAAK,EAAE,MAAM,8BAA8B,UAAU,SAAS,SAAS,+DAA+D,MAAM,mCAAmC,CAAC;AAAA,EACzL;AACA,MAAI,aAAa,kBAAkB,SAAS,MAAM,KAAK,CAAC,aAAa,YAAY,KAAK,CAAC,SAAS,SAAS,WAAW,SAAS,mBAAmB,SAAS,aAAa,GAAG;AACvK,WAAO,KAAK,EAAE,MAAM,uBAAuB,UAAU,SAAS,SAAS,qDAAqD,MAAM,2BAA2B,CAAC;AAAA,EAChK;AACA,MAAI,SAAS,aAAa,kBAAkB,SAAS,KAAK,KAAK,QAAQ,gBAAgB,eAAe,WAAW,GAAG;AAClH,WAAO,KAAK,EAAE,MAAM,8BAA8B,UAAU,SAAS,SAAS,uDAAuD,MAAM,iCAAiC,CAAC;AAAA,EAC/K;AACA,MAAI,SAAS,aAAa,kBAAkB,SAAS,KAAK,KAAK,QAAQ,gBAAgB,YAAY,WAAW,GAAG;AAC/G,WAAO,KAAK,EAAE,MAAM,2BAA2B,UAAU,SAAS,SAAS,kEAAkE,MAAM,8BAA8B,CAAC;AAAA,EACpL;AACA,MACE,YACI,QAAQ,aAAa,kBAAkB,SAAS,MAAM,KAAK,QAAQ,aAAa,kBAAkB,SAAS,KAAK,MACjH,CAAC,QAAQ,gBAAgB,QAAQ,KAAK,CAAC,WAAW,OAAO,UAAU,KACnE,CAAC,QAAQ,QAAQ,KAAK,YAAU,OAAO,kBAAkB,OAAO,SAAS,KAAK,OAAO,SAAS,OAAO,GACxG;AACA,WAAO,KAAK,EAAE,MAAM,gCAAgC,UAAU,SAAS,SAAS,gEAAgE,MAAM,0BAA0B,CAAC;AAAA,EACnL;AACF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import {
|
|
2
|
+
ExperimentImprovementOptionsSchema
|
|
3
|
+
} from "./chunk-GDJRGRHA.js";
|
|
4
|
+
import {
|
|
5
|
+
EvaluationCoordinationTransport
|
|
6
|
+
} from "./chunk-YCQVVFE5.js";
|
|
7
|
+
import {
|
|
8
|
+
ChatModelRefSchema
|
|
9
|
+
} from "./chunk-4NZW4LFB.js";
|
|
10
|
+
import {
|
|
11
|
+
ImmutableReleaseRefSchema,
|
|
12
|
+
contentHash
|
|
13
|
+
} from "./chunk-QUOW6URQ.js";
|
|
14
|
+
|
|
15
|
+
// src/advanced-refiner-evaluations.ts
|
|
16
|
+
import { z } from "zod";
|
|
17
|
+
import {
|
|
18
|
+
ProfileExternalDatasetBindingSchema,
|
|
19
|
+
verifyProfileExternalDatasetBinding
|
|
20
|
+
} from "@openpond/evals";
|
|
21
|
+
var Id = z.string().trim().min(1).max(240);
|
|
22
|
+
var Hash = z.string().regex(/^[a-f0-9]{64}$/);
|
|
23
|
+
var AdvancedRefinerEvaluationPinSchema = z.object({
|
|
24
|
+
schemaVersion: z.literal("openpond.advancedRefinerEvaluation.v1"),
|
|
25
|
+
operationId: Id,
|
|
26
|
+
actorId: Id,
|
|
27
|
+
teamId: Id,
|
|
28
|
+
projectId: Id.nullable(),
|
|
29
|
+
mode: z.enum(["review_quality", "downstream_adaptation"]),
|
|
30
|
+
profileRef: z.object({
|
|
31
|
+
source: z.enum(["local", "github", "openpond_git"]),
|
|
32
|
+
repositoryId: Id,
|
|
33
|
+
profileId: Id
|
|
34
|
+
}).strict(),
|
|
35
|
+
sourceRevision: z.string().min(1).max(500),
|
|
36
|
+
baselineRelease: ImmutableReleaseRefSchema,
|
|
37
|
+
refinerRelease: ImmutableReleaseRefSchema,
|
|
38
|
+
evidence: ImmutableReleaseRefSchema,
|
|
39
|
+
externalDatasetBinding: ProfileExternalDatasetBindingSchema,
|
|
40
|
+
adaptationDatasetBinding: ProfileExternalDatasetBindingSchema,
|
|
41
|
+
privateClosureHash: Hash,
|
|
42
|
+
adaptationTaskIds: z.array(Id).min(1).max(1e4),
|
|
43
|
+
holdoutTaskIds: z.array(Id).min(1).max(1e4),
|
|
44
|
+
adaptationSplit: z.enum(["train", "validation", "test", "frozen_eval"]),
|
|
45
|
+
holdoutSplit: z.enum(["train", "validation", "test", "frozen_eval"]),
|
|
46
|
+
environmentPolicy: z.discriminatedUnion("kind", [
|
|
47
|
+
z.object({ kind: z.literal("resettable"), environmentHash: Hash }).strict(),
|
|
48
|
+
z.object({
|
|
49
|
+
kind: z.literal("frozen_observations"),
|
|
50
|
+
environmentHash: Hash
|
|
51
|
+
}).strict()
|
|
52
|
+
]),
|
|
53
|
+
maximumCostUsd: z.number().finite().positive().max(1e4),
|
|
54
|
+
maximumDurationMs: z.number().int().min(1e3).max(864e5),
|
|
55
|
+
maximumModelSteps: z.number().int().min(1).max(1e3),
|
|
56
|
+
contentHash: Hash
|
|
57
|
+
}).strict();
|
|
58
|
+
function verifyAdvancedRefinerEvaluationPin(raw) {
|
|
59
|
+
const value = AdvancedRefinerEvaluationPinSchema.parse(raw), { contentHash: hash, ...body } = value;
|
|
60
|
+
if (contentHash(body) !== hash)
|
|
61
|
+
throw new Error("Advanced Refiner evaluation admission changed.");
|
|
62
|
+
const binding = verifyProfileExternalDatasetBinding(
|
|
63
|
+
value.externalDatasetBinding
|
|
64
|
+
);
|
|
65
|
+
if (binding.profileId !== value.profileRef.profileId || binding.protectedProfileClosureHash !== value.privateClosureHash)
|
|
66
|
+
throw new Error(
|
|
67
|
+
"Advanced evaluation changed its Profile or protected evaluator closure."
|
|
68
|
+
);
|
|
69
|
+
if (value.adaptationSplit === value.holdoutSplit || (/* @__PURE__ */ new Set([...value.adaptationTaskIds, ...value.holdoutTaskIds])).size !== value.adaptationTaskIds.length + value.holdoutTaskIds.length)
|
|
70
|
+
throw new Error(
|
|
71
|
+
"Adaptation and private holdout must be disjoint ordered cases."
|
|
72
|
+
);
|
|
73
|
+
const adaptation = verifyProfileExternalDatasetBinding(
|
|
74
|
+
value.adaptationDatasetBinding
|
|
75
|
+
);
|
|
76
|
+
if (adaptation.profileId !== binding.profileId || adaptation.packageHash !== binding.packageHash || contentHash(adaptation.dataset) !== contentHash(binding.dataset) || contentHash(adaptation.target) !== contentHash(binding.target) || adaptation.protectedProfileClosureHash !== binding.protectedProfileClosureHash)
|
|
77
|
+
throw new Error(
|
|
78
|
+
"This native advanced adapter needs separate adaptation and holdout recipes from the same exact released package and target."
|
|
79
|
+
);
|
|
80
|
+
if (adaptation.split !== value.adaptationSplit || binding.split !== value.holdoutSplit)
|
|
81
|
+
throw new Error(
|
|
82
|
+
"Advanced split policies differ from their exact admitted recipes."
|
|
83
|
+
);
|
|
84
|
+
const population = new Set(binding.population.map((item) => item.taskId));
|
|
85
|
+
const adapted = new Set(adaptation.population.map((item) => item.taskId));
|
|
86
|
+
if (value.holdoutTaskIds.some((id) => !population.has(id)) || value.adaptationTaskIds.some((id) => !adapted.has(id)))
|
|
87
|
+
throw new Error(
|
|
88
|
+
"Advanced evaluation selected cases outside its exact Dataset recipe."
|
|
89
|
+
);
|
|
90
|
+
return value;
|
|
91
|
+
}
|
|
92
|
+
function sealAdvancedRefinerEvaluationPin(body) {
|
|
93
|
+
return verifyAdvancedRefinerEvaluationPin({
|
|
94
|
+
...body,
|
|
95
|
+
contentHash: contentHash(body)
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
var AdvancedRefinerEvaluationStartSchema = z.object({
|
|
99
|
+
pin: AdvancedRefinerEvaluationPinSchema,
|
|
100
|
+
modelId: Id,
|
|
101
|
+
profileId: Id,
|
|
102
|
+
model: ChatModelRefSchema,
|
|
103
|
+
seed: z.number().int(),
|
|
104
|
+
reasoningEffort: z.enum(["none", "off", "low", "medium", "high", "xhigh", "max"]).nullable()
|
|
105
|
+
}).strict();
|
|
106
|
+
var AdvancedRefinerEvaluationCommandSchema = z.discriminatedUnion(
|
|
107
|
+
"operation",
|
|
108
|
+
[
|
|
109
|
+
z.object({
|
|
110
|
+
operation: z.literal("start"),
|
|
111
|
+
request: AdvancedRefinerEvaluationStartSchema
|
|
112
|
+
}).strict(),
|
|
113
|
+
z.object({ operation: z.enum(["read", "cancel", "resume"]), id: Id }).strict(),
|
|
114
|
+
z.object({ operation: z.literal("list") }).strict()
|
|
115
|
+
]
|
|
116
|
+
);
|
|
117
|
+
var AdvancedRefinerReviewCaseReceiptSchema = z.object({
|
|
118
|
+
taskId: Id,
|
|
119
|
+
sourceCaseHash: Hash,
|
|
120
|
+
reviewSessionId: Id,
|
|
121
|
+
reviewTurnId: Id,
|
|
122
|
+
inputHarness: ImmutableReleaseRefSchema,
|
|
123
|
+
trigger: ImmutableReleaseRefSchema,
|
|
124
|
+
decision: z.enum(["no_action", "route_deterministically", "queue_refiner"]),
|
|
125
|
+
outcome: ImmutableReleaseRefSchema.nullable(),
|
|
126
|
+
proposal: ImmutableReleaseRefSchema.nullable(),
|
|
127
|
+
validations: z.array(ImmutableReleaseRefSchema).max(100),
|
|
128
|
+
grade: z.object({
|
|
129
|
+
id: Id,
|
|
130
|
+
contentHash: Hash,
|
|
131
|
+
score: z.number().min(0).max(1).nullable(),
|
|
132
|
+
passed: z.boolean(),
|
|
133
|
+
failureClass: z.string().nullable()
|
|
134
|
+
}).strict(),
|
|
135
|
+
resetComplete: z.boolean()
|
|
136
|
+
}).strict();
|
|
137
|
+
var AdvancedRefinerReviewReceiptSchema = z.object({
|
|
138
|
+
schemaVersion: z.literal("openpond.advancedRefinerReviewReceipt.v1"),
|
|
139
|
+
id: Id,
|
|
140
|
+
runId: Id,
|
|
141
|
+
pinHash: Hash,
|
|
142
|
+
ownerActorId: Id,
|
|
143
|
+
teamId: Id,
|
|
144
|
+
privateClosureHash: Hash,
|
|
145
|
+
refinerRelease: ImmutableReleaseRefSchema,
|
|
146
|
+
cases: z.array(AdvancedRefinerReviewCaseReceiptSchema).min(1).max(1e4),
|
|
147
|
+
reviewScore: z.number().min(0).max(1).nullable(),
|
|
148
|
+
completedCases: z.number().int().nonnegative(),
|
|
149
|
+
noActionCases: z.number().int().nonnegative(),
|
|
150
|
+
routedCases: z.number().int().nonnegative(),
|
|
151
|
+
proposalCases: z.number().int().nonnegative(),
|
|
152
|
+
costUsd: z.number().finite().nonnegative().nullable(),
|
|
153
|
+
cleanupComplete: z.boolean(),
|
|
154
|
+
createdAt: z.string().datetime(),
|
|
155
|
+
contentHash: Hash
|
|
156
|
+
}).strict();
|
|
157
|
+
function sealAdvancedRefinerReviewReceipt(body) {
|
|
158
|
+
return AdvancedRefinerReviewReceiptSchema.parse({
|
|
159
|
+
...body,
|
|
160
|
+
contentHash: contentHash(body)
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
var AdvancedProfile = ExperimentImprovementOptionsSchema.shape.profiles.element;
|
|
164
|
+
var AdvancedRefinerEvaluationOptionsSchema = ExperimentImprovementOptionsSchema.extend({
|
|
165
|
+
profiles: z.array(
|
|
166
|
+
AdvancedProfile.extend({
|
|
167
|
+
components: z.array(
|
|
168
|
+
AdvancedProfile.shape.components.element.extend({
|
|
169
|
+
adaptationSplits: z.array(
|
|
170
|
+
z.object({
|
|
171
|
+
split: z.enum(["train", "validation", "test", "frozen_eval"]),
|
|
172
|
+
taskIds: z.array(Id)
|
|
173
|
+
})
|
|
174
|
+
),
|
|
175
|
+
environmentHash: Hash,
|
|
176
|
+
resettable: z.boolean(),
|
|
177
|
+
reviewQuality: z.boolean()
|
|
178
|
+
})
|
|
179
|
+
)
|
|
180
|
+
})
|
|
181
|
+
),
|
|
182
|
+
refinerReleases: z.array(ImmutableReleaseRefSchema),
|
|
183
|
+
activeRefiner: ImmutableReleaseRefSchema.nullable(),
|
|
184
|
+
models: z.array(z.object({ id: Id, profileId: Id, name: z.string(), location: z.enum(["local", "hosted"]) }))
|
|
185
|
+
});
|
|
186
|
+
var AdvancedRefinerEvaluationPrepareSchema = z.object({
|
|
187
|
+
operation: z.literal("prepare"),
|
|
188
|
+
evidence: ImmutableReleaseRefSchema,
|
|
189
|
+
profileOptionId: Id,
|
|
190
|
+
componentLabel: Id,
|
|
191
|
+
mode: z.enum(["review_quality", "downstream_adaptation"]),
|
|
192
|
+
adaptationSplit: z.enum(["train", "validation", "test", "frozen_eval"]),
|
|
193
|
+
operationId: Id,
|
|
194
|
+
refinerRelease: ImmutableReleaseRefSchema,
|
|
195
|
+
maximumCostUsd: z.number().positive(),
|
|
196
|
+
maximumDurationMs: z.number().int().positive(),
|
|
197
|
+
maximumModelSteps: z.number().int().positive()
|
|
198
|
+
}).strict();
|
|
199
|
+
var AdvancedRefinerEvaluationSetupCommandSchema = z.union([
|
|
200
|
+
AdvancedRefinerEvaluationCommandSchema,
|
|
201
|
+
AdvancedRefinerEvaluationPrepareSchema,
|
|
202
|
+
z.object({
|
|
203
|
+
operation: z.literal("options"),
|
|
204
|
+
evidence: ImmutableReleaseRefSchema
|
|
205
|
+
}).strict()
|
|
206
|
+
]);
|
|
207
|
+
var AdvancedRefinerEvaluationAccountingSchema = z.object({
|
|
208
|
+
modelRequests: z.number().int().nonnegative(),
|
|
209
|
+
knownSpendUsd: z.number().nonnegative(),
|
|
210
|
+
totalSpendUsd: z.number().nonnegative().nullable(),
|
|
211
|
+
heldUsd: z.number().nonnegative(),
|
|
212
|
+
uncertainRequests: z.number().int().nonnegative()
|
|
213
|
+
}).strict();
|
|
214
|
+
var OpenPondAdvancedRefinerEvaluationClient = class {
|
|
215
|
+
transport;
|
|
216
|
+
constructor(access) {
|
|
217
|
+
this.transport = new EvaluationCoordinationTransport(access);
|
|
218
|
+
}
|
|
219
|
+
async command(raw, signal) {
|
|
220
|
+
const request = AdvancedRefinerEvaluationSetupCommandSchema.parse(raw), access = this.transport.access;
|
|
221
|
+
const assertPin = (raw2) => {
|
|
222
|
+
const pin2 = verifyAdvancedRefinerEvaluationPin(raw2);
|
|
223
|
+
if (pin2.actorId !== access.actorId || pin2.teamId !== access.teamId || pin2.projectId !== access.projectId) throw new Error("The advanced pin belongs to another evaluation owner.");
|
|
224
|
+
return pin2;
|
|
225
|
+
};
|
|
226
|
+
if (request.operation === "start") assertPin(request.request.pin);
|
|
227
|
+
const value = await this.transport.post("advanced-refiner-evaluations", request, signal);
|
|
228
|
+
if (request.operation === "options") {
|
|
229
|
+
const result2 = AdvancedRefinerEvaluationOptionsSchema.parse(value);
|
|
230
|
+
if (result2.teamId !== access.teamId || contentHash(result2.evidence) !== contentHash(request.evidence)) throw new Error("Advanced options changed their reviewed evidence.");
|
|
231
|
+
return result2;
|
|
232
|
+
}
|
|
233
|
+
if (request.operation === "prepare") {
|
|
234
|
+
const result2 = z.object({ pin: AdvancedRefinerEvaluationPinSchema }).strict().parse(value);
|
|
235
|
+
const pin2 = assertPin(result2.pin);
|
|
236
|
+
if (pin2.operationId !== request.operationId || contentHash(pin2.profileRef) !== request.profileOptionId || contentHash(pin2.evidence) !== contentHash(request.evidence) || pin2.mode !== request.mode || contentHash(pin2.refinerRelease) !== contentHash(request.refinerRelease) || pin2.adaptationSplit !== request.adaptationSplit || pin2.maximumCostUsd !== request.maximumCostUsd || pin2.maximumDurationMs !== request.maximumDurationMs || pin2.maximumModelSteps !== request.maximumModelSteps) throw new Error("Advanced preparation changed its reviewed command.");
|
|
237
|
+
return { pin: pin2 };
|
|
238
|
+
}
|
|
239
|
+
if (request.operation === "list") return z.object({ items: z.array(z.object({ id: Id, status: z.enum(["prepared", "running", "succeeded", "failed", "cancelled"]), mode: z.enum(["review_quality", "downstream_adaptation"]), startedAt: z.string().datetime().nullable(), completedAt: z.string().datetime().nullable(), failure: z.string().nullable() }).strict()).max(1e4) }).strict().parse(value);
|
|
240
|
+
if (request.operation === "cancel") {
|
|
241
|
+
const result2 = z.object({ id: Id, status: z.enum(["prepared", "running", "succeeded", "failed", "cancelled"]) }).strict().parse(value);
|
|
242
|
+
if (result2.id !== request.id) throw new Error("Advanced cancellation returned another Run.");
|
|
243
|
+
return result2;
|
|
244
|
+
}
|
|
245
|
+
const result = z.object({ schemaVersion: z.literal("openpond.modelRun.v1"), id: Id, kind: z.literal("evaluation"), modelId: Id, profileId: Id, evaluation: z.object({ benchmarkId: z.literal("harness-refiner"), advancedEvaluation: AdvancedRefinerEvaluationPinSchema }).passthrough() }).passthrough().parse(value);
|
|
246
|
+
const pin = assertPin(result.evaluation.advancedEvaluation);
|
|
247
|
+
if (request.operation === "start" ? pin.contentHash !== request.request.pin.contentHash || result.modelId !== request.request.modelId || result.profileId !== request.request.profileId : result.id !== request.id) throw new Error("The advanced Run differs from its exact retained request.");
|
|
248
|
+
return result;
|
|
249
|
+
}
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
export {
|
|
253
|
+
AdvancedRefinerEvaluationPinSchema,
|
|
254
|
+
verifyAdvancedRefinerEvaluationPin,
|
|
255
|
+
sealAdvancedRefinerEvaluationPin,
|
|
256
|
+
AdvancedRefinerEvaluationStartSchema,
|
|
257
|
+
AdvancedRefinerEvaluationCommandSchema,
|
|
258
|
+
AdvancedRefinerReviewCaseReceiptSchema,
|
|
259
|
+
AdvancedRefinerReviewReceiptSchema,
|
|
260
|
+
sealAdvancedRefinerReviewReceipt,
|
|
261
|
+
AdvancedRefinerEvaluationOptionsSchema,
|
|
262
|
+
AdvancedRefinerEvaluationPrepareSchema,
|
|
263
|
+
AdvancedRefinerEvaluationSetupCommandSchema,
|
|
264
|
+
AdvancedRefinerEvaluationAccountingSchema,
|
|
265
|
+
OpenPondAdvancedRefinerEvaluationClient
|
|
266
|
+
};
|
|
267
|
+
//# sourceMappingURL=chunk-G3FWDZMZ.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/advanced-refiner-evaluations.ts"],
|
|
4
|
+
"sourcesContent": ["import {EvaluationCoordinationTransport,type EvaluationCoordinationAccess} from \"./evaluation-coordination-http.js\";\nimport { z } from \"zod\";\nimport { ExperimentImprovementOptionsSchema } from \"./experiment-improvement-contracts.js\";\nimport { ChatModelRefSchema } from \"@openpond/harness/models\";\nimport { ImmutableReleaseRefSchema, contentHash } from \"@openpond/harness\";\nimport {\n ProfileExternalDatasetBindingSchema,\n verifyProfileExternalDatasetBinding,\n} from \"@openpond/evals\";\nconst Id = z.string().trim().min(1).max(240),\n Hash = z.string().regex(/^[a-f0-9]{64}$/);\nexport const AdvancedRefinerEvaluationPinSchema = z\n .object({\n schemaVersion: z.literal(\"openpond.advancedRefinerEvaluation.v1\"),\n operationId: Id,\n actorId: Id,\n teamId: Id,\n projectId: Id.nullable(),\n mode: z.enum([\"review_quality\", \"downstream_adaptation\"]),\n profileRef: z\n .object({\n source: z.enum([\"local\", \"github\", \"openpond_git\"]),\n repositoryId: Id,\n profileId: Id,\n })\n .strict(),\n sourceRevision: z.string().min(1).max(500),\n baselineRelease: ImmutableReleaseRefSchema,\n refinerRelease: ImmutableReleaseRefSchema,\n evidence: ImmutableReleaseRefSchema,\n externalDatasetBinding: ProfileExternalDatasetBindingSchema,\n adaptationDatasetBinding: ProfileExternalDatasetBindingSchema,\n privateClosureHash: Hash,\n adaptationTaskIds: z.array(Id).min(1).max(10000),\n holdoutTaskIds: z.array(Id).min(1).max(10000),\n adaptationSplit: z.enum([\"train\", \"validation\", \"test\", \"frozen_eval\"]),\n holdoutSplit: z.enum([\"train\", \"validation\", \"test\", \"frozen_eval\"]),\n environmentPolicy: z.discriminatedUnion(\"kind\", [\n z\n .object({ kind: z.literal(\"resettable\"), environmentHash: Hash })\n .strict(),\n z\n .object({\n kind: z.literal(\"frozen_observations\"),\n environmentHash: Hash,\n })\n .strict(),\n ]),\n maximumCostUsd: z.number().finite().positive().max(10000),\n maximumDurationMs: z.number().int().min(1000).max(86400000),\n maximumModelSteps: z.number().int().min(1).max(1000),\n contentHash: Hash,\n })\n .strict();\nexport type AdvancedRefinerEvaluationPin = z.infer<\n typeof AdvancedRefinerEvaluationPinSchema\n>;\nexport function verifyAdvancedRefinerEvaluationPin(raw: unknown) {\n const value = AdvancedRefinerEvaluationPinSchema.parse(raw),\n { contentHash: hash, ...body } = value;\n if (contentHash(body) !== hash)\n throw new Error(\"Advanced Refiner evaluation admission changed.\");\n const binding = verifyProfileExternalDatasetBinding(\n value.externalDatasetBinding,\n );\n if (\n binding.profileId !== value.profileRef.profileId ||\n binding.protectedProfileClosureHash !== value.privateClosureHash\n )\n throw new Error(\n \"Advanced evaluation changed its Profile or protected evaluator closure.\",\n );\n if (\n value.adaptationSplit === value.holdoutSplit ||\n new Set([...value.adaptationTaskIds, ...value.holdoutTaskIds]).size !==\n value.adaptationTaskIds.length + value.holdoutTaskIds.length\n )\n throw new Error(\n \"Adaptation and private holdout must be disjoint ordered cases.\",\n );\n const adaptation = verifyProfileExternalDatasetBinding(\n value.adaptationDatasetBinding,\n );\n if (\n adaptation.profileId !== binding.profileId ||\n adaptation.packageHash !== binding.packageHash ||\n contentHash(adaptation.dataset) !== contentHash(binding.dataset) ||\n contentHash(adaptation.target) !== contentHash(binding.target) ||\n adaptation.protectedProfileClosureHash !==\n binding.protectedProfileClosureHash\n )\n throw new Error(\n \"This native advanced adapter needs separate adaptation and holdout recipes from the same exact released package and target.\",\n );\n if (\n adaptation.split !== value.adaptationSplit ||\n binding.split !== value.holdoutSplit\n )\n throw new Error(\n \"Advanced split policies differ from their exact admitted recipes.\",\n );\n const population = new Set(binding.population.map((item) => item.taskId));\n const adapted = new Set(adaptation.population.map((item) => item.taskId));\n if (\n value.holdoutTaskIds.some((id) => !population.has(id)) ||\n value.adaptationTaskIds.some((id) => !adapted.has(id))\n )\n throw new Error(\n \"Advanced evaluation selected cases outside its exact Dataset recipe.\",\n );\n return value;\n}\nexport function sealAdvancedRefinerEvaluationPin(\n body: Omit<AdvancedRefinerEvaluationPin, \"contentHash\">,\n) {\n return verifyAdvancedRefinerEvaluationPin({\n ...body,\n contentHash: contentHash(body),\n });\n}\nexport const AdvancedRefinerEvaluationStartSchema = z\n .object({\n pin: AdvancedRefinerEvaluationPinSchema,\n modelId: Id,\n profileId: Id,\n model: ChatModelRefSchema,\n seed: z.number().int(),\n reasoningEffort: z\n .enum([\"none\", \"off\", \"low\", \"medium\", \"high\", \"xhigh\", \"max\"])\n .nullable(),\n })\n .strict();\nexport const AdvancedRefinerEvaluationCommandSchema = z.discriminatedUnion(\n \"operation\",\n [\n z\n .object({\n operation: z.literal(\"start\"),\n request: AdvancedRefinerEvaluationStartSchema,\n })\n .strict(),\n z\n .object({ operation: z.enum([\"read\", \"cancel\", \"resume\"]), id: Id })\n .strict(),\n z.object({ operation: z.literal(\"list\") }).strict(),\n ],\n);\nexport const AdvancedRefinerReviewCaseReceiptSchema = z\n .object({\n taskId: Id,\n sourceCaseHash: Hash,\n reviewSessionId: Id,\n reviewTurnId: Id,\n inputHarness: ImmutableReleaseRefSchema,\n trigger: ImmutableReleaseRefSchema,\n decision: z.enum([\"no_action\", \"route_deterministically\", \"queue_refiner\"]),\n outcome: ImmutableReleaseRefSchema.nullable(),\n proposal: ImmutableReleaseRefSchema.nullable(),\n validations: z.array(ImmutableReleaseRefSchema).max(100),\n grade: z\n .object({\n id: Id,\n contentHash: Hash,\n score: z.number().min(0).max(1).nullable(),\n passed: z.boolean(),\n failureClass: z.string().nullable(),\n })\n .strict(),\n resetComplete: z.boolean(),\n })\n .strict();\nexport const AdvancedRefinerReviewReceiptSchema = z\n .object({\n schemaVersion: z.literal(\"openpond.advancedRefinerReviewReceipt.v1\"),\n id: Id,\n runId: Id,\n pinHash: Hash,\n ownerActorId: Id,\n teamId: Id,\n privateClosureHash: Hash,\n refinerRelease: ImmutableReleaseRefSchema,\n cases: z.array(AdvancedRefinerReviewCaseReceiptSchema).min(1).max(10000),\n reviewScore: z.number().min(0).max(1).nullable(),\n completedCases: z.number().int().nonnegative(),\n noActionCases: z.number().int().nonnegative(),\n routedCases: z.number().int().nonnegative(),\n proposalCases: z.number().int().nonnegative(),\n costUsd: z.number().finite().nonnegative().nullable(),\n cleanupComplete: z.boolean(),\n createdAt: z.string().datetime(),\n contentHash: Hash,\n })\n .strict();\nexport type AdvancedRefinerReviewReceipt = z.infer<\n typeof AdvancedRefinerReviewReceiptSchema\n>;\nexport function sealAdvancedRefinerReviewReceipt(\n body: Omit<AdvancedRefinerReviewReceipt, \"contentHash\">,\n) {\n return AdvancedRefinerReviewReceiptSchema.parse({\n ...body,\n contentHash: contentHash(body),\n });\n}\n\nconst AdvancedProfile =\n ExperimentImprovementOptionsSchema.shape.profiles.element;\nexport const AdvancedRefinerEvaluationOptionsSchema =\n ExperimentImprovementOptionsSchema.extend({\n profiles: z.array(\n AdvancedProfile.extend({\n components: z.array(\n AdvancedProfile.shape.components.element.extend({\n adaptationSplits: z.array(\n z.object({\n split: z.enum([\"train\", \"validation\", \"test\", \"frozen_eval\"]),\n taskIds: z.array(Id),\n }),\n ),\n environmentHash: Hash,\n resettable: z.boolean(),\n reviewQuality: z.boolean(),\n }),\n ),\n }),\n ),\n refinerReleases: z.array(ImmutableReleaseRefSchema),\n activeRefiner: ImmutableReleaseRefSchema.nullable(),\n models: z.array(z.object({ id: Id, profileId: Id, name: z.string(), location:z.enum([\"local\",\"hosted\"]) })),\n });\nexport const AdvancedRefinerEvaluationPrepareSchema = z\n .object({\n operation: z.literal(\"prepare\"),\n evidence: ImmutableReleaseRefSchema,\n profileOptionId: Id,\n componentLabel: Id,\n mode: z.enum([\"review_quality\", \"downstream_adaptation\"]),\n adaptationSplit: z.enum([\"train\", \"validation\", \"test\", \"frozen_eval\"]),\n operationId: Id,\n refinerRelease: ImmutableReleaseRefSchema,\n maximumCostUsd: z.number().positive(),\n maximumDurationMs: z.number().int().positive(),\n maximumModelSteps: z.number().int().positive(),\n })\n .strict();\nexport const AdvancedRefinerEvaluationSetupCommandSchema = z.union([\n AdvancedRefinerEvaluationCommandSchema,\n AdvancedRefinerEvaluationPrepareSchema,\n z\n .object({\n operation: z.literal(\"options\"),\n evidence: ImmutableReleaseRefSchema,\n })\n .strict(),\n]);\nexport type AdvancedRefinerEvaluationOptions = z.infer<\n typeof AdvancedRefinerEvaluationOptionsSchema\n>;\n\nexport const AdvancedRefinerEvaluationAccountingSchema = z\n .object({\n modelRequests: z.number().int().nonnegative(),\n knownSpendUsd: z.number().nonnegative(),\n totalSpendUsd: z.number().nonnegative().nullable(),\n heldUsd: z.number().nonnegative(),\n uncertainRequests: z.number().int().nonnegative(),\n })\n .strict();\n\n/** The owner's canonical ModelRun remains intact in readback. This boundary\n * validates its retained advanced pin rather than manufacturing results. */\nexport class OpenPondAdvancedRefinerEvaluationClient {\n private readonly transport:EvaluationCoordinationTransport;\n constructor(access:EvaluationCoordinationAccess){this.transport=new EvaluationCoordinationTransport(access);}\n async command(raw:z.input<typeof AdvancedRefinerEvaluationSetupCommandSchema>,signal?:AbortSignal){\n const request=AdvancedRefinerEvaluationSetupCommandSchema.parse(raw),access=this.transport.access;\n const assertPin=(raw:unknown)=>{const pin=verifyAdvancedRefinerEvaluationPin(raw);if(pin.actorId!==access.actorId||pin.teamId!==access.teamId||pin.projectId!==access.projectId)throw new Error(\"The advanced pin belongs to another evaluation owner.\");return pin;};\n if(request.operation===\"start\")assertPin(request.request.pin);\n const value=await this.transport.post(\"advanced-refiner-evaluations\",request,signal);\n if(request.operation===\"options\"){const result=AdvancedRefinerEvaluationOptionsSchema.parse(value);if(result.teamId!==access.teamId||contentHash(result.evidence)!==contentHash(request.evidence))throw new Error(\"Advanced options changed their reviewed evidence.\");return result;}\n if(request.operation===\"prepare\"){const result=z.object({pin:AdvancedRefinerEvaluationPinSchema}).strict().parse(value);const pin=assertPin(result.pin);if(pin.operationId!==request.operationId||contentHash(pin.profileRef)!==request.profileOptionId||contentHash(pin.evidence)!==contentHash(request.evidence)||pin.mode!==request.mode||contentHash(pin.refinerRelease)!==contentHash(request.refinerRelease)||pin.adaptationSplit!==request.adaptationSplit||pin.maximumCostUsd!==request.maximumCostUsd||pin.maximumDurationMs!==request.maximumDurationMs||pin.maximumModelSteps!==request.maximumModelSteps)throw new Error(\"Advanced preparation changed its reviewed command.\");return {pin};}\n if(request.operation===\"list\")return z.object({items:z.array(z.object({id:Id,status:z.enum([\"prepared\",\"running\",\"succeeded\",\"failed\",\"cancelled\"]),mode:z.enum([\"review_quality\",\"downstream_adaptation\"]),startedAt:z.string().datetime().nullable(),completedAt:z.string().datetime().nullable(),failure:z.string().nullable()}).strict()).max(10000)}).strict().parse(value);\n if(request.operation===\"cancel\"){const result=z.object({id:Id,status:z.enum([\"prepared\",\"running\",\"succeeded\",\"failed\",\"cancelled\"])}).strict().parse(value);if(result.id!==request.id)throw new Error(\"Advanced cancellation returned another Run.\");return result;}\n const result=z.object({schemaVersion:z.literal(\"openpond.modelRun.v1\"),id:Id,kind:z.literal(\"evaluation\"),modelId:Id,profileId:Id,evaluation:z.object({benchmarkId:z.literal(\"harness-refiner\"),advancedEvaluation:AdvancedRefinerEvaluationPinSchema}).passthrough()}).passthrough().parse(value);\n const pin=assertPin(result.evaluation.advancedEvaluation);\n if(request.operation===\"start\"?(pin.contentHash!==request.request.pin.contentHash||result.modelId!==request.request.modelId||result.profileId!==request.request.profileId):result.id!==request.id)throw new Error(\"The advanced Run differs from its exact retained request.\");\n return result;\n }\n}\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;AACA,SAAS,SAAS;AAIlB;AAAA,EACE;AAAA,EACA;AAAA,OACK;AACP,IAAM,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG;AAA3C,IACE,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB;AACnC,IAAM,qCAAqC,EAC/C,OAAO;AAAA,EACN,eAAe,EAAE,QAAQ,uCAAuC;AAAA,EAChE,aAAa;AAAA,EACb,SAAS;AAAA,EACT,QAAQ;AAAA,EACR,WAAW,GAAG,SAAS;AAAA,EACvB,MAAM,EAAE,KAAK,CAAC,kBAAkB,uBAAuB,CAAC;AAAA,EACxD,YAAY,EACT,OAAO;AAAA,IACN,QAAQ,EAAE,KAAK,CAAC,SAAS,UAAU,cAAc,CAAC;AAAA,IAClD,cAAc;AAAA,IACd,WAAW;AAAA,EACb,CAAC,EACA,OAAO;AAAA,EACV,gBAAgB,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG;AAAA,EACzC,iBAAiB;AAAA,EACjB,gBAAgB;AAAA,EAChB,UAAU;AAAA,EACV,wBAAwB;AAAA,EACxB,0BAA0B;AAAA,EAC1B,oBAAoB;AAAA,EACpB,mBAAmB,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,IAAI,GAAK;AAAA,EAC/C,gBAAgB,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,IAAI,GAAK;AAAA,EAC5C,iBAAiB,EAAE,KAAK,CAAC,SAAS,cAAc,QAAQ,aAAa,CAAC;AAAA,EACtE,cAAc,EAAE,KAAK,CAAC,SAAS,cAAc,QAAQ,aAAa,CAAC;AAAA,EACnE,mBAAmB,EAAE,mBAAmB,QAAQ;AAAA,IAC9C,EACG,OAAO,EAAE,MAAM,EAAE,QAAQ,YAAY,GAAG,iBAAiB,KAAK,CAAC,EAC/D,OAAO;AAAA,IACV,EACG,OAAO;AAAA,MACN,MAAM,EAAE,QAAQ,qBAAqB;AAAA,MACrC,iBAAiB;AAAA,IACnB,CAAC,EACA,OAAO;AAAA,EACZ,CAAC;AAAA,EACD,gBAAgB,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,GAAK;AAAA,EACxD,mBAAmB,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,GAAI,EAAE,IAAI,KAAQ;AAAA,EAC1D,mBAAmB,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,GAAI;AAAA,EACnD,aAAa;AACf,CAAC,EACA,OAAO;AAIH,SAAS,mCAAmC,KAAc;AAC/D,QAAM,QAAQ,mCAAmC,MAAM,GAAG,GACxD,EAAE,aAAa,MAAM,GAAG,KAAK,IAAI;AACnC,MAAI,YAAY,IAAI,MAAM;AACxB,UAAM,IAAI,MAAM,gDAAgD;AAClE,QAAM,UAAU;AAAA,IACd,MAAM;AAAA,EACR;AACA,MACE,QAAQ,cAAc,MAAM,WAAW,aACvC,QAAQ,gCAAgC,MAAM;AAE9C,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AACF,MACE,MAAM,oBAAoB,MAAM,iBAChC,oBAAI,IAAI,CAAC,GAAG,MAAM,mBAAmB,GAAG,MAAM,cAAc,CAAC,GAAE,SAC7D,MAAM,kBAAkB,SAAS,MAAM,eAAe;AAExD,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AACF,QAAM,aAAa;AAAA,IACjB,MAAM;AAAA,EACR;AACA,MACE,WAAW,cAAc,QAAQ,aACjC,WAAW,gBAAgB,QAAQ,eACnC,YAAY,WAAW,OAAO,MAAM,YAAY,QAAQ,OAAO,KAC/D,YAAY,WAAW,MAAM,MAAM,YAAY,QAAQ,MAAM,KAC7D,WAAW,gCACT,QAAQ;AAEV,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AACF,MACE,WAAW,UAAU,MAAM,mBAC3B,QAAQ,UAAU,MAAM;AAExB,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AACF,QAAM,aAAa,IAAI,IAAI,QAAQ,WAAW,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC;AACxE,QAAM,UAAU,IAAI,IAAI,WAAW,WAAW,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC;AACxE,MACE,MAAM,eAAe,KAAK,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,KACrD,MAAM,kBAAkB,KAAK,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;AAErD,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AACF,SAAO;AACT;AACO,SAAS,iCACd,MACA;AACA,SAAO,mCAAmC;AAAA,IACxC,GAAG;AAAA,IACH,aAAa,YAAY,IAAI;AAAA,EAC/B,CAAC;AACH;AACO,IAAM,uCAAuC,EACjD,OAAO;AAAA,EACN,KAAK;AAAA,EACL,SAAS;AAAA,EACT,WAAW;AAAA,EACX,OAAO;AAAA,EACP,MAAM,EAAE,OAAO,EAAE,IAAI;AAAA,EACrB,iBAAiB,EACd,KAAK,CAAC,QAAQ,OAAO,OAAO,UAAU,QAAQ,SAAS,KAAK,CAAC,EAC7D,SAAS;AACd,CAAC,EACA,OAAO;AACH,IAAM,yCAAyC,EAAE;AAAA,EACtD;AAAA,EACA;AAAA,IACE,EACG,OAAO;AAAA,MACN,WAAW,EAAE,QAAQ,OAAO;AAAA,MAC5B,SAAS;AAAA,IACX,CAAC,EACA,OAAO;AAAA,IACV,EACG,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,UAAU,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,EAClE,OAAO;AAAA,IACV,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,MAAM,EAAE,CAAC,EAAE,OAAO;AAAA,EACpD;AACF;AACO,IAAM,yCAAyC,EACnD,OAAO;AAAA,EACN,QAAQ;AAAA,EACR,gBAAgB;AAAA,EAChB,iBAAiB;AAAA,EACjB,cAAc;AAAA,EACd,cAAc;AAAA,EACd,SAAS;AAAA,EACT,UAAU,EAAE,KAAK,CAAC,aAAa,2BAA2B,eAAe,CAAC;AAAA,EAC1E,SAAS,0BAA0B,SAAS;AAAA,EAC5C,UAAU,0BAA0B,SAAS;AAAA,EAC7C,aAAa,EAAE,MAAM,yBAAyB,EAAE,IAAI,GAAG;AAAA,EACvD,OAAO,EACJ,OAAO;AAAA,IACN,IAAI;AAAA,IACJ,aAAa;AAAA,IACb,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,SAAS;AAAA,IACzC,QAAQ,EAAE,QAAQ;AAAA,IAClB,cAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EACpC,CAAC,EACA,OAAO;AAAA,EACV,eAAe,EAAE,QAAQ;AAC3B,CAAC,EACA,OAAO;AACH,IAAM,qCAAqC,EAC/C,OAAO;AAAA,EACN,eAAe,EAAE,QAAQ,0CAA0C;AAAA,EACnE,IAAI;AAAA,EACJ,OAAO;AAAA,EACP,SAAS;AAAA,EACT,cAAc;AAAA,EACd,QAAQ;AAAA,EACR,oBAAoB;AAAA,EACpB,gBAAgB;AAAA,EAChB,OAAO,EAAE,MAAM,sCAAsC,EAAE,IAAI,CAAC,EAAE,IAAI,GAAK;AAAA,EACvE,aAAa,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,SAAS;AAAA,EAC/C,gBAAgB,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY;AAAA,EAC7C,eAAe,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY;AAAA,EAC5C,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY;AAAA,EAC1C,eAAe,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY;AAAA,EAC5C,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS;AAAA,EACpD,iBAAiB,EAAE,QAAQ;AAAA,EAC3B,WAAW,EAAE,OAAO,EAAE,SAAS;AAAA,EAC/B,aAAa;AACf,CAAC,EACA,OAAO;AAIH,SAAS,iCACd,MACA;AACA,SAAO,mCAAmC,MAAM;AAAA,IAC9C,GAAG;AAAA,IACH,aAAa,YAAY,IAAI;AAAA,EAC/B,CAAC;AACH;AAEA,IAAM,kBACJ,mCAAmC,MAAM,SAAS;AAC7C,IAAM,yCACX,mCAAmC,OAAO;AAAA,EACxC,UAAU,EAAE;AAAA,IACV,gBAAgB,OAAO;AAAA,MACrB,YAAY,EAAE;AAAA,QACZ,gBAAgB,MAAM,WAAW,QAAQ,OAAO;AAAA,UAC9C,kBAAkB,EAAE;AAAA,YAClB,EAAE,OAAO;AAAA,cACP,OAAO,EAAE,KAAK,CAAC,SAAS,cAAc,QAAQ,aAAa,CAAC;AAAA,cAC5D,SAAS,EAAE,MAAM,EAAE;AAAA,YACrB,CAAC;AAAA,UACH;AAAA,UACA,iBAAiB;AAAA,UACjB,YAAY,EAAE,QAAQ;AAAA,UACtB,eAAe,EAAE,QAAQ;AAAA,QAC3B,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EACA,iBAAiB,EAAE,MAAM,yBAAyB;AAAA,EAClD,eAAe,0BAA0B,SAAS;AAAA,EAClD,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,IAAI,WAAW,IAAI,MAAM,EAAE,OAAO,GAAG,UAAS,EAAE,KAAK,CAAC,SAAQ,QAAQ,CAAC,EAAE,CAAC,CAAC;AAC5G,CAAC;AACI,IAAM,yCAAyC,EACnD,OAAO;AAAA,EACN,WAAW,EAAE,QAAQ,SAAS;AAAA,EAC9B,UAAU;AAAA,EACV,iBAAiB;AAAA,EACjB,gBAAgB;AAAA,EAChB,MAAM,EAAE,KAAK,CAAC,kBAAkB,uBAAuB,CAAC;AAAA,EACxD,iBAAiB,EAAE,KAAK,CAAC,SAAS,cAAc,QAAQ,aAAa,CAAC;AAAA,EACtE,aAAa;AAAA,EACb,gBAAgB;AAAA,EAChB,gBAAgB,EAAE,OAAO,EAAE,SAAS;AAAA,EACpC,mBAAmB,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS;AAAA,EAC7C,mBAAmB,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS;AAC/C,CAAC,EACA,OAAO;AACH,IAAM,8CAA8C,EAAE,MAAM;AAAA,EACjE;AAAA,EACA;AAAA,EACA,EACG,OAAO;AAAA,IACN,WAAW,EAAE,QAAQ,SAAS;AAAA,IAC9B,UAAU;AAAA,EACZ,CAAC,EACA,OAAO;AACZ,CAAC;AAKM,IAAM,4CAA4C,EACtD,OAAO;AAAA,EACN,eAAe,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY;AAAA,EAC5C,eAAe,EAAE,OAAO,EAAE,YAAY;AAAA,EACtC,eAAe,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS;AAAA,EACjD,SAAS,EAAE,OAAO,EAAE,YAAY;AAAA,EAChC,mBAAmB,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY;AAClD,CAAC,EACA,OAAO;AAIH,IAAM,0CAAN,MAA8C;AAAA,EAClC;AAAA,EACjB,YAAY,QAAoC;AAAC,SAAK,YAAU,IAAI,gCAAgC,MAAM;AAAA,EAAE;AAAA,EAC5G,MAAM,QAAQ,KAAgE,QAAoB;AAChG,UAAM,UAAQ,4CAA4C,MAAM,GAAG,GAAE,SAAO,KAAK,UAAU;AAC3F,UAAM,YAAU,CAACA,SAAc;AAAC,YAAMC,OAAI,mCAAmCD,IAAG;AAAE,UAAGC,KAAI,YAAU,OAAO,WAASA,KAAI,WAAS,OAAO,UAAQA,KAAI,cAAY,OAAO,UAAU,OAAM,IAAI,MAAM,uDAAuD;AAAE,aAAOA;AAAA,IAAI;AACpQ,QAAG,QAAQ,cAAY,QAAQ,WAAU,QAAQ,QAAQ,GAAG;AAC5D,UAAM,QAAM,MAAM,KAAK,UAAU,KAAK,gCAA+B,SAAQ,MAAM;AACnF,QAAG,QAAQ,cAAY,WAAU;AAAC,YAAMC,UAAO,uCAAuC,MAAM,KAAK;AAAE,UAAGA,QAAO,WAAS,OAAO,UAAQ,YAAYA,QAAO,QAAQ,MAAI,YAAY,QAAQ,QAAQ,EAAE,OAAM,IAAI,MAAM,mDAAmD;AAAE,aAAOA;AAAA,IAAO;AACrR,QAAG,QAAQ,cAAY,WAAU;AAAC,YAAMA,UAAO,EAAE,OAAO,EAAC,KAAI,mCAAkC,CAAC,EAAE,OAAO,EAAE,MAAM,KAAK;AAAE,YAAMD,OAAI,UAAUC,QAAO,GAAG;AAAE,UAAGD,KAAI,gBAAc,QAAQ,eAAa,YAAYA,KAAI,UAAU,MAAI,QAAQ,mBAAiB,YAAYA,KAAI,QAAQ,MAAI,YAAY,QAAQ,QAAQ,KAAGA,KAAI,SAAO,QAAQ,QAAM,YAAYA,KAAI,cAAc,MAAI,YAAY,QAAQ,cAAc,KAAGA,KAAI,oBAAkB,QAAQ,mBAAiBA,KAAI,mBAAiB,QAAQ,kBAAgBA,KAAI,sBAAoB,QAAQ,qBAAmBA,KAAI,sBAAoB,QAAQ,kBAAkB,OAAM,IAAI,MAAM,oDAAoD;AAAE,aAAO,EAAC,KAAAA,KAAG;AAAA,IAAE;AACxqB,QAAG,QAAQ,cAAY,OAAO,QAAO,EAAE,OAAO,EAAC,OAAM,EAAE,MAAM,EAAE,OAAO,EAAC,IAAG,IAAG,QAAO,EAAE,KAAK,CAAC,YAAW,WAAU,aAAY,UAAS,WAAW,CAAC,GAAE,MAAK,EAAE,KAAK,CAAC,kBAAiB,uBAAuB,CAAC,GAAE,WAAU,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,GAAE,aAAY,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,GAAE,SAAQ,EAAE,OAAO,EAAE,SAAS,EAAC,CAAC,EAAE,OAAO,CAAC,EAAE,IAAI,GAAK,EAAC,CAAC,EAAE,OAAO,EAAE,MAAM,KAAK;AAC/W,QAAG,QAAQ,cAAY,UAAS;AAAC,YAAMC,UAAO,EAAE,OAAO,EAAC,IAAG,IAAG,QAAO,EAAE,KAAK,CAAC,YAAW,WAAU,aAAY,UAAS,WAAW,CAAC,EAAC,CAAC,EAAE,OAAO,EAAE,MAAM,KAAK;AAAE,UAAGA,QAAO,OAAK,QAAQ,GAAG,OAAM,IAAI,MAAM,6CAA6C;AAAE,aAAOA;AAAA,IAAO;AACpQ,UAAM,SAAO,EAAE,OAAO,EAAC,eAAc,EAAE,QAAQ,sBAAsB,GAAE,IAAG,IAAG,MAAK,EAAE,QAAQ,YAAY,GAAE,SAAQ,IAAG,WAAU,IAAG,YAAW,EAAE,OAAO,EAAC,aAAY,EAAE,QAAQ,iBAAiB,GAAE,oBAAmB,mCAAkC,CAAC,EAAE,YAAY,EAAC,CAAC,EAAE,YAAY,EAAE,MAAM,KAAK;AACjS,UAAM,MAAI,UAAU,OAAO,WAAW,kBAAkB;AACxD,QAAG,QAAQ,cAAY,UAAS,IAAI,gBAAc,QAAQ,QAAQ,IAAI,eAAa,OAAO,YAAU,QAAQ,QAAQ,WAAS,OAAO,cAAY,QAAQ,QAAQ,YAAW,OAAO,OAAK,QAAQ,GAAG,OAAM,IAAI,MAAM,2DAA2D;AAC7Q,WAAO;AAAA,EACT;AACF;",
|
|
6
|
+
"names": ["raw", "pin", "result"]
|
|
7
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import {
|
|
2
|
+
ExperimentScoringRequestSchema
|
|
3
|
+
} from "./chunk-SVU4RZ4K.js";
|
|
4
|
+
import {
|
|
5
|
+
ImmutableReleaseRefSchema,
|
|
6
|
+
contentHash
|
|
7
|
+
} from "./chunk-QUOW6URQ.js";
|
|
8
|
+
|
|
9
|
+
// src/experiment-improvement-contracts.ts
|
|
10
|
+
import { z } from "zod";
|
|
11
|
+
import { HumanComparisonSelectionsSchema, HumanComparisonProjectionSchema, verifyHumanComparisonProjection } from "@openpond/evals/human-review";
|
|
12
|
+
import { ExperimentManifestSchema, ExperimentResultSchema, verifyExperimentEvidence, compareExperiments } from "@openpond/evals/experiments";
|
|
13
|
+
import { ProfileExternalDatasetBindingSchema, verifyProfileExternalDatasetBinding } from "@openpond/evals";
|
|
14
|
+
var ExperimentImprovementScoringSchema = z.object({ sourcePass: ImmutableReleaseRefSchema, graders: ExperimentScoringRequestSchema.shape.graders, mappings: ExperimentScoringRequestSchema.shape.mappings, maximumCostUsd: z.number().finite().positive().max(1e4) }).strict();
|
|
15
|
+
var ExperimentImprovementTestRecipeSchema = z.object({ externalDatasetBinding: ProfileExternalDatasetBindingSchema, scoring: ExperimentImprovementScoringSchema.optional(), modelId: z.string().min(1).max(300), maximumCostUsd: z.number().finite().positive().max(1e4) }).strict();
|
|
16
|
+
var OpenPondProfileRefSchema = z.object({ source: z.enum(["local", "github", "openpond_git"]), repositoryId: z.string().trim().min(1), profileId: z.string().trim().min(1) }).strict();
|
|
17
|
+
var Id = z.string().min(1).max(240);
|
|
18
|
+
var Hash = z.string().regex(/^[a-f0-9]{64}$/);
|
|
19
|
+
var ExperimentImprovementStateSchema = z.object({
|
|
20
|
+
schemaVersion: z.literal("openpond.experimentImprovement.v1"),
|
|
21
|
+
id: Id,
|
|
22
|
+
operationId: Id,
|
|
23
|
+
requestHash: Hash,
|
|
24
|
+
actorId: Id,
|
|
25
|
+
teamId: Id,
|
|
26
|
+
projectId: Id.nullable(),
|
|
27
|
+
ownerWorkspaceId: Id,
|
|
28
|
+
ownerRevision: z.number().int().nonnegative(),
|
|
29
|
+
ownerSourceRevision: z.string().min(1).max(500),
|
|
30
|
+
baseRelease: ImmutableReleaseRefSchema,
|
|
31
|
+
profileRef: OpenPondProfileRefSchema,
|
|
32
|
+
baseProfileSourceRevision: z.string().min(1).max(500),
|
|
33
|
+
evidence: z.object({ manifest: ExperimentManifestSchema, result: ExperimentResultSchema }).strict(),
|
|
34
|
+
component: z.object({ kind: z.enum(["skill", "workflow", "agent", "instruction"]), path: z.string().min(1).max(2e3), workflowId: Id.optional() }).strict(),
|
|
35
|
+
mode: z.enum(["manual", "llm_assisted"]),
|
|
36
|
+
status: z.enum(["draft", "authoring_queued", "authoring", "frozen", "testing", "ready_for_review", "adopting", "accepted", "rolled_back", "declined", "failed", "cancelled"]),
|
|
37
|
+
revision: z.number().int().positive(),
|
|
38
|
+
limits: z.object({ maximumCostUsd: z.number().finite().positive(), maximumAuthoringSteps: z.number().int().min(1).max(500), maximumDurationMs: z.number().int().min(1e3).max(864e5) }).strict(),
|
|
39
|
+
authoringSteps: z.number().int().nonnegative().max(500),
|
|
40
|
+
work: z.object({ sessionId: Id, turnId: Id, revision: z.number().int().positive(), startedAt: z.string().datetime() }).strict().nullable(),
|
|
41
|
+
frozen: z.object({
|
|
42
|
+
release: ImmutableReleaseRefSchema,
|
|
43
|
+
sourceRevision: z.string().min(1).max(500),
|
|
44
|
+
profileSourceRevision: z.string().min(1).max(500),
|
|
45
|
+
authoringHash: Hash,
|
|
46
|
+
partitionHash: Hash,
|
|
47
|
+
executableSourceHash: Hash,
|
|
48
|
+
protectedClosureHash: Hash,
|
|
49
|
+
candidateRevision: z.number().int().positive(),
|
|
50
|
+
diff: z.array(z.object({ path: z.string(), before: Hash.nullable(), after: Hash.nullable() }).strict()).max(1e4)
|
|
51
|
+
}).strict().nullable(),
|
|
52
|
+
test: z.object({ recipe: ExperimentImprovementTestRecipeSchema, recipeHash: z.string().regex(/^[a-f0-9]{64}$/), baselineOperationId: z.string(), candidateOperationId: z.string(), cancelRequestedAt: z.string().datetime().nullable(), dispatching: z.object({ kind: z.enum(["baseline", "candidate"]), token: Id, pid: z.number().int().positive(), processStart: z.string().nullable(), startedAt: z.string().datetime() }).strict().nullable() }).strict().nullable(),
|
|
53
|
+
comparison: z.object({ baseline: z.object({ manifest: ExperimentManifestSchema, result: ExperimentResultSchema }).strict(), candidate: z.object({ manifest: ExperimentManifestSchema, result: ExperimentResultSchema }).strict(), frozenHash: Hash, humanProjection: HumanComparisonProjectionSchema.optional() }).strict().nullable(),
|
|
54
|
+
adoptionIntent: z.object({ operationId: Id, expectedOwnerRevision: z.number().int().nonnegative(), frozenHash: Hash, humanProjection: HumanComparisonProjectionSchema.optional() }).strict().nullable(),
|
|
55
|
+
acceptance: ImmutableReleaseRefSchema.nullable(),
|
|
56
|
+
rollback: ImmutableReleaseRefSchema.nullable(),
|
|
57
|
+
error: z.string().max(2e3).nullable(),
|
|
58
|
+
createdAt: z.string().datetime(),
|
|
59
|
+
updatedAt: z.string().datetime(),
|
|
60
|
+
contentHash: Hash
|
|
61
|
+
}).strict();
|
|
62
|
+
function verifyExperimentImprovementState(raw) {
|
|
63
|
+
const value = ExperimentImprovementStateSchema.parse(raw), { contentHash: hash, ...body } = value;
|
|
64
|
+
if (contentHash(body) !== hash) throw new Error("Improvement state integrity failed.");
|
|
65
|
+
verifyExperimentEvidence(value.evidence);
|
|
66
|
+
if (value.evidence.manifest.teamId !== value.teamId || value.evidence.result.status !== "completed" || !value.evidence.result.cases.some((row) => row.status === "completed")) throw new Error("Improvement requires usable completed owner evidence.");
|
|
67
|
+
if (value.test && (contentHash(value.test.recipe) !== value.test.recipeHash || verifyProfileExternalDatasetBinding(value.test.recipe.externalDatasetBinding).profileId !== value.profileRef.profileId)) throw new Error("The test recipe changed its exact selected Profile.");
|
|
68
|
+
if (value.comparison?.humanProjection) {
|
|
69
|
+
const projection = verifyHumanComparisonProjection(value.comparison.humanProjection);
|
|
70
|
+
if (projection.teamId !== value.teamId || projection.projectId !== value.projectId || contentHash(projection.baseline) !== contentHash(value.comparison.baseline) || contentHash(projection.candidate) !== contentHash(value.comparison.candidate)) throw new Error("Human comparison projection differs from its exact owner evidence.");
|
|
71
|
+
}
|
|
72
|
+
if (value.comparison) {
|
|
73
|
+
verifyExperimentEvidence(value.comparison.baseline);
|
|
74
|
+
verifyExperimentEvidence(value.comparison.candidate);
|
|
75
|
+
if (!value.frozen || value.comparison.frozenHash !== contentHash(value.frozen) || !compareExperiments(value.comparison.baseline, value.comparison.candidate).comparable) throw new Error("Improvement comparison is not bound to a comparable frozen candidate.");
|
|
76
|
+
}
|
|
77
|
+
if (value.status === "ready_for_review" && (!value.frozen || !value.comparison)) throw new Error("Improvement is missing qualification.");
|
|
78
|
+
if (value.status === "adopting" && (!value.adoptionIntent || !value.frozen || value.adoptionIntent.frozenHash !== contentHash(value.frozen))) throw new Error("Improvement adoption intent is not bound to its tested revision.");
|
|
79
|
+
if (value.status === "accepted" && (!value.acceptance || !value.comparison || !value.frozen)) throw new Error("Improvement acceptance is missing its exact qualification receipt.");
|
|
80
|
+
if (value.status === "rolled_back" && (!value.acceptance || !value.rollback)) throw new Error("Profile rollback is missing its durable owner receipt.");
|
|
81
|
+
return value;
|
|
82
|
+
}
|
|
83
|
+
function sealExperimentImprovementState(input) {
|
|
84
|
+
return verifyExperimentImprovementState({ ...input, contentHash: contentHash(input) });
|
|
85
|
+
}
|
|
86
|
+
var ExperimentImprovementListSchema = z.object({ states: z.array(ExperimentImprovementStateSchema).max(50), nextCursor: Id.nullable() }).strict();
|
|
87
|
+
var StartExperimentImprovementSchema = z.object({
|
|
88
|
+
operationId: Id,
|
|
89
|
+
ownerWorkspaceId: Id,
|
|
90
|
+
expectedOwnerRevision: z.number().int().nonnegative(),
|
|
91
|
+
baseRelease: ImmutableReleaseRefSchema,
|
|
92
|
+
profileRef: OpenPondProfileRefSchema,
|
|
93
|
+
profileSourceRevision: z.string().min(1).max(500),
|
|
94
|
+
projectId: Id.nullable(),
|
|
95
|
+
evidence: ImmutableReleaseRefSchema,
|
|
96
|
+
component: ExperimentImprovementStateSchema.shape.component,
|
|
97
|
+
mode: ExperimentImprovementStateSchema.shape.mode,
|
|
98
|
+
limits: ExperimentImprovementStateSchema.shape.limits
|
|
99
|
+
}).strict();
|
|
100
|
+
var ExperimentImprovementCommandSchema = z.discriminatedUnion("operation", [
|
|
101
|
+
z.object({ operation: z.literal("list"), limit: z.number().int().min(1).max(50).default(30), cursor: Id.optional() }).strict(),
|
|
102
|
+
z.object({ operation: z.literal("options"), evidence: ImmutableReleaseRefSchema }).strict(),
|
|
103
|
+
z.object({ operation: z.literal("start"), request: StartExperimentImprovementSchema }).strict(),
|
|
104
|
+
z.object({ operation: z.enum(["read", "humanComparisonOptions"]), id: Id }).strict(),
|
|
105
|
+
z.object({ operation: z.literal("instructions"), id: Id }).strict(),
|
|
106
|
+
z.object({ operation: z.literal("edit"), id: Id, revision: z.number().int().positive(), expectedFileHash: Hash, text: z.string().max(25e4) }).strict(),
|
|
107
|
+
z.object({ operation: z.literal("mode"), id: Id, revision: z.number().int().positive(), mode: z.enum(["manual", "llm_assisted"]) }).strict(),
|
|
108
|
+
z.object({ operation: z.literal("author"), id: Id, revision: z.number().int().positive(), modelId: Id, prompt: z.string().min(1).max(2e4) }).strict(),
|
|
109
|
+
z.object({ operation: z.enum(["freeze", "finishAuthoring", "useCandidate", "discard", "cancel", "rollback"]), id: Id, revision: z.number().int().positive() }).strict(),
|
|
110
|
+
z.object({ operation: z.literal("test"), id: Id, revision: z.number().int().positive(), recipe: ExperimentImprovementTestRecipeSchema }).strict(),
|
|
111
|
+
z.object({ operation: z.literal("compare"), id: Id, revision: z.number().int().positive(), baseline: ImmutableReleaseRefSchema, candidate: ImmutableReleaseRefSchema, humanSelections: HumanComparisonSelectionsSchema.optional() }).strict()
|
|
112
|
+
]);
|
|
113
|
+
var ExperimentImprovementOptionsSchema = z.object({ teamId: Id, evidence: ImmutableReleaseRefSchema, scoring: ExperimentImprovementScoringSchema.optional(), profiles: z.array(z.object({ id: Hash, name: z.string(), profileRef: OpenPondProfileRefSchema, ownerWorkspaceId: Id, ownerRevision: z.number().int().nonnegative(), baseRelease: ImmutableReleaseRefSchema, profileSourceRevision: z.string(), components: z.array(z.object({ kind: z.enum(["skill", "instruction", "agent", "workflow"]), path: z.string(), agentId: z.string().optional(), workflowId: z.string().optional(), label: z.string(), binding: ProfileExternalDatasetBindingSchema }).strict()).max(500) }).strict()).max(100) }).strict();
|
|
114
|
+
var ExperimentImprovementHumanOptionsSchema = z.object({ candidateId: Id, actorId: Id, teamId: Id, projectId: Id.nullable(), candidateRevision: z.number().int().positive(), recipeHash: Hash, baseline: z.object({ manifest: ExperimentManifestSchema, result: ExperimentResultSchema }).strict(), candidate: z.object({ manifest: ExperimentManifestSchema, result: ExperimentResultSchema }).strict(), contentHash: Hash }).strict();
|
|
115
|
+
function verifyExperimentImprovementHumanOptions(raw) {
|
|
116
|
+
const value = ExperimentImprovementHumanOptionsSchema.parse(raw), { contentHash: hash, ...body } = value;
|
|
117
|
+
if (contentHash(body) !== hash) throw new Error("Accepted Human discovery integrity failed.");
|
|
118
|
+
for (const lane of [value.baseline, value.candidate]) {
|
|
119
|
+
verifyExperimentEvidence(lane);
|
|
120
|
+
if (lane.manifest.teamId !== value.teamId || lane.manifest.lineage?.humanProjection) throw new Error("Discovery must retain original owner evidence.");
|
|
121
|
+
}
|
|
122
|
+
return value;
|
|
123
|
+
}
|
|
124
|
+
function sealExperimentImprovementHumanOptions(state, pair) {
|
|
125
|
+
if (!state.test || state.test.cancelRequestedAt) throw new Error("The paired test is unavailable.");
|
|
126
|
+
const body = { candidateId: state.id, actorId: state.actorId, teamId: state.teamId, projectId: state.projectId, candidateRevision: state.revision, recipeHash: state.test.recipeHash, ...pair };
|
|
127
|
+
return verifyExperimentImprovementHumanOptions({ ...body, contentHash: contentHash(body) });
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export {
|
|
131
|
+
ExperimentImprovementScoringSchema,
|
|
132
|
+
ExperimentImprovementTestRecipeSchema,
|
|
133
|
+
ExperimentImprovementStateSchema,
|
|
134
|
+
verifyExperimentImprovementState,
|
|
135
|
+
sealExperimentImprovementState,
|
|
136
|
+
ExperimentImprovementListSchema,
|
|
137
|
+
StartExperimentImprovementSchema,
|
|
138
|
+
ExperimentImprovementCommandSchema,
|
|
139
|
+
ExperimentImprovementOptionsSchema,
|
|
140
|
+
ExperimentImprovementHumanOptionsSchema,
|
|
141
|
+
verifyExperimentImprovementHumanOptions,
|
|
142
|
+
sealExperimentImprovementHumanOptions
|
|
143
|
+
};
|
|
144
|
+
//# sourceMappingURL=chunk-GDJRGRHA.js.map
|