openpond-sdk 0.12.3 → 0.12.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actions-local.js +2 -1
- package/dist/actions-local.js.map +1 -7
- package/dist/actions.js +2 -1
- package/dist/actions.js.map +1 -7
- package/dist/advanced-refiner-evaluations.js +13 -12
- package/dist/advanced-refiner-evaluations.js.map +1 -7
- package/dist/candidate-evaluations.js +9 -8
- package/dist/candidate-evaluations.js.map +1 -7
- package/dist/{chunk-77N3QEIL.js → chunk-25L5IXHA.js} +5 -5
- package/dist/chunk-25L5IXHA.js.map +1 -0
- package/dist/{chunk-YCQVVFE5.js → chunk-3B5KSH3V.js} +2 -2
- package/dist/chunk-3B5KSH3V.js.map +1 -0
- package/dist/{chunk-2DMYAJLS.js → chunk-3DWPLYFJ.js} +2 -2
- package/dist/chunk-3DWPLYFJ.js.map +1 -0
- package/dist/{chunk-ZJ3H3VS7.js → chunk-6GQ3GIAP.js} +7 -5
- package/dist/chunk-6GQ3GIAP.js.map +1 -0
- package/dist/{chunk-36WJTS5V.js → chunk-6KKM2XRR.js} +75 -15
- package/dist/chunk-6KKM2XRR.js.map +1 -0
- package/dist/{chunk-BSK3VLM3.js → chunk-6MFFKAU6.js} +3 -3
- package/dist/chunk-6MFFKAU6.js.map +1 -0
- package/dist/{chunk-UPS52OPZ.js → chunk-6Y67YHVH.js} +8 -8
- package/dist/chunk-6Y67YHVH.js.map +1 -0
- package/dist/{chunk-G6Y6CQYV.js → chunk-AUNNXJQE.js} +4 -4
- package/dist/chunk-AUNNXJQE.js.map +1 -0
- package/dist/{chunk-XYQD4LDI.js → chunk-BGNPKYBJ.js} +3 -3
- package/dist/chunk-BGNPKYBJ.js.map +1 -0
- package/dist/{chunk-K5WRVSOL.js → chunk-BJ5MKLWH.js} +8 -8
- package/dist/chunk-BJ5MKLWH.js.map +1 -0
- package/dist/{chunk-VY5AFE7A.js → chunk-BYNVSWIQ.js} +4 -4
- package/dist/chunk-BYNVSWIQ.js.map +1 -0
- package/dist/{chunk-TCHAWJMB.js → chunk-CAMLCKSF.js} +3 -3
- package/dist/chunk-CAMLCKSF.js.map +1 -0
- package/dist/{chunk-3KXDH6OB.js → chunk-CL3IOOVL.js} +1 -1
- package/dist/chunk-CL3IOOVL.js.map +1 -0
- package/dist/{chunk-CTVKI3BV.js → chunk-DBKCSQVH.js} +3 -3
- package/dist/chunk-DBKCSQVH.js.map +1 -0
- package/dist/{chunk-AG5DDL7T.js → chunk-DH6O4JFB.js} +2 -2
- package/dist/chunk-DH6O4JFB.js.map +1 -0
- package/dist/chunk-EJZ7HOIS.js +8709 -0
- package/dist/chunk-EJZ7HOIS.js.map +1 -0
- package/dist/{chunk-QEIT5VXJ.js → chunk-F4GITADW.js} +4 -4
- package/dist/chunk-F4GITADW.js.map +1 -0
- package/dist/{chunk-MEKWESBF.js → chunk-FBZTDDRC.js} +2 -2
- package/dist/chunk-FBZTDDRC.js.map +1 -0
- package/dist/{chunk-FY7NHCEH.js → chunk-FLNWE5ZA.js} +5 -5
- package/dist/chunk-FLNWE5ZA.js.map +1 -0
- package/dist/{chunk-YVUIHH6Z.js → chunk-GPQLBVQC.js} +1 -1
- package/dist/chunk-GPQLBVQC.js.map +1 -0
- package/dist/{chunk-YUKV7RCD.js → chunk-GWGDWZVN.js} +5 -5
- package/dist/chunk-GWGDWZVN.js.map +1 -0
- package/dist/{chunk-KAE7AMYE.js → chunk-H3367ISL.js} +2 -2
- package/dist/chunk-H3367ISL.js.map +1 -0
- package/dist/{chunk-COPVBJPL.js → chunk-H3JZ3P5T.js} +2 -1
- package/dist/chunk-H3JZ3P5T.js.map +1 -0
- package/dist/{chunk-3LP7DNLQ.js → chunk-H4NTJFEE.js} +2 -2
- package/dist/chunk-H4NTJFEE.js.map +1 -0
- package/dist/{chunk-CEZEVILU.js → chunk-HJ2AIAKX.js} +1 -1
- package/dist/chunk-HJ2AIAKX.js.map +1 -0
- package/dist/{chunk-BFR3VC25.js → chunk-HOOHJCE7.js} +18 -18
- package/dist/chunk-HOOHJCE7.js.map +1 -0
- package/dist/{chunk-TSMGMA6R.js → chunk-KKL6OICH.js} +3 -3
- package/dist/chunk-KKL6OICH.js.map +1 -0
- package/dist/{chunk-IWT6JUZD.js → chunk-L6EJX2VQ.js} +2 -2
- package/dist/chunk-L6EJX2VQ.js.map +1 -0
- package/dist/{chunk-QTJQIHDZ.js → chunk-MJELLVQP.js} +4 -3
- package/dist/chunk-MJELLVQP.js.map +1 -0
- package/dist/{chunk-CK3ONRRC.js → chunk-O54KGALH.js} +3 -3
- package/dist/chunk-O54KGALH.js.map +1 -0
- package/dist/{chunk-T4APHDES.js → chunk-O6FKHAWN.js} +1 -1
- package/dist/chunk-O6FKHAWN.js.map +1 -0
- package/dist/{chunk-P27DQ65Z.js → chunk-OVGDDKO2.js} +1 -1
- package/dist/chunk-OVGDDKO2.js.map +1 -0
- package/dist/{chunk-O4GE6VTR.js → chunk-PWW5QKZS.js} +2 -2
- package/dist/chunk-PWW5QKZS.js.map +1 -0
- package/dist/{chunk-DUKW3R7O.js → chunk-Q4TKI2MV.js} +2 -2
- package/dist/chunk-Q4TKI2MV.js.map +1 -0
- package/dist/{chunk-HKWX624G.js → chunk-QFBSNSUL.js} +5 -5
- package/dist/chunk-QFBSNSUL.js.map +1 -0
- package/dist/{chunk-MLPPSCDQ.js → chunk-RE426Q7Y.js} +4 -4
- package/dist/chunk-RE426Q7Y.js.map +1 -0
- package/dist/{chunk-G53MA5XX.js → chunk-T3MFWGMX.js} +3 -3
- package/dist/chunk-T3MFWGMX.js.map +1 -0
- package/dist/{chunk-2U54EOXJ.js → chunk-TK3DXH7A.js} +1 -1
- package/dist/chunk-TK3DXH7A.js.map +1 -0
- package/dist/{chunk-6Q4I33CH.js → chunk-VZPNRPXL.js} +5 -5
- package/dist/chunk-VZPNRPXL.js.map +1 -0
- package/dist/{chunk-ROBAX5TN.js → chunk-WQFU635Q.js} +8 -6
- package/dist/chunk-WQFU635Q.js.map +1 -0
- package/dist/{chunk-TMUAKJWX.js → chunk-XACJ63U4.js} +14 -5
- package/dist/chunk-XACJ63U4.js.map +1 -0
- package/dist/chunk-YLJ4XMA6.js +35 -0
- package/dist/chunk-YLJ4XMA6.js.map +1 -0
- package/dist/{chunk-K3JCP4FO.js → chunk-Z252BNAZ.js} +1 -1
- package/dist/chunk-Z252BNAZ.js.map +1 -0
- package/dist/{chunk-CWXM57QR.js → chunk-ZFYI4P6C.js} +5 -5
- package/dist/chunk-ZFYI4P6C.js.map +1 -0
- package/dist/connected-evidence.js +22 -21
- package/dist/connected-evidence.js.map +1 -7
- package/dist/dataset-commerce.js +2 -0
- package/dist/dataset-commerce.js.map +1 -7
- package/dist/dataset-marketplace.js +2 -0
- package/dist/dataset-marketplace.js.map +1 -7
- package/dist/dataset-workspaces.js +22 -21
- package/dist/dataset-workspaces.js.map +1 -7
- package/dist/experiment-evaluation-schedules.js +28 -27
- package/dist/experiment-evaluation-schedules.js.map +1 -7
- package/dist/experiment-improvements.js +12 -11
- package/dist/experiment-improvements.js.map +1 -7
- package/dist/experiments.js +26 -25
- package/dist/experiments.js.map +1 -7
- package/dist/human-review.js +3 -2
- package/dist/human-review.js.map +1 -7
- package/dist/index.js +154 -62
- package/dist/index.js.map +1 -7
- package/dist/learning.js +4 -3
- package/dist/learning.js.map +1 -7
- package/dist/model-batch-review.js +2 -1
- package/dist/model-batch-review.js.map +1 -7
- package/dist/model-projects.js +3 -2
- package/dist/model-projects.js.map +1 -7
- package/dist/model-starter-attempts.js +5 -4
- package/dist/model-starter-attempts.js.map +1 -7
- package/dist/model-starter-catalog.js +15 -14
- package/dist/model-starter-catalog.js.map +1 -7
- package/dist/model-starters.js +15 -14
- package/dist/model-starters.js.map +1 -7
- package/dist/model-taskset-authoring.js +2 -1
- package/dist/model-taskset-authoring.js.map +1 -7
- package/dist/model-taskset-runs.js +6 -5
- package/dist/model-taskset-runs.js.map +1 -7
- package/dist/post-training.js +9 -8
- package/dist/post-training.js.map +1 -7
- package/dist/profile-actions.js +3 -2
- package/dist/profile-actions.js.map +1 -7
- package/dist/project-actions.js +3 -2
- package/dist/project-actions.js.map +1 -7
- package/dist/refiner.js +3 -2
- package/dist/refiner.js.map +1 -7
- package/dist/taskset-catalog.js +2 -1
- package/dist/taskset-catalog.js.map +1 -7
- package/dist/taskset-drafts.js +19 -18
- package/dist/taskset-drafts.js.map +1 -7
- package/dist/taskset-packages.js +25 -22
- package/dist/taskset-packages.js.map +1 -7
- package/dist/training-bundle.js +44 -19
- package/dist/training-bundle.js.map +1 -7
- package/dist/training-projects.js +2 -1
- package/dist/training-projects.js.map +1 -7
- package/dist/training.js +11 -10
- package/dist/training.js.map +1 -7
- package/dist/types/packages/cloud/src/api/core.d.ts.map +1 -1
- package/dist/types/packages/cloud/src/hosted-chat.d.ts +28 -0
- package/dist/types/packages/cloud/src/hosted-chat.d.ts.map +1 -1
- package/dist/types/packages/cloud/src/sandbox/client-handles.d.ts.map +1 -1
- package/dist/types/packages/cloud/src/sandbox/client.d.ts.map +1 -1
- package/dist/types/packages/cloud/src/sandbox/sandbox-instance-client.d.ts.map +1 -1
- package/dist/types/packages/cloud/src/sandbox/types/runtime-profiles.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/advanced-refiner-evaluations.d.ts +487 -487
- package/dist/types/packages/sdk/src/advanced-refiner-evaluations.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/candidate-evaluation-client.d.ts +375 -351
- package/dist/types/packages/sdk/src/candidate-evaluation-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/candidate-evaluations.d.ts +80 -54
- package/dist/types/packages/sdk/src/candidate-evaluations.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/connected-collection-status.d.ts +5 -5
- package/dist/types/packages/sdk/src/connected-dataset-source.d.ts +10 -10
- package/dist/types/packages/sdk/src/connected-evidence-client.d.ts +464 -460
- package/dist/types/packages/sdk/src/connected-evidence-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/connected-evidence-contracts.d.ts +23 -23
- package/dist/types/packages/sdk/src/connected-recorded-execution.d.ts +77 -74
- package/dist/types/packages/sdk/src/connected-recorded-execution.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/connected-sync.d.ts +30 -30
- package/dist/types/packages/sdk/src/connected-sync.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/dataset-commerce-contracts.d.ts +14 -14
- package/dist/types/packages/sdk/src/dataset-commerce-files.d.ts +5 -5
- package/dist/types/packages/sdk/src/dataset-commerce-files.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/dataset-commerce-responses.d.ts +82 -82
- package/dist/types/packages/sdk/src/dataset-commerce.d.ts +58 -58
- package/dist/types/packages/sdk/src/dataset-commerce.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/dataset-marketplace.d.ts +4 -4
- package/dist/types/packages/sdk/src/dataset-marketplace.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/dataset-population.d.ts +10 -10
- package/dist/types/packages/sdk/src/dataset-population.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/dataset-preparation-authoring-contracts.d.ts +172 -172
- package/dist/types/packages/sdk/src/dataset-preparation-client.d.ts +47 -47
- package/dist/types/packages/sdk/src/dataset-preparation-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/dataset-preparation-contracts.d.ts +90 -90
- package/dist/types/packages/sdk/src/dataset-preparation-qualification-contracts.d.ts +7 -7
- package/dist/types/packages/sdk/src/dataset-preparation-research-contracts.d.ts +26 -26
- package/dist/types/packages/sdk/src/dataset-preparation-starting-state.d.ts +8 -8
- package/dist/types/packages/sdk/src/dataset-preparation-workspace.d.ts +111 -110
- package/dist/types/packages/sdk/src/dataset-preparation-workspace.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/dataset-workspace-client.d.ts +896 -889
- package/dist/types/packages/sdk/src/dataset-workspace-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/dataset-workspace-contracts.d.ts +225 -223
- package/dist/types/packages/sdk/src/dataset-workspace-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/dataset-workspace-operations.d.ts +226 -224
- package/dist/types/packages/sdk/src/dataset-workspace-operations.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/evaluation-coordination-http.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/evaluation-project-context.d.ts +1 -1
- package/dist/types/packages/sdk/src/experiment-client.d.ts +2102 -2014
- package/dist/types/packages/sdk/src/experiment-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiment-contracts.d.ts +165 -113
- package/dist/types/packages/sdk/src/experiment-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiment-evaluation-schedules.d.ts +587 -516
- package/dist/types/packages/sdk/src/experiment-evaluation-schedules.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiment-execution-selection.d.ts +5 -5
- package/dist/types/packages/sdk/src/experiment-execution-selection.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiment-field-mappings.d.ts +8 -8
- package/dist/types/packages/sdk/src/experiment-grading-contracts.d.ts +16 -16
- package/dist/types/packages/sdk/src/experiment-improvement-client.d.ts +2524 -2524
- package/dist/types/packages/sdk/src/experiment-improvement-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiment-improvement-contracts.d.ts +1557 -1557
- package/dist/types/packages/sdk/src/experiment-improvement-hosted-adoption.d.ts +150 -150
- package/dist/types/packages/sdk/src/experiment-inspection.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiment-run-contracts.d.ts +38 -25
- package/dist/types/packages/sdk/src/experiment-run-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiment-run-details.d.ts +303 -243
- package/dist/types/packages/sdk/src/experiment-run-details.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/experiment-scoring-contracts.d.ts +21 -21
- package/dist/types/packages/sdk/src/grader-inspection.d.ts +104 -104
- package/dist/types/packages/sdk/src/grader-inspection.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/harness-experiment-policy.d.ts +33 -20
- package/dist/types/packages/sdk/src/harness-experiment-policy.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/human-review-client.d.ts +288 -288
- package/dist/types/packages/sdk/src/human-review-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/index.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/learning-batch-assets.d.ts +15 -13
- package/dist/types/packages/sdk/src/learning-batch-assets.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/learning-client.d.ts +5 -5
- package/dist/types/packages/sdk/src/learning-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/learning-credentials.d.ts +4 -4
- package/dist/types/packages/sdk/src/learning-credentials.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/managed-training-preparation.d.ts +69 -69
- package/dist/types/packages/sdk/src/managed-training-preparation.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-batch-review-contracts.d.ts +65 -61
- package/dist/types/packages/sdk/src/model-batch-review-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-batch-review-preparation.d.ts +74 -72
- package/dist/types/packages/sdk/src/model-batch-review-preparation.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-batch-review.d.ts +96 -92
- package/dist/types/packages/sdk/src/model-batch-review.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-bound-taskset-authoring.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-learning-intent.d.ts +1 -1
- package/dist/types/packages/sdk/src/model-learning-policy.d.ts +247 -247
- package/dist/types/packages/sdk/src/model-learning-policy.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-projects.d.ts +342 -322
- package/dist/types/packages/sdk/src/model-projects.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-starter-attempts.d.ts +187 -187
- package/dist/types/packages/sdk/src/model-starter-attempts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-starter-catalog.d.ts +181 -179
- package/dist/types/packages/sdk/src/model-starter-catalog.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-starter-execution.d.ts +40 -40
- package/dist/types/packages/sdk/src/model-starters.d.ts +291 -287
- package/dist/types/packages/sdk/src/model-starters.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-authoring-contracts.d.ts +3 -3
- package/dist/types/packages/sdk/src/model-taskset-derivation.d.ts +114 -112
- package/dist/types/packages/sdk/src/model-taskset-derivation.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-resources.d.ts +18 -18
- package/dist/types/packages/sdk/src/model-taskset-runs-contracts.d.ts +179 -127
- package/dist/types/packages/sdk/src/model-taskset-runs-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/model-taskset-runs.d.ts +865 -833
- package/dist/types/packages/sdk/src/model-taskset-runs.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/post-training.d.ts +444 -375
- package/dist/types/packages/sdk/src/post-training.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/profile-actions.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/profile-evaluation-discovery.d.ts +150 -150
- package/dist/types/packages/sdk/src/profile-evaluation-discovery.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/profile-external-dataset-package.d.ts +254 -251
- package/dist/types/packages/sdk/src/profile-external-dataset-package.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/project-actions.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/protocol.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/run-diagnostics.d.ts +3 -3
- package/dist/types/packages/sdk/src/task-design-contracts.d.ts +50 -50
- package/dist/types/packages/sdk/src/task-inventory.d.ts +4 -4
- package/dist/types/packages/sdk/src/task-inventory.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-authored-contracts.d.ts +190 -189
- package/dist/types/packages/sdk/src/taskset-authored-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-authored-files.d.ts +149 -148
- package/dist/types/packages/sdk/src/taskset-authored-files.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-authored-portable-release.d.ts +15 -6
- package/dist/types/packages/sdk/src/taskset-authored-portable-release.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-calibration-fixtures.d.ts +64 -63
- package/dist/types/packages/sdk/src/taskset-calibration-fixtures.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-catalog.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-api-contracts.d.ts +245 -243
- package/dist/types/packages/sdk/src/taskset-draft-api-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-authoring.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-client.d.ts +745 -738
- package/dist/types/packages/sdk/src/taskset-draft-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-core.d.ts +133 -131
- package/dist/types/packages/sdk/src/taskset-draft-core.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-dataset-artifacts.d.ts +80 -80
- package/dist/types/packages/sdk/src/taskset-draft-dataset-sources.d.ts +34 -34
- package/dist/types/packages/sdk/src/taskset-draft-document.d.ts +119 -117
- package/dist/types/packages/sdk/src/taskset-draft-document.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-package-compiler.d.ts +500 -492
- package/dist/types/packages/sdk/src/taskset-draft-package-compiler.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-source.d.ts +236 -234
- package/dist/types/packages/sdk/src/taskset-draft-source.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-draft-workspace.d.ts +228 -224
- package/dist/types/packages/sdk/src/taskset-draft-workspace.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-intake.d.ts +1 -1
- package/dist/types/packages/sdk/src/taskset-package-client.d.ts +4600 -4576
- package/dist/types/packages/sdk/src/taskset-package-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-contracts.d.ts +2068 -2054
- package/dist/types/packages/sdk/src/taskset-package-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-files.d.ts +1 -1
- package/dist/types/packages/sdk/src/taskset-package-grader-projection.d.ts +21 -21
- package/dist/types/packages/sdk/src/taskset-package-learning.d.ts +88 -86
- package/dist/types/packages/sdk/src/taskset-package-learning.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/taskset-package-projection.d.ts +387 -383
- package/dist/types/packages/sdk/src/taskset-package-projection.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-acceptance.d.ts +26 -26
- package/dist/types/packages/sdk/src/training-acceptance.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-bundle-contracts.d.ts +14 -10
- package/dist/types/packages/sdk/src/training-bundle-contracts.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-bundle.d.ts +2 -0
- package/dist/types/packages/sdk/src/training-bundle.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-candidate-decisions.d.ts +2 -2
- package/dist/types/packages/sdk/src/training-candidate-decisions.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-evaluation-results.d.ts +3 -3
- package/dist/types/packages/sdk/src/training-evaluation-source.d.ts +1 -1
- package/dist/types/packages/sdk/src/training-evaluation-source.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-grading-plan.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-harness-sources.d.ts +14 -14
- package/dist/types/packages/sdk/src/training-harness-sources.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-learning-batch.d.ts +344 -340
- package/dist/types/packages/sdk/src/training-learning-batch.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-preparations.d.ts +700 -700
- package/dist/types/packages/sdk/src/training-preparations.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-project-client.d.ts +272 -272
- package/dist/types/packages/sdk/src/training-project-client.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training-project-contracts.d.ts +161 -161
- package/dist/types/packages/sdk/src/training-qualification-preparation.d.ts +75 -71
- package/dist/types/packages/sdk/src/training-qualification-preparation.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/training.d.ts +1442 -1416
- package/dist/types/packages/sdk/src/training.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/work.d.ts +4 -0
- package/dist/types/packages/sdk/src/work.d.ts.map +1 -1
- package/dist/types/packages/sdk/src/workflows.d.ts.map +1 -1
- package/dist/workflows.js +3 -2
- package/dist/workflows.js.map +1 -7
- package/package.json +3 -3
- package/dist/chunk-2DMYAJLS.js.map +0 -7
- package/dist/chunk-2U54EOXJ.js.map +0 -7
- package/dist/chunk-36WJTS5V.js.map +0 -7
- package/dist/chunk-3KXDH6OB.js.map +0 -7
- package/dist/chunk-3LP7DNLQ.js.map +0 -7
- package/dist/chunk-6Q4I33CH.js.map +0 -7
- package/dist/chunk-77N3QEIL.js.map +0 -7
- package/dist/chunk-AG5DDL7T.js.map +0 -7
- package/dist/chunk-BFR3VC25.js.map +0 -7
- package/dist/chunk-BSK3VLM3.js.map +0 -7
- package/dist/chunk-CEZEVILU.js.map +0 -7
- package/dist/chunk-CK3ONRRC.js.map +0 -7
- package/dist/chunk-COPVBJPL.js.map +0 -7
- package/dist/chunk-CTVKI3BV.js.map +0 -7
- package/dist/chunk-CWXM57QR.js.map +0 -7
- package/dist/chunk-DUKW3R7O.js.map +0 -7
- package/dist/chunk-FHCSDXIA.js +0 -1417
- package/dist/chunk-FHCSDXIA.js.map +0 -7
- package/dist/chunk-FY7NHCEH.js.map +0 -7
- package/dist/chunk-G53MA5XX.js.map +0 -7
- package/dist/chunk-G6Y6CQYV.js.map +0 -7
- package/dist/chunk-HKWX624G.js.map +0 -7
- package/dist/chunk-IWT6JUZD.js.map +0 -7
- package/dist/chunk-K3JCP4FO.js.map +0 -7
- package/dist/chunk-K5WRVSOL.js.map +0 -7
- package/dist/chunk-KAE7AMYE.js.map +0 -7
- package/dist/chunk-MEKWESBF.js.map +0 -7
- package/dist/chunk-MLPPSCDQ.js.map +0 -7
- package/dist/chunk-O4GE6VTR.js.map +0 -7
- package/dist/chunk-P27DQ65Z.js.map +0 -7
- package/dist/chunk-QEIT5VXJ.js.map +0 -7
- package/dist/chunk-QTJQIHDZ.js.map +0 -7
- package/dist/chunk-ROBAX5TN.js.map +0 -7
- package/dist/chunk-T4APHDES.js.map +0 -7
- package/dist/chunk-TCHAWJMB.js.map +0 -7
- package/dist/chunk-TMUAKJWX.js.map +0 -7
- package/dist/chunk-TSMGMA6R.js.map +0 -7
- package/dist/chunk-UPS52OPZ.js.map +0 -7
- package/dist/chunk-VY5AFE7A.js.map +0 -7
- package/dist/chunk-XYQD4LDI.js.map +0 -7
- package/dist/chunk-YCQVVFE5.js.map +0 -7
- package/dist/chunk-YUKV7RCD.js.map +0 -7
- package/dist/chunk-YVUIHH6Z.js.map +0 -7
- package/dist/chunk-ZJ3H3VS7.js.map +0 -7
package/dist/chunk-FHCSDXIA.js
DELETED
|
@@ -1,1417 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
ModelTasksetAuthoringSchema
|
|
3
|
-
} from "./chunk-2U54EOXJ.js";
|
|
4
|
-
import {
|
|
5
|
-
ChatModelRefSchema
|
|
6
|
-
} from "./chunk-KAE7AMYE.js";
|
|
7
|
-
import {
|
|
8
|
-
ModelProjectVersionedRefSchema,
|
|
9
|
-
canonicalJson
|
|
10
|
-
} from "./chunk-TCHAWJMB.js";
|
|
11
|
-
import {
|
|
12
|
-
ImmutableAssetRefSchema,
|
|
13
|
-
ImmutableReleaseRefSchema,
|
|
14
|
-
ReleaseHashSchema,
|
|
15
|
-
ReleaseIdSchema,
|
|
16
|
-
contentHash,
|
|
17
|
-
sha256
|
|
18
|
-
} from "./chunk-COPVBJPL.js";
|
|
19
|
-
|
|
20
|
-
// src/connected-dataset-source.ts
|
|
21
|
-
import { z } from "zod";
|
|
22
|
-
import { ConnectedSourceKindSchema } from "@openpond/evals/connected-evidence";
|
|
23
|
-
var ConnectedDatasetSourceRefSchema = z.object({
|
|
24
|
-
schemaVersion: z.literal("openpond.connectedDatasetSource.v1"),
|
|
25
|
-
kind: z.literal("connected_activity"),
|
|
26
|
-
id: z.string().min(1).max(240),
|
|
27
|
-
profileId: z.string().min(1).max(240),
|
|
28
|
-
title: z.string().min(1).max(500),
|
|
29
|
-
sourceHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
30
|
-
occurredAt: z.string().datetime(),
|
|
31
|
-
licensingStatus: z.enum(["pending", "approved", "review", "blocked"]),
|
|
32
|
-
secretScanStatus: z.enum(["pending", "passed", "blocked"]),
|
|
33
|
-
piiScanStatus: z.enum(["pending", "passed", "review", "blocked"]),
|
|
34
|
-
origin: ConnectedSourceKindSchema,
|
|
35
|
-
sourceId: z.string().min(1).max(500),
|
|
36
|
-
sessionId: z.string().min(1).max(500),
|
|
37
|
-
normalizerVersion: z.string().min(1).max(200),
|
|
38
|
-
projection: z.enum(["turn", "conversation"]),
|
|
39
|
-
ownerUserId: z.string().min(1).max(500),
|
|
40
|
-
purpose: z.literal("recorded_evaluation"),
|
|
41
|
-
sourceFileHashes: z.array(z.string().regex(/^[a-f0-9]{64}$/)).max(100).optional(),
|
|
42
|
-
metadata: z.record(z.string(), z.unknown()).default({})
|
|
43
|
-
}).strict();
|
|
44
|
-
|
|
45
|
-
// src/taskset-draft-dataset-artifacts.ts
|
|
46
|
-
import { z as z2 } from "zod";
|
|
47
|
-
var IdSchema = z2.string().trim().min(1).max(240);
|
|
48
|
-
var TimestampSchema = z2.string().trim().min(1);
|
|
49
|
-
var HashSchema = z2.string().trim().min(8).max(256);
|
|
50
|
-
var RelativePathSchema = z2.string().trim().min(1).max(1e3).refine(
|
|
51
|
-
(value) => !value.startsWith("/") && !value.startsWith("\\") && !value.split(/[\\/]/).includes(".."),
|
|
52
|
-
"Artifact paths must be relative and stay inside the Dataset."
|
|
53
|
-
);
|
|
54
|
-
var DatasetSplitSchema = z2.enum([
|
|
55
|
-
"train",
|
|
56
|
-
"validation",
|
|
57
|
-
"test",
|
|
58
|
-
"frozen_eval"
|
|
59
|
-
]);
|
|
60
|
-
var DatasetSemanticFieldSchema = z2.object({
|
|
61
|
-
name: z2.string().trim().min(1).max(500),
|
|
62
|
-
semanticRole: z2.enum([
|
|
63
|
-
"row_id",
|
|
64
|
-
"cluster_id",
|
|
65
|
-
"prompt",
|
|
66
|
-
"messages",
|
|
67
|
-
"demonstration",
|
|
68
|
-
"chosen",
|
|
69
|
-
"rejected",
|
|
70
|
-
"expected_output",
|
|
71
|
-
"privileged_context",
|
|
72
|
-
"reward",
|
|
73
|
-
"feedback",
|
|
74
|
-
"tag",
|
|
75
|
-
"metadata"
|
|
76
|
-
]),
|
|
77
|
-
logicalType: z2.enum([
|
|
78
|
-
"string",
|
|
79
|
-
"integer",
|
|
80
|
-
"float",
|
|
81
|
-
"boolean",
|
|
82
|
-
"json",
|
|
83
|
-
"messages"
|
|
84
|
-
]),
|
|
85
|
-
nullable: z2.boolean(),
|
|
86
|
-
policy: z2.enum(["visible", "privileged", "metadata"])
|
|
87
|
-
});
|
|
88
|
-
var DatasetSemanticSchemaSchema = z2.object({
|
|
89
|
-
schemaVersion: z2.literal("openpond.datasetSemanticSchema.v1"),
|
|
90
|
-
fields: z2.array(DatasetSemanticFieldSchema).min(1).max(1e4),
|
|
91
|
-
schemaHash: HashSchema
|
|
92
|
-
});
|
|
93
|
-
var DatasetShardRefSchema = z2.object({
|
|
94
|
-
id: IdSchema,
|
|
95
|
-
split: DatasetSplitSchema,
|
|
96
|
-
path: RelativePathSchema,
|
|
97
|
-
contentHash: HashSchema,
|
|
98
|
-
schemaHash: HashSchema,
|
|
99
|
-
sizeBytes: z2.number().int().nonnegative(),
|
|
100
|
-
rowCount: z2.number().int().positive(),
|
|
101
|
-
rowGroupCount: z2.number().int().positive()
|
|
102
|
-
});
|
|
103
|
-
var DatasetArtifactManifestSchema = z2.object({
|
|
104
|
-
schemaVersion: z2.literal("openpond.datasetArtifact.v1"),
|
|
105
|
-
id: IdSchema,
|
|
106
|
-
tasksetId: IdSchema,
|
|
107
|
-
tasksetRevision: z2.number().int().positive(),
|
|
108
|
-
contentHash: HashSchema,
|
|
109
|
-
format: z2.literal("parquet"),
|
|
110
|
-
schema: DatasetSemanticSchemaSchema,
|
|
111
|
-
shards: z2.array(DatasetShardRefSchema).min(1).max(1e5),
|
|
112
|
-
rowCount: z2.number().int().positive(),
|
|
113
|
-
splitCounts: z2.record(DatasetSplitSchema, z2.number().int().nonnegative()),
|
|
114
|
-
sourceReceiptRefs: z2.array(IdSchema).min(1).max(1e5),
|
|
115
|
-
mappingHash: HashSchema,
|
|
116
|
-
qualityReportHash: HashSchema,
|
|
117
|
-
createdAt: TimestampSchema
|
|
118
|
-
});
|
|
119
|
-
var DatasetArtifactSummarySchema = z2.object({
|
|
120
|
-
schemaVersion: z2.literal("openpond.datasetArtifactSummary.v1"),
|
|
121
|
-
artifactId: IdSchema,
|
|
122
|
-
tasksetId: IdSchema,
|
|
123
|
-
tasksetRevision: z2.number().int().positive(),
|
|
124
|
-
format: z2.literal("parquet"),
|
|
125
|
-
rowCount: z2.number().int().positive(),
|
|
126
|
-
splitCounts: z2.record(DatasetSplitSchema, z2.number().int().nonnegative()),
|
|
127
|
-
sizeBytes: z2.number().int().nonnegative(),
|
|
128
|
-
contentHash: HashSchema,
|
|
129
|
-
available: z2.boolean(),
|
|
130
|
-
unavailableReason: z2.string().trim().min(1).max(2e3).nullable(),
|
|
131
|
-
createdAt: TimestampSchema
|
|
132
|
-
});
|
|
133
|
-
var DatasetCatalogItemSchema = z2.object({
|
|
134
|
-
schemaVersion: z2.literal("openpond.datasetCatalogItem.v1"),
|
|
135
|
-
tasksetId: IdSchema,
|
|
136
|
-
tasksetRevision: z2.number().int().positive(),
|
|
137
|
-
artifactId: IdSchema.nullable(),
|
|
138
|
-
name: z2.string().trim().min(1).max(500),
|
|
139
|
-
status: z2.enum([
|
|
140
|
-
"draft",
|
|
141
|
-
"awaiting_disclosure_approval",
|
|
142
|
-
"awaiting_materialization_approval",
|
|
143
|
-
"materializing",
|
|
144
|
-
"validating",
|
|
145
|
-
"needs_review",
|
|
146
|
-
"baselining",
|
|
147
|
-
"ready",
|
|
148
|
-
"blocked",
|
|
149
|
-
"failed",
|
|
150
|
-
"archived"
|
|
151
|
-
]),
|
|
152
|
-
storageKind: z2.enum(["inline", "parquet"]),
|
|
153
|
-
rowCount: z2.number().int().nonnegative(),
|
|
154
|
-
splitCounts: z2.record(DatasetSplitSchema, z2.number().int().nonnegative()),
|
|
155
|
-
sizeBytes: z2.number().int().nonnegative().nullable(),
|
|
156
|
-
available: z2.boolean(),
|
|
157
|
-
unavailableReason: z2.string().trim().min(1).max(2e3).nullable(),
|
|
158
|
-
createdAt: TimestampSchema,
|
|
159
|
-
updatedAt: TimestampSchema
|
|
160
|
-
});
|
|
161
|
-
var DatasetCatalogResponseSchema = z2.object({
|
|
162
|
-
schemaVersion: z2.literal("openpond.datasetCatalog.v1"),
|
|
163
|
-
profileId: IdSchema,
|
|
164
|
-
datasets: z2.array(DatasetCatalogItemSchema),
|
|
165
|
-
generatedAt: TimestampSchema
|
|
166
|
-
});
|
|
167
|
-
var DatasetArtifactRegistryEntrySchema = z2.object({
|
|
168
|
-
schemaVersion: z2.literal("openpond.datasetArtifactRegistry.v1"),
|
|
169
|
-
manifest: DatasetArtifactManifestSchema,
|
|
170
|
-
profileId: IdSchema,
|
|
171
|
-
storageRoot: z2.string().trim().min(1).max(4e3),
|
|
172
|
-
relativeManifestPath: RelativePathSchema,
|
|
173
|
-
available: z2.boolean(),
|
|
174
|
-
unavailableReason: z2.string().trim().min(1).max(2e3).nullable(),
|
|
175
|
-
verifiedAt: TimestampSchema.nullable(),
|
|
176
|
-
createdAt: TimestampSchema,
|
|
177
|
-
updatedAt: TimestampSchema
|
|
178
|
-
});
|
|
179
|
-
var DatasetRowPageRequestSchema = z2.object({
|
|
180
|
-
split: DatasetSplitSchema.nullable().default(null),
|
|
181
|
-
cursor: z2.string().trim().min(1).max(2e3).nullable().default(null),
|
|
182
|
-
limit: z2.number().int().min(1).max(100).default(25),
|
|
183
|
-
columns: z2.array(z2.string().trim().min(1).max(500)).max(100).default([])
|
|
184
|
-
});
|
|
185
|
-
var DatasetRowPageSchema = z2.object({
|
|
186
|
-
schemaVersion: z2.literal("openpond.datasetRowPage.v1"),
|
|
187
|
-
tasksetId: IdSchema,
|
|
188
|
-
tasksetRevision: z2.number().int().positive(),
|
|
189
|
-
artifactHash: HashSchema,
|
|
190
|
-
split: DatasetSplitSchema.nullable(),
|
|
191
|
-
rows: z2.array(z2.record(z2.string(), z2.unknown())).max(100),
|
|
192
|
-
nextCursor: z2.string().max(2e3).nullable(),
|
|
193
|
-
totalRows: z2.number().int().nonnegative(),
|
|
194
|
-
returnedRows: z2.number().int().nonnegative()
|
|
195
|
-
});
|
|
196
|
-
var DatasetStorageSettingsSchema = z2.object({
|
|
197
|
-
schemaVersion: z2.literal("openpond.datasetStorageSettings.v1"),
|
|
198
|
-
datasetStorePath: z2.string().trim().min(1).max(4e3),
|
|
199
|
-
updatedAt: TimestampSchema
|
|
200
|
-
});
|
|
201
|
-
var DatasetStorageRootSchema = z2.object({
|
|
202
|
-
id: IdSchema,
|
|
203
|
-
label: z2.string().trim().min(1).max(500),
|
|
204
|
-
path: z2.string().trim().min(1).max(4e3),
|
|
205
|
-
datasetStorePath: z2.string().trim().min(1).max(4e3),
|
|
206
|
-
kind: z2.enum(["local", "network", "removable", "cache"]),
|
|
207
|
-
configured: z2.boolean(),
|
|
208
|
-
mounted: z2.boolean(),
|
|
209
|
-
writable: z2.boolean(),
|
|
210
|
-
totalBytes: z2.number().int().nonnegative().nullable(),
|
|
211
|
-
freeBytes: z2.number().int().nonnegative().nullable()
|
|
212
|
-
});
|
|
213
|
-
var DatasetStorageStateSchema = z2.object({
|
|
214
|
-
schemaVersion: z2.literal("openpond.datasetStorageState.v1"),
|
|
215
|
-
settings: DatasetStorageSettingsSchema,
|
|
216
|
-
storageRoots: z2.array(DatasetStorageRootSchema)
|
|
217
|
-
});
|
|
218
|
-
var UpdateDatasetStorageSettingsRequestSchema = z2.object({
|
|
219
|
-
datasetStorePath: z2.string().trim().min(1).max(4e3)
|
|
220
|
-
});
|
|
221
|
-
|
|
222
|
-
// src/taskset-draft-dataset-sources.ts
|
|
223
|
-
import { z as z3 } from "zod";
|
|
224
|
-
var IdSchema2 = z3.string().trim().min(1).max(240);
|
|
225
|
-
var TimestampSchema2 = z3.string().trim().min(1);
|
|
226
|
-
var HashSchema2 = z3.string().trim().min(8).max(256);
|
|
227
|
-
var MetadataSchema = z3.record(z3.string(), z3.unknown()).default({});
|
|
228
|
-
var RevisionSchema = z3.string().trim().min(7).max(256);
|
|
229
|
-
var DatasetSourceReviewStatusSchema = z3.enum([
|
|
230
|
-
"pending",
|
|
231
|
-
"approved",
|
|
232
|
-
"review",
|
|
233
|
-
"blocked"
|
|
234
|
-
]);
|
|
235
|
-
var ExternalDatasetSourceBaseSchema = z3.object({
|
|
236
|
-
id: IdSchema2,
|
|
237
|
-
profileId: IdSchema2,
|
|
238
|
-
title: z3.string().trim().min(1).max(500),
|
|
239
|
-
sourceHash: HashSchema2,
|
|
240
|
-
occurredAt: TimestampSchema2,
|
|
241
|
-
licensingStatus: DatasetSourceReviewStatusSchema,
|
|
242
|
-
secretScanStatus: z3.enum(["pending", "passed", "blocked"]),
|
|
243
|
-
piiScanStatus: z3.enum(["pending", "passed", "review", "blocked"]),
|
|
244
|
-
metadata: MetadataSchema
|
|
245
|
-
});
|
|
246
|
-
var HuggingFaceDatasetSourceRefSchema = ExternalDatasetSourceBaseSchema.extend({
|
|
247
|
-
schemaVersion: z3.literal("openpond.huggingFaceDatasetSource.v1"),
|
|
248
|
-
kind: z3.literal("huggingface"),
|
|
249
|
-
repositoryId: z3.string().trim().regex(/^[^/\s]+\/[^/\s]+$/).max(500),
|
|
250
|
-
repositoryUrl: z3.string().url().max(2e3),
|
|
251
|
-
revision: RevisionSchema,
|
|
252
|
-
configuration: z3.string().trim().min(1).max(500),
|
|
253
|
-
upstreamSplits: z3.array(z3.string().trim().min(1).max(500)).min(1).max(1e3),
|
|
254
|
-
gated: z3.boolean(),
|
|
255
|
-
private: z3.boolean(),
|
|
256
|
-
declaredLicense: z3.string().trim().min(1).max(500).nullable(),
|
|
257
|
-
sourceFileHashes: z3.array(HashSchema2).min(1).max(1e5)
|
|
258
|
-
});
|
|
259
|
-
var UploadedFileDatasetSourceRefSchema = ExternalDatasetSourceBaseSchema.extend({
|
|
260
|
-
schemaVersion: z3.literal("openpond.uploadedFileDatasetSource.v1"),
|
|
261
|
-
kind: z3.literal("uploaded_file"),
|
|
262
|
-
originalFileNames: z3.array(z3.string().trim().min(1).max(500)).min(1).max(1e3),
|
|
263
|
-
mediaTypes: z3.array(z3.string().trim().min(1).max(200)).min(1).max(1e3),
|
|
264
|
-
sourceFileHashes: z3.array(HashSchema2).min(1).max(1e3),
|
|
265
|
-
totalBytes: z3.number().int().nonnegative(),
|
|
266
|
-
parserVersion: z3.string().trim().min(1).max(200)
|
|
267
|
-
});
|
|
268
|
-
var GeneratedDatasetSourceRefSchema = ExternalDatasetSourceBaseSchema.extend({
|
|
269
|
-
schemaVersion: z3.literal("openpond.generatedDatasetSource.v1"),
|
|
270
|
-
kind: z3.literal("generated"),
|
|
271
|
-
generatorId: IdSchema2,
|
|
272
|
-
generatorVersion: z3.string().trim().min(1).max(200),
|
|
273
|
-
seed: z3.number().int(),
|
|
274
|
-
generatorHash: HashSchema2
|
|
275
|
-
});
|
|
276
|
-
var LearningBatchDatasetSourceRefSchema = ExternalDatasetSourceBaseSchema.extend({
|
|
277
|
-
schemaVersion: z3.literal("openpond.learningBatchDatasetSource.v1"),
|
|
278
|
-
kind: z3.literal("learning_batch"),
|
|
279
|
-
batch: z3.object({ id: IdSchema2, revision: z3.number().int().positive(), contentHash: HashSchema2 }).strict(),
|
|
280
|
-
taskDefinition: z3.object({ id: IdSchema2, revision: z3.number().int().positive(), contentHash: HashSchema2 }).strict(),
|
|
281
|
-
admittedBy: IdSchema2
|
|
282
|
-
});
|
|
283
|
-
var ExternalDatasetSourceRefSchema = z3.discriminatedUnion("kind", [
|
|
284
|
-
HuggingFaceDatasetSourceRefSchema,
|
|
285
|
-
UploadedFileDatasetSourceRefSchema,
|
|
286
|
-
GeneratedDatasetSourceRefSchema,
|
|
287
|
-
LearningBatchDatasetSourceRefSchema
|
|
288
|
-
]);
|
|
289
|
-
|
|
290
|
-
// src/taskset-draft-harness-actions.ts
|
|
291
|
-
import { z as z4 } from "zod";
|
|
292
|
-
var HarnessActionBindingSchema = z4.object({
|
|
293
|
-
schemaVersion: z4.literal("openpond.harnessActionBinding.v1"),
|
|
294
|
-
actionId: ReleaseIdSchema,
|
|
295
|
-
modelToolName: z4.string().trim().min(1).max(64).regex(/^[a-zA-Z][a-zA-Z0-9_-]*$/),
|
|
296
|
-
description: z4.string().trim().min(1).max(1e3),
|
|
297
|
-
inputSchema: z4.record(z4.string(), z4.unknown()),
|
|
298
|
-
actionSchemaHash: ReleaseHashSchema,
|
|
299
|
-
agentRelease: ImmutableReleaseRefSchema,
|
|
300
|
-
implementationHash: ReleaseHashSchema,
|
|
301
|
-
runtimeBindingId: ReleaseIdSchema,
|
|
302
|
-
capabilityReceiptHash: ReleaseHashSchema,
|
|
303
|
-
sideEffect: z4.enum(["read", "write"]),
|
|
304
|
-
studentVisible: z4.boolean(),
|
|
305
|
-
timeoutMs: z4.number().int().positive().max(36e5),
|
|
306
|
-
episodeArgumentBindings: z4.array(
|
|
307
|
-
z4.object({
|
|
308
|
-
argument: ReleaseIdSchema,
|
|
309
|
-
source: z4.literal("case_id")
|
|
310
|
-
}).strict()
|
|
311
|
-
).max(20).default([])
|
|
312
|
-
}).strict();
|
|
313
|
-
|
|
314
|
-
// src/taskset-draft-core.ts
|
|
315
|
-
import { z as z5 } from "zod";
|
|
316
|
-
import { CustomVerifierRuntimeSchema } from "@openpond/evals/tasksets";
|
|
317
|
-
import { HumanFormSchema } from "@openpond/evals/human-review";
|
|
318
|
-
var IdSchema3 = z5.string().trim().min(1).max(240);
|
|
319
|
-
var TimestampSchema3 = z5.string().trim().min(1);
|
|
320
|
-
var HashSchema3 = z5.string().trim().min(8).max(256);
|
|
321
|
-
var Sha256Schema = z5.string().trim().regex(/^[a-f0-9]{64}$/);
|
|
322
|
-
var CodeIdentifierSchema = z5.string().trim().regex(/^[A-Za-z_$][A-Za-z0-9_$]*$/);
|
|
323
|
-
var MetadataSchema2 = z5.record(z5.string(), z5.unknown()).default({});
|
|
324
|
-
var NullableIdSchema = IdSchema3.nullable();
|
|
325
|
-
function safeRelativeFilePath(value) {
|
|
326
|
-
const normalized = value.trim().replaceAll("\\", "/");
|
|
327
|
-
if (!normalized || normalized.startsWith("/") || normalized === "." || normalized === "..") {
|
|
328
|
-
return false;
|
|
329
|
-
}
|
|
330
|
-
return !normalized.split("/").some(
|
|
331
|
-
(segment) => !segment || segment === "." || segment === ".."
|
|
332
|
-
);
|
|
333
|
-
}
|
|
334
|
-
function safeFileName(value) {
|
|
335
|
-
const normalized = value.trim();
|
|
336
|
-
return normalized.length > 0 && normalized !== "." && normalized !== ".." && !normalized.includes("/") && !normalized.includes("\\") && !normalized.includes("\0");
|
|
337
|
-
}
|
|
338
|
-
var TasksetSplitSchema = DatasetSplitSchema;
|
|
339
|
-
var TasksetPurposeSchema = z5.enum(["general", "benchmark"]);
|
|
340
|
-
var TasksetBenchmarkBindingSchema = z5.object({
|
|
341
|
-
schemaVersion: z5.literal("openpond.tasksetBenchmark.v1"),
|
|
342
|
-
definitionId: IdSchema3,
|
|
343
|
-
releaseId: IdSchema3,
|
|
344
|
-
releaseHash: Sha256Schema,
|
|
345
|
-
managedReleasePath: z5.string().trim().min(1).max(1e3).refine(safeRelativeFilePath, "Benchmark release paths must remain relative."),
|
|
346
|
-
adaptationSplit: TasksetSplitSchema,
|
|
347
|
-
evaluationSplit: TasksetSplitSchema,
|
|
348
|
-
primaryMetric: z5.enum([
|
|
349
|
-
"foreground_tokens",
|
|
350
|
-
"success_rate",
|
|
351
|
-
"latency_ms",
|
|
352
|
-
"cost_usd"
|
|
353
|
-
]),
|
|
354
|
-
qualityGate: z5.enum(["none", "non_regression", "all_pass"]),
|
|
355
|
-
source: z5.enum(["builtin", "imported"]),
|
|
356
|
-
metadata: MetadataSchema2
|
|
357
|
-
});
|
|
358
|
-
var TasksetPreferenceComparisonBindingSchema = z5.object({
|
|
359
|
-
schemaVersion: z5.literal("openpond.tasksetPreferenceComparisonBinding.v1"),
|
|
360
|
-
releaseId: IdSchema3,
|
|
361
|
-
releaseHash: Sha256Schema,
|
|
362
|
-
publishedAt: TimestampSchema3,
|
|
363
|
-
metadata: MetadataSchema2
|
|
364
|
-
});
|
|
365
|
-
var TrainingSourceConsentSchema = z5.object({
|
|
366
|
-
status: z5.enum(["pending", "granted", "denied", "revoked"]),
|
|
367
|
-
scope: z5.enum(["metadata_only", "selected_turns", "full_session"]),
|
|
368
|
-
grantedBy: NullableIdSchema,
|
|
369
|
-
grantedAt: TimestampSchema3.nullable(),
|
|
370
|
-
purpose: z5.literal("task_authoring_and_evaluation")
|
|
371
|
-
});
|
|
372
|
-
var TrainingSourceRefSchema = z5.object({
|
|
373
|
-
schemaVersion: z5.literal("openpond.trainingSource.v1"),
|
|
374
|
-
id: IdSchema3,
|
|
375
|
-
profileId: IdSchema3,
|
|
376
|
-
sessionId: IdSchema3,
|
|
377
|
-
turnIds: z5.array(IdSchema3).max(1e3).default([]),
|
|
378
|
-
workspaceId: NullableIdSchema,
|
|
379
|
-
sourceHash: HashSchema3,
|
|
380
|
-
clusterKey: IdSchema3,
|
|
381
|
-
title: z5.string().trim().min(1).max(500),
|
|
382
|
-
occurredAt: TimestampSchema3,
|
|
383
|
-
consent: TrainingSourceConsentSchema,
|
|
384
|
-
connectedAppIds: z5.array(IdSchema3).max(100).default([]),
|
|
385
|
-
secretScanStatus: z5.enum(["pending", "passed", "blocked"]),
|
|
386
|
-
piiScanStatus: z5.enum(["pending", "passed", "review", "blocked"]),
|
|
387
|
-
licensingStatus: z5.enum(["pending", "approved", "review", "blocked"]),
|
|
388
|
-
metadata: MetadataSchema2
|
|
389
|
-
});
|
|
390
|
-
var TasksetSourceRefSchema = z5.union([
|
|
391
|
-
TrainingSourceRefSchema,
|
|
392
|
-
ExternalDatasetSourceRefSchema,
|
|
393
|
-
ConnectedDatasetSourceRefSchema
|
|
394
|
-
]);
|
|
395
|
-
var TaskPolicyBoundarySchema = z5.object({
|
|
396
|
-
policyVisibleFields: z5.array(IdSchema3).max(1e3).default([]),
|
|
397
|
-
privilegedFields: z5.array(IdSchema3).max(1e3).default([]),
|
|
398
|
-
hiddenGraderRefs: z5.array(IdSchema3).max(100).default([]),
|
|
399
|
-
connectedAppScopes: z5.array(IdSchema3).max(100).default([])
|
|
400
|
-
});
|
|
401
|
-
var TaskAssetRefSchema = z5.object({
|
|
402
|
-
id: IdSchema3,
|
|
403
|
-
sourceRefId: IdSchema3,
|
|
404
|
-
artifactRef: z5.string().trim().min(1).max(4e3).refine(safeRelativeFilePath, "Task asset references must be safe relative paths."),
|
|
405
|
-
fileName: z5.string().trim().min(1).max(500).refine(safeFileName, "Task asset file names must not contain path separators."),
|
|
406
|
-
stagingPath: z5.string().trim().min(1).max(1e3).refine(safeRelativeFilePath, "Task asset staging paths must stay inside Work inputs.").optional(),
|
|
407
|
-
mediaType: z5.string().trim().min(1).max(200),
|
|
408
|
-
sha256: Sha256Schema,
|
|
409
|
-
sizeBytes: z5.number().int().nonnegative().max(25e7),
|
|
410
|
-
split: TasksetSplitSchema,
|
|
411
|
-
metadata: MetadataSchema2
|
|
412
|
-
}).superRefine((value, ctx) => {
|
|
413
|
-
if (value.stagingPath && value.stagingPath.split("/").at(-1) !== value.fileName)
|
|
414
|
-
ctx.addIssue({ code: "custom", message: "Staged task assets must preserve their source file name.", path: ["stagingPath"] });
|
|
415
|
-
});
|
|
416
|
-
var TaskRequiredOutputSchema = z5.object({
|
|
417
|
-
path: z5.string().trim().min(1).max(1e3).refine(safeRelativeFilePath, "Required output paths must stay inside the Work output directory."),
|
|
418
|
-
mediaType: z5.string().trim().min(1).max(200),
|
|
419
|
-
schemaRef: IdSchema3.nullable().optional(),
|
|
420
|
-
maxBytes: z5.number().int().positive().max(1e7).optional(),
|
|
421
|
-
metadata: MetadataSchema2
|
|
422
|
-
});
|
|
423
|
-
var TaskDataRecordSchema = z5.object({
|
|
424
|
-
schemaVersion: z5.literal("openpond.taskData.v1"),
|
|
425
|
-
id: IdSchema3,
|
|
426
|
-
clusterKey: IdSchema3,
|
|
427
|
-
split: TasksetSplitSchema,
|
|
428
|
-
input: z5.record(z5.string(), z5.unknown()),
|
|
429
|
-
expectedOutput: z5.record(z5.string(), z5.unknown()).nullable(),
|
|
430
|
-
policyVisibleContext: z5.record(z5.string(), z5.unknown()).default({}),
|
|
431
|
-
privilegedContextRef: NullableIdSchema,
|
|
432
|
-
sourceRefs: z5.array(IdSchema3).min(1).max(100),
|
|
433
|
-
assets: z5.array(TaskAssetRefSchema).max(1e3).optional(),
|
|
434
|
-
resourceRefs: z5.array(IdSchema3).max(1e3).optional(),
|
|
435
|
-
requiredOutputs: z5.array(TaskRequiredOutputSchema).max(100).optional(),
|
|
436
|
-
tags: z5.array(IdSchema3).max(100).default([]),
|
|
437
|
-
metadata: MetadataSchema2
|
|
438
|
-
});
|
|
439
|
-
var TasksetEnvironmentResourceSchema = z5.object({
|
|
440
|
-
id: IdSchema3,
|
|
441
|
-
kind: z5.enum(["file", "catalog", "configuration", "code_module"]),
|
|
442
|
-
path: z5.string().trim().min(1).max(1e3).refine(safeRelativeFilePath, "Environment resource paths must remain relative."),
|
|
443
|
-
mediaType: z5.string().trim().min(1).max(200).nullable().optional(),
|
|
444
|
-
visibility: z5.enum(["policy_visible", "policy_hidden", "privileged"]),
|
|
445
|
-
required: z5.boolean(),
|
|
446
|
-
metadata: MetadataSchema2
|
|
447
|
-
});
|
|
448
|
-
var LearningSignalBaseSchema = z5.object({
|
|
449
|
-
id: IdSchema3,
|
|
450
|
-
taskId: NullableIdSchema,
|
|
451
|
-
sourceRefs: z5.array(IdSchema3).min(1).max(100),
|
|
452
|
-
artifactRef: IdSchema3,
|
|
453
|
-
approved: z5.boolean(),
|
|
454
|
-
confidence: z5.number().min(0).max(1),
|
|
455
|
-
metadata: MetadataSchema2
|
|
456
|
-
});
|
|
457
|
-
var DemonstrationSignalSchema = LearningSignalBaseSchema.extend({
|
|
458
|
-
kind: z5.literal("demonstration"),
|
|
459
|
-
prompt: z5.string().max(1e5).nullable().default(null),
|
|
460
|
-
response: z5.string().max(2e5).nullable().default(null)
|
|
461
|
-
});
|
|
462
|
-
var PreferenceSignalSchema = LearningSignalBaseSchema.extend({
|
|
463
|
-
kind: z5.literal("preference"),
|
|
464
|
-
prompt: z5.string().max(1e5),
|
|
465
|
-
chosen: z5.string().max(2e5),
|
|
466
|
-
rejected: z5.string().max(2e5),
|
|
467
|
-
rationale: z5.string().max(1e5).nullable().default(null)
|
|
468
|
-
});
|
|
469
|
-
var CorrectionSignalSchema = LearningSignalBaseSchema.extend({
|
|
470
|
-
kind: z5.literal("correction"),
|
|
471
|
-
original: z5.string().max(2e5),
|
|
472
|
-
corrected: z5.string().max(2e5),
|
|
473
|
-
rationale: z5.string().max(1e5).nullable().default(null)
|
|
474
|
-
});
|
|
475
|
-
var FeedbackSignalSchema = LearningSignalBaseSchema.extend({
|
|
476
|
-
kind: z5.literal("feedback"),
|
|
477
|
-
feedback: z5.string().max(1e5),
|
|
478
|
-
polarity: z5.enum(["positive", "negative", "mixed", "neutral"])
|
|
479
|
-
});
|
|
480
|
-
var RewardSignalSchema = LearningSignalBaseSchema.extend({
|
|
481
|
-
kind: z5.literal("reward"),
|
|
482
|
-
task: z5.string().max(1e5),
|
|
483
|
-
rules: z5.array(z5.object({
|
|
484
|
-
id: IdSchema3,
|
|
485
|
-
points: z5.number().finite(),
|
|
486
|
-
condition: z5.string().trim().min(1).max(1e5)
|
|
487
|
-
})).min(1).max(1e3),
|
|
488
|
-
otherwisePoints: z5.number().finite(),
|
|
489
|
-
executable: z5.boolean()
|
|
490
|
-
});
|
|
491
|
-
var LabelSignalSchema = LearningSignalBaseSchema.extend({
|
|
492
|
-
kind: z5.literal("label"),
|
|
493
|
-
labelKind: z5.literal("rubric"),
|
|
494
|
-
task: z5.string().max(1e5),
|
|
495
|
-
criteria: z5.array(z5.object({
|
|
496
|
-
id: IdSchema3,
|
|
497
|
-
label: z5.string().trim().min(1).max(500),
|
|
498
|
-
description: z5.string().trim().min(1).max(1e5)
|
|
499
|
-
})).min(1).max(1e3),
|
|
500
|
-
calibrationExamples: z5.object({
|
|
501
|
-
positive: z5.string().trim().min(1).max(2e5),
|
|
502
|
-
negative: z5.string().trim().min(1).max(2e5),
|
|
503
|
-
boundary: z5.string().trim().min(1).max(2e5)
|
|
504
|
-
})
|
|
505
|
-
});
|
|
506
|
-
var LearningSignalRefSchema = z5.discriminatedUnion("kind", [
|
|
507
|
-
DemonstrationSignalSchema,
|
|
508
|
-
PreferenceSignalSchema,
|
|
509
|
-
CorrectionSignalSchema,
|
|
510
|
-
FeedbackSignalSchema,
|
|
511
|
-
RewardSignalSchema,
|
|
512
|
-
LabelSignalSchema
|
|
513
|
-
]);
|
|
514
|
-
var LearningSignalInventorySchema = z5.object({
|
|
515
|
-
demonstrations: z5.array(DemonstrationSignalSchema).max(1e5).default([]),
|
|
516
|
-
preferences: z5.array(PreferenceSignalSchema).max(1e5).default([]),
|
|
517
|
-
corrections: z5.array(CorrectionSignalSchema).max(1e5).default([]),
|
|
518
|
-
feedback: z5.array(FeedbackSignalSchema).max(1e5).default([]),
|
|
519
|
-
rewards: z5.array(RewardSignalSchema).max(1e5).default([]),
|
|
520
|
-
labels: z5.array(LabelSignalSchema).max(1e5).default([])
|
|
521
|
-
});
|
|
522
|
-
var TasksetEnvironmentContractSchema = z5.object({
|
|
523
|
-
protocolVersion: z5.literal("openpond.taskEnvironment.v1"),
|
|
524
|
-
kind: z5.enum(["chat", "agent", "program", "stateful_harness", "work"]),
|
|
525
|
-
entrypoint: z5.string().trim().min(1).max(1e3),
|
|
526
|
-
stateful: z5.boolean(),
|
|
527
|
-
deterministicSeeds: z5.boolean(),
|
|
528
|
-
toolNames: z5.array(IdSchema3).max(200).default([]),
|
|
529
|
-
actionBindings: z5.array(HarnessActionBindingSchema).max(200).optional(),
|
|
530
|
-
lifecycle: z5.array(z5.enum(["create", "reset", "step", "grade", "cleanup"])).min(1),
|
|
531
|
-
defaultTimeoutMs: z5.number().int().positive().max(36e5),
|
|
532
|
-
networkPolicy: z5.enum(["none", "declared_read_only", "declared_scoped"]),
|
|
533
|
-
resources: z5.array(TasksetEnvironmentResourceSchema).max(1e4).optional(),
|
|
534
|
-
metadata: MetadataSchema2
|
|
535
|
-
});
|
|
536
|
-
var TasksetCapabilityManifestSchema = z5.object({
|
|
537
|
-
schemaVersion: z5.literal("openpond.tasksetCapabilities.v1"),
|
|
538
|
-
taskKind: z5.enum(["chat", "single_agent", "multi_agent", "custom_program"]),
|
|
539
|
-
supportedSignals: z5.array(z5.enum(["demonstration", "preference", "correction", "feedback", "reward", "label"])),
|
|
540
|
-
compatibleMethods: z5.array(z5.enum(["none", "retrieval", "sft", "dpo", "grpo", "ppo", "sdft", "opd", "opsd", "sdpo"])),
|
|
541
|
-
rewardKinds: z5.array(z5.enum(["none", "exact", "deterministic", "model_judge", "human"])),
|
|
542
|
-
requiresTools: z5.boolean(),
|
|
543
|
-
requiresState: z5.boolean(),
|
|
544
|
-
requiresPrivilegedGrading: z5.boolean(),
|
|
545
|
-
environmentPlacements: z5.array(z5.enum(["local", "remote", "colocated", "provider_native"])),
|
|
546
|
-
exportable: z5.boolean(),
|
|
547
|
-
portabilityBlockers: z5.array(z5.string().trim().min(1).max(2e3)).default([])
|
|
548
|
-
});
|
|
549
|
-
var GraderBaseSchema = z5.object({
|
|
550
|
-
id: IdSchema3,
|
|
551
|
-
taskIds: z5.array(IdSchema3).min(1).max(1e4).refine((ids) => new Set(ids).size === ids.length, "Grader task IDs must be unique.").optional(),
|
|
552
|
-
version: z5.string().trim().min(1).max(100),
|
|
553
|
-
label: z5.string().trim().min(1).max(500),
|
|
554
|
-
weight: z5.number().min(0).max(1e3).default(1),
|
|
555
|
-
hardGate: z5.boolean().default(false),
|
|
556
|
-
rewardEligible: z5.boolean().default(false),
|
|
557
|
-
privileged: z5.boolean().default(false),
|
|
558
|
-
metadata: MetadataSchema2
|
|
559
|
-
});
|
|
560
|
-
var DeterministicGraderSpecSchema = GraderBaseSchema.extend({
|
|
561
|
-
kind: z5.enum(["content", "schema", "file", "diff", "test", "runtime_event", "state"]),
|
|
562
|
-
config: z5.record(z5.string(), z5.unknown())
|
|
563
|
-
});
|
|
564
|
-
var RubricGraderSpecSchema = GraderBaseSchema.extend({
|
|
565
|
-
kind: z5.literal("model_judge"),
|
|
566
|
-
rubric: z5.string().trim().min(1).max(5e4),
|
|
567
|
-
judge: ChatModelRefSchema,
|
|
568
|
-
calibrationFixtureRefs: z5.array(IdSchema3).min(1).max(500),
|
|
569
|
-
calibrationStatus: z5.enum(["pending", "passed", "failed"]),
|
|
570
|
-
temperature: z5.number().min(0).max(2).default(0)
|
|
571
|
-
});
|
|
572
|
-
var HumanGraderSpecSchema = GraderBaseSchema.extend({
|
|
573
|
-
kind: z5.literal("human"),
|
|
574
|
-
form: HumanFormSchema.optional(),
|
|
575
|
-
rubric: z5.string().trim().min(1).max(5e4),
|
|
576
|
-
reviewerRole: z5.string().trim().min(1).max(500)
|
|
577
|
-
});
|
|
578
|
-
var CustomVerifierGraderSpecSchema = GraderBaseSchema.extend({
|
|
579
|
-
kind: z5.literal("custom_verifier"),
|
|
580
|
-
runtime: CustomVerifierRuntimeSchema.optional(),
|
|
581
|
-
module: z5.string().trim().min(1).max(1e3).refine(safeRelativeFilePath, "Custom verifier modules must use a safe relative path."),
|
|
582
|
-
exportName: CodeIdentifierSchema,
|
|
583
|
-
timeoutMs: z5.number().int().positive().max(3e5),
|
|
584
|
-
networkPolicy: z5.literal("none")
|
|
585
|
-
});
|
|
586
|
-
var GraderSpecSchema = z5.union([
|
|
587
|
-
DeterministicGraderSpecSchema,
|
|
588
|
-
RubricGraderSpecSchema,
|
|
589
|
-
HumanGraderSpecSchema,
|
|
590
|
-
CustomVerifierGraderSpecSchema
|
|
591
|
-
]);
|
|
592
|
-
var GraderFixtureLabelSchema = z5.enum([
|
|
593
|
-
"positive",
|
|
594
|
-
"negative",
|
|
595
|
-
"boundary",
|
|
596
|
-
"adversarial",
|
|
597
|
-
"prompt_injection",
|
|
598
|
-
"infrastructure_failure"
|
|
599
|
-
]);
|
|
600
|
-
var GraderFixtureSchema = z5.object({
|
|
601
|
-
id: IdSchema3,
|
|
602
|
-
taskId: IdSchema3,
|
|
603
|
-
label: GraderFixtureLabelSchema,
|
|
604
|
-
output: z5.record(z5.string(), z5.unknown()),
|
|
605
|
-
infrastructureError: z5.string().trim().min(1).max(1e4).nullable(),
|
|
606
|
-
expectedPassed: z5.boolean(),
|
|
607
|
-
expectedRewardEligible: z5.boolean(),
|
|
608
|
-
metadata: MetadataSchema2
|
|
609
|
-
});
|
|
610
|
-
|
|
611
|
-
// src/model-starter-execution.ts
|
|
612
|
-
import { z as z6 } from "zod";
|
|
613
|
-
import { EnvironmentReleaseSchema, VerifierSetReleaseSchema } from "@openpond/evals";
|
|
614
|
-
import { JavaScriptEnvironmentDefinitionSchema, assertJavaScriptEnvironmentDefinition } from "@openpond/evals/javascript-environment/contracts";
|
|
615
|
-
import { createLearningTextAsset, learningRef, sealLearningContent, verifyLearningTextAsset } from "@openpond/evals/learning";
|
|
616
|
-
import { assertBoundedTaskJson } from "@openpond/evals/task-schema";
|
|
617
|
-
import {
|
|
618
|
-
NativeHarnessExperimentAttemptSchema,
|
|
619
|
-
verifyNativeHarnessExperimentAttempt
|
|
620
|
-
} from "@openpond/evals/experiments";
|
|
621
|
-
var HashSchema4 = z6.string().regex(/^[a-f0-9]{64}$/);
|
|
622
|
-
var ModelStarterEnvironmentAttemptSchema = /* @__PURE__ */ (() => z6.object({
|
|
623
|
-
schemaVersion: z6.literal("openpond.javascriptEnvironmentAttempt.v1"),
|
|
624
|
-
taskId: z6.string(),
|
|
625
|
-
status: z6.enum(["completed", "budget_exhausted", "cancelled", "timed_out", "policy_failure", "environment_failure"]),
|
|
626
|
-
output: z6.string().nullable(),
|
|
627
|
-
collected: z6.boolean(),
|
|
628
|
-
environmentCleanupComplete: z6.boolean(),
|
|
629
|
-
snapshot: z6.object({ definition: ModelProjectVersionedRefSchema, inputHash: HashSchema4, seed: z6.number().int(), initialStateHash: HashSchema4, finalStateHash: HashSchema4, state: z6.record(z6.string(), z6.unknown()), events: z6.array(z6.record(z6.string(), z6.unknown())) }).strict().nullable(),
|
|
630
|
-
messages: z6.array(z6.unknown()),
|
|
631
|
-
error: z6.string().nullable(),
|
|
632
|
-
contentHash: HashSchema4
|
|
633
|
-
}).strict())();
|
|
634
|
-
function verifyModelStarterEnvironmentAttempt(value, context) {
|
|
635
|
-
assertBoundedTaskJson(value, 16777216);
|
|
636
|
-
const result = ModelStarterEnvironmentAttemptSchema.parse(value);
|
|
637
|
-
const { contentHash: contentHash2, ...content } = result;
|
|
638
|
-
if (sealLearningContent(content).contentHash !== contentHash2 || result.taskId !== context.taskId || result.snapshot && (!equal(result.snapshot.definition, context.javascript) || result.snapshot.inputHash !== sealLearningContent(context.input).contentHash || result.snapshot.seed !== context.seed || result.snapshot.finalStateHash !== sealLearningContent(result.snapshot.state).contentHash)) throw new Error("Starter environment attempt differs from its admitted task or recorded bytes.");
|
|
639
|
-
return result;
|
|
640
|
-
}
|
|
641
|
-
var ModelStarterExecutionSchema = z6.object({
|
|
642
|
-
environment: EnvironmentReleaseSchema,
|
|
643
|
-
verifierSet: VerifierSetReleaseSchema,
|
|
644
|
-
javascript: JavaScriptEnvironmentDefinitionSchema
|
|
645
|
-
}).strict();
|
|
646
|
-
var ModelStarterToolFixtureScriptSchema = z6.object({
|
|
647
|
-
actions: z6.array(z6.object({ name: z6.string().min(1).max(64), arguments: z6.record(z6.string(), z6.unknown()) }).strict()).max(1e3)
|
|
648
|
-
}).strict();
|
|
649
|
-
function modelStarterExecutionAssetId(context) {
|
|
650
|
-
if (!context.environmentRelease || !context.verifierSetRelease) throw new Error("Starter execution release references are missing.");
|
|
651
|
-
return `starter-execution-${sealLearningContent({ environment: context.environmentRelease, verifiers: context.verifierSetRelease }).contentHash}`;
|
|
652
|
-
}
|
|
653
|
-
function createModelStarterExecutionAsset(value) {
|
|
654
|
-
const execution = ModelStarterExecutionSchema.parse(value);
|
|
655
|
-
assertExecutionIntegrity(execution);
|
|
656
|
-
const id = modelStarterExecutionAssetId({ environmentRelease: { id: execution.environment.id, contentHash: execution.environment.contentHash }, verifierSetRelease: { id: execution.verifierSet.id, contentHash: execution.verifierSet.contentHash } });
|
|
657
|
-
const generated = createLearningTextAsset({ text: canonicalJson(execution), path: "environment/execution.json", mediaType: "application/json", visibility: "host_private" });
|
|
658
|
-
const { contentHash: _hash, ...content } = generated;
|
|
659
|
-
return sealLearningContent({ ...content, id, asset: { ...content.asset, id } });
|
|
660
|
-
}
|
|
661
|
-
function resolveModelStarterExecutionAsset(context, asset, assets) {
|
|
662
|
-
if (asset.id !== modelStarterExecutionAssetId(context) || asset.asset.visibility !== "host_private" || asset.asset.mediaType !== "application/json") throw new Error("Starter execution asset differs from its Taskset references.");
|
|
663
|
-
const execution = ModelStarterExecutionSchema.parse(JSON.parse(verifyLearningTextAsset(asset, asset.asset)));
|
|
664
|
-
validateModelStarterExecution(execution, context, assets);
|
|
665
|
-
return execution;
|
|
666
|
-
}
|
|
667
|
-
function equal(left, right) {
|
|
668
|
-
return sealLearningContent({ value: left }).contentHash === sealLearningContent({ value: right }).contentHash;
|
|
669
|
-
}
|
|
670
|
-
function assertExecutionIntegrity(execution) {
|
|
671
|
-
assertJavaScriptEnvironmentDefinition(execution.javascript);
|
|
672
|
-
for (const resource of [execution.environment, execution.verifierSet]) {
|
|
673
|
-
const { contentHash: contentHash2, ...content } = resource;
|
|
674
|
-
if (sealLearningContent(content).contentHash !== contentHash2) throw new Error(`Starter execution resource integrity failed: ${resource.id}.`);
|
|
675
|
-
}
|
|
676
|
-
if (!equal(execution.environment.metadata.javascriptEnvironment, learningRef(execution.javascript))) throw new Error("Starter execution resources differ from its declared task context.");
|
|
677
|
-
}
|
|
678
|
-
function validateModelStarterExecution(execution, taskset, assets) {
|
|
679
|
-
if (!execution) {
|
|
680
|
-
if (taskset.environment.entrypoint === "openpond.javascript-environment.v1") throw new Error("Starter JavaScript execution resources are missing.");
|
|
681
|
-
return;
|
|
682
|
-
}
|
|
683
|
-
const { environment, verifierSet, javascript } = execution;
|
|
684
|
-
assertExecutionIntegrity(execution);
|
|
685
|
-
if (!equal(taskset.environmentRelease, { id: environment.id, contentHash: environment.contentHash }) || !equal(taskset.verifierSetRelease, { id: verifierSet.id, contentHash: verifierSet.contentHash }) || !equal(environment.contract, taskset.environment) || !equal(verifierSet.graders, taskset.graders) || !equal(taskset.tools, javascript.tools) || !equal(environment.metadata.javascriptEnvironment, learningRef(javascript))) {
|
|
686
|
-
throw new Error("Starter execution resources differ from its declared task context.");
|
|
687
|
-
}
|
|
688
|
-
const contract = environment.contract;
|
|
689
|
-
if (contract.entrypoint !== "openpond.javascript-environment.v1" || contract.kind !== "agent" || !contract.stateful || !contract.deterministicSeeds || contract.networkPolicy !== "none" || new Set(contract.lifecycle).size !== 5 || taskset.capabilities.length || taskset.policy.connectedAppScopes.length) {
|
|
690
|
-
throw new Error("Starter JavaScript environment declares unsupported host capabilities.");
|
|
691
|
-
}
|
|
692
|
-
for (const reference of [javascript.module, environment.actionSchemaRef, environment.observationSchemaRef, environment.stateSchemaRef]) {
|
|
693
|
-
if (!reference) continue;
|
|
694
|
-
const asset = assets.find((asset2) => asset2.id === reference.id);
|
|
695
|
-
if (!asset) throw new Error(`Starter environment asset is missing: ${reference.id}.`);
|
|
696
|
-
verifyLearningTextAsset(asset, reference);
|
|
697
|
-
}
|
|
698
|
-
for (const task of taskset.tasks) {
|
|
699
|
-
const asset = assets.find((asset2) => asset2.id === task.privilegedContextRef);
|
|
700
|
-
if (!asset || asset.asset.visibility !== "host_private" || asset.asset.mediaType !== "application/json") {
|
|
701
|
-
throw new Error(`Starter private initial state is missing: ${task.id}.`);
|
|
702
|
-
}
|
|
703
|
-
const state = JSON.parse(verifyLearningTextAsset(asset, asset.asset));
|
|
704
|
-
if (!state || typeof state !== "object" || Array.isArray(state)) throw new Error(`Starter initial state must be a JSON object: ${task.id}.`);
|
|
705
|
-
assertBoundedTaskJson(state, javascript.maxStateBytes);
|
|
706
|
-
}
|
|
707
|
-
}
|
|
708
|
-
|
|
709
|
-
// src/model-taskset-resources.ts
|
|
710
|
-
import { z as z7 } from "zod";
|
|
711
|
-
import { EnvironmentReleaseSchema as EnvironmentReleaseSchema2, VerifierSetReleaseSchema as VerifierSetReleaseSchema2, verifyEnvironmentRelease, verifyVerifierSetRelease } from "@openpond/evals";
|
|
712
|
-
import { createLearningTextAsset as createLearningTextAsset2, sealLearningContent as sealLearningContent2, verifyLearningTextAsset as verifyLearningTextAsset2 } from "@openpond/evals/learning";
|
|
713
|
-
var ModelTasksetExecutionResourcesSchema = z7.object({ environment: EnvironmentReleaseSchema2, verifierSet: VerifierSetReleaseSchema2 }).strict();
|
|
714
|
-
function modelTasksetExecutionResourcesAssetId(context) {
|
|
715
|
-
if (!context.environmentRelease || !context.verifierSetRelease) throw new Error("Taskset execution references are missing.");
|
|
716
|
-
return `taskset-execution-${sealLearningContent2({ environment: context.environmentRelease, verifiers: context.verifierSetRelease }).contentHash}`;
|
|
717
|
-
}
|
|
718
|
-
function createModelTasksetExecutionResourcesAsset(value) {
|
|
719
|
-
const resources = ModelTasksetExecutionResourcesSchema.parse({ environment: value.environment, verifierSet: value.verifierSet });
|
|
720
|
-
assertIntegrity(resources);
|
|
721
|
-
const id = modelTasksetExecutionResourcesAssetId({ environmentRelease: { id: resources.environment.id, contentHash: resources.environment.contentHash }, verifierSetRelease: { id: resources.verifierSet.id, contentHash: resources.verifierSet.contentHash } });
|
|
722
|
-
const { contentHash: _hash, ...asset } = createLearningTextAsset2({ text: canonicalJson(resources), path: "environment/resources.json", mediaType: "application/json", visibility: "host_private" });
|
|
723
|
-
return sealLearningContent2({ ...asset, id, asset: { ...asset.asset, id } });
|
|
724
|
-
}
|
|
725
|
-
function resolveModelTasksetExecutionResourcesAsset(context, asset) {
|
|
726
|
-
if (asset.id !== modelTasksetExecutionResourcesAssetId(context) || asset.asset.visibility !== "host_private" || asset.asset.mediaType !== "application/json") throw new Error("Taskset execution asset differs from its published references.");
|
|
727
|
-
const resources = ModelTasksetExecutionResourcesSchema.parse(JSON.parse(verifyLearningTextAsset2(asset, asset.asset)));
|
|
728
|
-
assertIntegrity(resources);
|
|
729
|
-
if (asset.id !== createModelTasksetExecutionResourcesAsset(resources).id || canonicalJson(context.environment) !== canonicalJson(resources.environment.contract) || canonicalJson(context.graders) !== canonicalJson(resources.verifierSet.graders)) throw new Error("Taskset differs from its execution resources.");
|
|
730
|
-
return resources;
|
|
731
|
-
}
|
|
732
|
-
function assertIntegrity(resources) {
|
|
733
|
-
if (!verifyEnvironmentRelease(resources.environment) || !verifyVerifierSetRelease(resources.verifierSet)) throw new Error("Taskset execution resource integrity failed.");
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
// src/model-taskset-derivation.ts
|
|
737
|
-
import { z as z8 } from "zod";
|
|
738
|
-
import { createEnvironmentRelease, createVerifierSetRelease, verifyEnvironmentRelease as verifyEnvironmentRelease2, verifyVerifierSetRelease as verifyVerifierSetRelease2 } from "@openpond/evals";
|
|
739
|
-
import { LearningTextAssetSchema, TaskDefinitionSchema, assertLearningContentHash, learningRef as learningRef2, sameLearningRef, sealLearningContent as sealLearningContent3, verifyLearningTextAsset as verifyLearningTextAsset3 } from "@openpond/evals/learning";
|
|
740
|
-
import { RewardBindingSchema, RewardReleaseSchema, compileBoundGraders } from "@openpond/evals/rewards";
|
|
741
|
-
import { TasksetReleaseSchema, assertTasksetRelease } from "@openpond/evals/tasksets";
|
|
742
|
-
import { validateTaskValue } from "@openpond/evals/task-schema";
|
|
743
|
-
var OwnerSchema = z8.object({ scopeId: z8.string().trim().min(1).max(500), modelId: z8.string().trim().min(1).max(500) }).strict();
|
|
744
|
-
var ModelTasksetDerivationSchema = z8.object({
|
|
745
|
-
schemaVersion: z8.literal("openpond.modelTasksetDerivation.v1"),
|
|
746
|
-
owner: OwnerSchema,
|
|
747
|
-
root: ModelProjectVersionedRefSchema,
|
|
748
|
-
parent: ModelProjectVersionedRefSchema
|
|
749
|
-
}).strict();
|
|
750
|
-
var ModelTasksetPackageSchema = z8.object({
|
|
751
|
-
taskset: TasksetReleaseSchema,
|
|
752
|
-
taskDefinition: TaskDefinitionSchema,
|
|
753
|
-
rewardBinding: RewardBindingSchema,
|
|
754
|
-
rewards: z8.array(RewardReleaseSchema).min(1).max(100),
|
|
755
|
-
assets: z8.array(LearningTextAssetSchema).max(1e3),
|
|
756
|
-
executionResources: ModelTasksetExecutionResourcesSchema.optional(),
|
|
757
|
-
execution: ModelStarterExecutionSchema.optional(),
|
|
758
|
-
instructionMode: z8.literal("per_task").optional()
|
|
759
|
-
}).strict();
|
|
760
|
-
var same = (left, right) => sealLearningContent3({ value: left }).contentHash === sealLearningContent3({ value: right }).contentHash;
|
|
761
|
-
var identity = (owner, root) => `model-taskset-${sealLearningContent3({ owner, root }).contentHash}`;
|
|
762
|
-
function deriveModelTaskset(input) {
|
|
763
|
-
return compileModelTasksetReward({ ...input, source: validateModelTasksetPackage(input.source) });
|
|
764
|
-
}
|
|
765
|
-
function compileModelTasksetReward(input) {
|
|
766
|
-
const owner = OwnerSchema.parse(input.owner);
|
|
767
|
-
const source = input.source;
|
|
768
|
-
const edited = input.edits ?? source;
|
|
769
|
-
const previous = source.taskset.metadata.modelTasksetDerivation === void 0 ? null : ModelTasksetDerivationSchema.parse(source.taskset.metadata.modelTasksetDerivation);
|
|
770
|
-
if (previous && source.taskset.id !== identity(previous.owner, previous.root)) throw new Error("Derived Taskset identity differs from its ownership lineage.");
|
|
771
|
-
const owned = previous && same(previous.owner, owner);
|
|
772
|
-
const root = owned ? previous.root : learningRef2(source.taskset);
|
|
773
|
-
const id = owned ? source.taskset.id : identity(owner, root);
|
|
774
|
-
const revision = owned ? source.taskset.revision + 1 : 1;
|
|
775
|
-
const rewardBinding = RewardBindingSchema.parse(input.rewardBinding);
|
|
776
|
-
const rewards = input.rewards.map((reward) => RewardReleaseSchema.parse(reward));
|
|
777
|
-
const graders = compileBoundGraders(rewardBinding, rewards);
|
|
778
|
-
if (rewardBinding.recipeRef) throw new Error("Resolve the Reward recipe into one standalone binding before deriving a Taskset.");
|
|
779
|
-
const policy = { ...edited.taskset.policy, hiddenGraderRefs: graders.filter((grader) => grader.privileged).map((grader) => grader.id) };
|
|
780
|
-
const previousResources = edited.executionResources ?? edited.execution;
|
|
781
|
-
const environment = previousResources?.environment ?? createEnvironmentRelease({
|
|
782
|
-
schemaVersion: "openpond.environmentRelease.v1",
|
|
783
|
-
id: `${id}-environment`,
|
|
784
|
-
revision: 1,
|
|
785
|
-
contract: edited.taskset.environment,
|
|
786
|
-
actionSchemaRef: null,
|
|
787
|
-
observationSchemaRef: null,
|
|
788
|
-
stateSchemaRef: null,
|
|
789
|
-
artifactCollection: { maxArtifacts: 1e5, maxTotalBytes: 25e7 },
|
|
790
|
-
adapterConformanceHashes: {},
|
|
791
|
-
metadata: {}
|
|
792
|
-
});
|
|
793
|
-
const verifierSet = createVerifierSetRelease({
|
|
794
|
-
schemaVersion: "openpond.verifierSetRelease.v1",
|
|
795
|
-
id: `${id}-verifiers`,
|
|
796
|
-
revision,
|
|
797
|
-
graders,
|
|
798
|
-
isolation: previousResources?.verifierSet.isolation ?? { processBoundary: "isolated_process", networkPolicy: "none", defaultTimeoutMs: Math.min(edited.taskset.environment.defaultTimeoutMs, 3e5) },
|
|
799
|
-
calibrationReceiptRefs: [],
|
|
800
|
-
metadata: previousResources ? { sourceVerifierSet: learningRef2(previousResources.verifierSet) } : {}
|
|
801
|
-
});
|
|
802
|
-
const executionResources = { environment, verifierSet };
|
|
803
|
-
const execution = edited.execution ? { ...edited.execution, verifierSet } : void 0;
|
|
804
|
-
const { contentHash: _taskHash, ...taskContent } = edited.taskset;
|
|
805
|
-
const { contentHash: _definitionHash, ...definitionContent } = edited.taskDefinition;
|
|
806
|
-
const taskExecution = { ...definitionContent.execution, policy, environmentRelease: { id: environment.id, contentHash: environment.contentHash }, verifierSetRelease: { id: verifierSet.id, contentHash: verifierSet.contentHash } };
|
|
807
|
-
const taskDefinition = TaskDefinitionSchema.parse(sealLearningContent3({ ...definitionContent, id: `${id}-definition`, revision, rewardBinding: learningRef2(rewardBinding), execution: taskExecution }));
|
|
808
|
-
const authoring = source.taskset.metadata.starterAuthoring ?? source.taskset.metadata.ordinaryAuthoring;
|
|
809
|
-
const graderFixtures = authoring && typeof authoring === "object" && !Array.isArray(authoring) && "graderFixtures" in authoring ? authoring.graderFixtures : void 0;
|
|
810
|
-
const taskset = TasksetReleaseSchema.parse(sealLearningContent3({
|
|
811
|
-
...taskContent,
|
|
812
|
-
...taskExecution,
|
|
813
|
-
id,
|
|
814
|
-
revision,
|
|
815
|
-
graders,
|
|
816
|
-
metadata: {
|
|
817
|
-
starter: { taskDefinition: learningRef2(taskDefinition), rewardBinding: learningRef2(rewardBinding) },
|
|
818
|
-
rewardExecution: { binding: rewardBinding, rewards },
|
|
819
|
-
...graderFixtures === void 0 ? {} : { starterAuthoring: { graderFixtures } },
|
|
820
|
-
...edited.taskset.metadata.environmentResources === void 0 ? {} : { environmentResources: edited.taskset.metadata.environmentResources },
|
|
821
|
-
...edited.taskset.metadata.workAssetStagingPaths === void 0 ? {} : { workAssetStagingPaths: edited.taskset.metadata.workAssetStagingPaths },
|
|
822
|
-
...edited.taskset.metadata.tasksetOutputContract === void 0 ? {} : { tasksetOutputContract: edited.taskset.metadata.tasksetOutputContract },
|
|
823
|
-
modelTasksetDerivation: { schemaVersion: "openpond.modelTasksetDerivation.v1", owner, root, parent: learningRef2(source.taskset) }
|
|
824
|
-
}
|
|
825
|
-
}));
|
|
826
|
-
return validateModelTasksetPackage({ taskset, taskDefinition, rewardBinding, rewards, assets: input.assets, executionResources, ...execution ? { execution } : {}, ...source.instructionMode ? { instructionMode: source.instructionMode } : {} });
|
|
827
|
-
}
|
|
828
|
-
function validateModelTasksetPackage(value) {
|
|
829
|
-
const result = ModelTasksetPackageSchema.parse(value);
|
|
830
|
-
const { taskset, taskDefinition, rewardBinding, rewards, assets, execution } = result;
|
|
831
|
-
for (const resource of [taskset, taskDefinition, rewardBinding, ...rewards, ...assets]) assertLearningContentHash(resource);
|
|
832
|
-
assertTasksetRelease(taskset);
|
|
833
|
-
if (rewardBinding.recipeRef) throw new Error("Resolve the Reward recipe into one standalone binding before deriving a Taskset.");
|
|
834
|
-
if (!sameLearningRef(taskDefinition.rewardBinding, learningRef2(rewardBinding))) throw new Error("Taskset definition differs from its Reward binding.");
|
|
835
|
-
const { policy, environment, environmentRelease, tools, capabilities, verifierSetRelease } = taskset;
|
|
836
|
-
if (!same(taskDefinition.execution, { policy, environment, ...environmentRelease ? { environmentRelease } : {}, tools, capabilities, ...verifierSetRelease ? { verifierSetRelease } : {} })) throw new Error("Taskset definition differs from its execution resources.");
|
|
837
|
-
if (!same(taskset.graders, compileBoundGraders(rewardBinding, rewards))) throw new Error("Taskset graders differ from its Reward binding.");
|
|
838
|
-
if (taskset.metadata.starter !== void 0) {
|
|
839
|
-
const metadata = z8.object({ taskDefinition: ModelProjectVersionedRefSchema, rewardBinding: ModelProjectVersionedRefSchema }).passthrough().parse(taskset.metadata.starter);
|
|
840
|
-
if (!sameLearningRef(metadata.taskDefinition, learningRef2(taskDefinition)) || !sameLearningRef(metadata.rewardBinding, learningRef2(rewardBinding))) throw new Error("Taskset metadata differs from its immutable grading resources.");
|
|
841
|
-
}
|
|
842
|
-
if (taskset.metadata.rewardExecution !== void 0 && !same(taskset.metadata.rewardExecution, { binding: rewardBinding, rewards })) throw new Error("Taskset embedded Rewards differ from its immutable grading resources.");
|
|
843
|
-
if (taskset.metadata.rewardBinding !== void 0 && !sameLearningRef(ModelProjectVersionedRefSchema.parse(taskset.metadata.rewardBinding), learningRef2(rewardBinding))) throw new Error("Taskset Reward reference differs from its immutable grading resources.");
|
|
844
|
-
const resources = result.executionResources ?? execution;
|
|
845
|
-
if (taskset.environmentRelease && !resources) throw new Error("Deriving a Taskset requires its pinned execution resources.");
|
|
846
|
-
if (resources && (!verifyEnvironmentRelease2(resources.environment) || !verifyVerifierSetRelease2(resources.verifierSet) || !same(taskset.environmentRelease, { id: resources.environment.id, contentHash: resources.environment.contentHash }) || !same(taskset.verifierSetRelease, { id: resources.verifierSet.id, contentHash: resources.verifierSet.contentHash }) || !same(taskset.environment, resources.environment.contract) || !same(taskset.graders, resources.verifierSet.graders))) throw new Error("Taskset differs from its pinned execution resources.");
|
|
847
|
-
validateModelStarterExecution(execution, taskset, assets);
|
|
848
|
-
if (new Set(assets.map((asset) => asset.id)).size !== assets.length) throw new Error("Taskset asset identities must be unique.");
|
|
849
|
-
for (const asset of assets) verifyLearningTextAsset3(asset, asset.asset);
|
|
850
|
-
for (const reference of resources ? [resources.environment.actionSchemaRef, resources.environment.observationSchemaRef, resources.environment.stateSchemaRef] : []) {
|
|
851
|
-
if (!reference) continue;
|
|
852
|
-
const asset = assets.find((asset2) => asset2.id === reference.id);
|
|
853
|
-
if (!asset) throw new Error(`Taskset environment asset is missing: ${reference.id}.`);
|
|
854
|
-
verifyLearningTextAsset3(asset, reference);
|
|
855
|
-
}
|
|
856
|
-
for (const task of taskset.tasks) {
|
|
857
|
-
if (taskDefinition.requiredOutputs && !same(task.requiredOutputs, taskDefinition.requiredOutputs)) throw new Error(`Taskset outputs differ from its reviewed definition: ${task.id}.`);
|
|
858
|
-
if (result.instructionMode !== "per_task" && task.policyVisibleContext.instructions !== taskDefinition.instructions || !validateTaskValue(taskDefinition.inputSchema, task.input).valid || task.expectedOutput !== null && !validateTaskValue(taskDefinition.outputSchema, task.expectedOutput).valid) throw new Error(`Taskset task differs from its declared format: ${task.id}.`);
|
|
859
|
-
if (task.privilegedContextRef && !assets.some((asset) => asset.id === task.privilegedContextRef && asset.asset.visibility === "host_private")) throw new Error(`Taskset private context is missing: ${task.id}.`);
|
|
860
|
-
}
|
|
861
|
-
for (const reward of rewards) {
|
|
862
|
-
const implementation = reward.implementation;
|
|
863
|
-
const references = [...reward.assets, ..."verifierRef" in implementation ? [implementation.verifierRef] : [], ..."rubricRef" in implementation ? [implementation.rubricRef] : [], ..."inputContract" in implementation ? [implementation.inputContract] : []];
|
|
864
|
-
for (const reference of references) {
|
|
865
|
-
const asset = assets.find((asset2) => asset2.id === reference.id);
|
|
866
|
-
if (!asset || reference.visibility === "policy") throw new Error(`Taskset Reward requires its private asset: ${reference.id}.`);
|
|
867
|
-
verifyLearningTextAsset3(asset, reference);
|
|
868
|
-
}
|
|
869
|
-
}
|
|
870
|
-
return result;
|
|
871
|
-
}
|
|
872
|
-
|
|
873
|
-
// src/taskset-package-learning.ts
|
|
874
|
-
import { z as z9 } from "zod";
|
|
875
|
-
import {
|
|
876
|
-
TaskBatchSchema,
|
|
877
|
-
TaskEvidenceSchema,
|
|
878
|
-
TaskAdmissionDecisionSchema,
|
|
879
|
-
LearningSourceSchema,
|
|
880
|
-
LearningTextAssetSchema as LearningTextAssetSchema2,
|
|
881
|
-
assertLearningContentHash as assertLearningContentHash2,
|
|
882
|
-
compileTaskBatch,
|
|
883
|
-
learningRef as learningRef3,
|
|
884
|
-
sameLearningRef as sameLearningRef2,
|
|
885
|
-
taskBatchPackageMetadata,
|
|
886
|
-
verifyLearningTextAsset as verifyLearningTextAsset4
|
|
887
|
-
} from "@openpond/evals/learning";
|
|
888
|
-
var TasksetPackageLearningResourcesSchema = z9.object({
|
|
889
|
-
batch: TaskBatchSchema,
|
|
890
|
-
evidence: z9.array(TaskEvidenceSchema).min(1).max(1e4),
|
|
891
|
-
decisions: z9.array(TaskAdmissionDecisionSchema).min(1).max(1e4),
|
|
892
|
-
sources: z9.array(LearningSourceSchema).min(1).max(1e4),
|
|
893
|
-
assets: z9.array(LearningTextAssetSchema2).max(1e3)
|
|
894
|
-
}).strict();
|
|
895
|
-
function learningPackageContextFiles(resources) {
|
|
896
|
-
return resources.evidence.filter((evidence) => evidence.submission.evaluatorContext !== null).map((evidence) => {
|
|
897
|
-
const text = canonicalJson(evidence.submission.evaluatorContext);
|
|
898
|
-
const bytes = new TextEncoder().encode(text);
|
|
899
|
-
return {
|
|
900
|
-
asset: {
|
|
901
|
-
id: `task-evidence:${evidence.contentHash}`,
|
|
902
|
-
path: `learning/context/${evidence.contentHash}.json`,
|
|
903
|
-
contentHash: sha256(bytes),
|
|
904
|
-
sizeBytes: bytes.byteLength,
|
|
905
|
-
mediaType: "application/json",
|
|
906
|
-
visibility: "host_private"
|
|
907
|
-
},
|
|
908
|
-
base64: btoa(Array.from(bytes, (byte) => String.fromCharCode(byte)).join(""))
|
|
909
|
-
};
|
|
910
|
-
});
|
|
911
|
-
}
|
|
912
|
-
function validateTasksetLearningResources(taskset, input) {
|
|
913
|
-
const resources = TasksetPackageLearningResourcesSchema.parse(input);
|
|
914
|
-
const metadata = taskBatchPackageMetadata(taskset);
|
|
915
|
-
if (!sameLearningRef2(learningRef3(resources.batch), metadata.batch)) throw new Error("Taskset learning batch differs from its admission metadata.");
|
|
916
|
-
for (const collection of [resources.evidence, resources.decisions, resources.sources, resources.assets]) {
|
|
917
|
-
const identities = /* @__PURE__ */ new Set();
|
|
918
|
-
for (const resource of collection) {
|
|
919
|
-
assertLearningContentHash2(resource);
|
|
920
|
-
const key = `${resource.id}:${resource.revision}`;
|
|
921
|
-
if (identities.has(key)) throw new Error("Taskset learning resource identities must be unique.");
|
|
922
|
-
identities.add(key);
|
|
923
|
-
}
|
|
924
|
-
}
|
|
925
|
-
if (resources.evidence.length !== resources.batch.examples.length || resources.decisions.length !== resources.batch.examples.length) throw new Error("Taskset learning snapshots must match the admitted batch inventory.");
|
|
926
|
-
for (const evidence of resources.evidence) {
|
|
927
|
-
const source = resources.sources.find((source2) => sameLearningRef2(learningRef3(source2), evidence.source));
|
|
928
|
-
if (!source || !sameLearningRef2(source.taskDefinition, learningRef3(metadata.definition))) throw new Error("Taskset evidence source differs from its task definition.");
|
|
929
|
-
}
|
|
930
|
-
if (resources.sources.some((source) => !resources.evidence.some((evidence) => sameLearningRef2(evidence.source, learningRef3(source))))) throw new Error("Taskset package includes an unrelated learning source.");
|
|
931
|
-
const rewardAssets = metadata.rewards.flatMap((reward) => [
|
|
932
|
-
...reward.assets,
|
|
933
|
-
...reward.implementation.kind === "custom_verifier" ? [reward.implementation.verifierRef] : reward.implementation.kind === "model_judge" || reward.implementation.kind === "human" ? [reward.implementation.rubricRef] : [],
|
|
934
|
-
..."inputContract" in reward.implementation ? [reward.implementation.inputContract] : []
|
|
935
|
-
]);
|
|
936
|
-
for (const ref of rewardAssets) {
|
|
937
|
-
const asset = resources.assets.find((asset2) => asset2.id === ref.id);
|
|
938
|
-
if (!asset) throw new Error("Taskset learning Reward asset is missing.");
|
|
939
|
-
verifyLearningTextAsset4(asset, ref);
|
|
940
|
-
}
|
|
941
|
-
if (resources.assets.some((asset) => !rewardAssets.some((ref) => ref.id === asset.id))) throw new Error("Taskset package includes an unrelated learning asset.");
|
|
942
|
-
const compiled = compileTaskBatch({
|
|
943
|
-
batch: resources.batch,
|
|
944
|
-
definition: metadata.definition,
|
|
945
|
-
binding: metadata.binding,
|
|
946
|
-
rewards: metadata.rewards,
|
|
947
|
-
evidence: resources.evidence,
|
|
948
|
-
decisions: resources.decisions
|
|
949
|
-
});
|
|
950
|
-
const compiledMetadata = taskBatchPackageMetadata(compiled);
|
|
951
|
-
if (contentHash(compiledMetadata) !== contentHash(metadata)) throw new Error("Taskset admission metadata differs from its reviewed snapshots.");
|
|
952
|
-
const execution = (release) => ({ policy: release.policy, environment: release.environment, tools: release.tools });
|
|
953
|
-
if (contentHash(execution(compiled)) !== contentHash(execution(taskset))) throw new Error("Taskset execution differs from its reviewed task definition.");
|
|
954
|
-
const taskContent = (task) => ({
|
|
955
|
-
id: task.id,
|
|
956
|
-
clusterKey: task.clusterKey,
|
|
957
|
-
split: task.split,
|
|
958
|
-
input: task.input,
|
|
959
|
-
expectedOutput: task.expectedOutput,
|
|
960
|
-
policyVisibleContext: task.policyVisibleContext,
|
|
961
|
-
privilegedContextRef: task.privilegedContextRef,
|
|
962
|
-
artifactRefs: task.artifactRefs,
|
|
963
|
-
requiredOutputs: task.requiredOutputs ?? [],
|
|
964
|
-
tags: task.tags
|
|
965
|
-
});
|
|
966
|
-
if (contentHash(compiled.tasks.map(taskContent)) !== contentHash(taskset.tasks.map(taskContent))) throw new Error("Taskset rows differ from their reviewed evidence.");
|
|
967
|
-
return resources;
|
|
968
|
-
}
|
|
969
|
-
|
|
970
|
-
// src/taskset-package-files.ts
|
|
971
|
-
import { z as z10 } from "zod";
|
|
972
|
-
var MAX_TASKSET_PACKAGE_BYTES = 64 * 1024 * 1024;
|
|
973
|
-
var TasksetPackageFileSchema = z10.object({
|
|
974
|
-
asset: ImmutableAssetRefSchema,
|
|
975
|
-
base64: z10.string().max(Math.ceil(MAX_TASKSET_PACKAGE_BYTES / 3) * 4)
|
|
976
|
-
}).strict();
|
|
977
|
-
function decodeTasksetPackageFile(value) {
|
|
978
|
-
const file = TasksetPackageFileSchema.parse(value);
|
|
979
|
-
const raw = atob(file.base64);
|
|
980
|
-
if (btoa(raw) !== file.base64) throw new Error(`Taskset asset ${file.asset.id} has noncanonical base64.`);
|
|
981
|
-
const bytes = new Uint8Array(raw.length);
|
|
982
|
-
for (let index = 0; index < raw.length; index++) bytes[index] = raw.charCodeAt(index);
|
|
983
|
-
if (bytes.byteLength !== file.asset.sizeBytes || sha256(bytes) !== file.asset.contentHash) throw new Error(`Taskset asset ${file.asset.id} differs from its immutable bytes.`);
|
|
984
|
-
return bytes;
|
|
985
|
-
}
|
|
986
|
-
|
|
987
|
-
// src/taskset-package-execution.ts
|
|
988
|
-
import { LearningTextAssetSchema as LearningTextAssetSchema3, sealLearningContent as sealLearningContent4 } from "@openpond/evals/learning";
|
|
989
|
-
function resolveTasksetPackageExecution(input) {
|
|
990
|
-
if (input.taskset.environment.entrypoint !== "openpond.javascript-environment.v1") return null;
|
|
991
|
-
const byId = new Map(input.files.map((file) => [file.asset.id, file]));
|
|
992
|
-
const read = (id) => {
|
|
993
|
-
const file = byId.get(id);
|
|
994
|
-
if (!file) throw new Error(`Taskset environment asset is missing: ${id}.`);
|
|
995
|
-
if (file.asset.sizeBytes > 524288) throw new Error("Taskset environment source exceeds the authored asset limit.");
|
|
996
|
-
const text = new TextDecoder("utf-8", { fatal: true }).decode(decodeTasksetPackageFile(file));
|
|
997
|
-
return LearningTextAssetSchema3.parse(sealLearningContent4({ schemaVersion: "openpond.learningTextAsset.v1", id, revision: 1, asset: file.asset, text }));
|
|
998
|
-
};
|
|
999
|
-
let execution;
|
|
1000
|
-
if (input.modelResources) {
|
|
1001
|
-
if (!input.modelResources.execution) throw new Error("Bound Taskset JavaScript execution resources are missing.");
|
|
1002
|
-
execution = ModelStarterExecutionSchema.parse(input.modelResources.execution);
|
|
1003
|
-
} else {
|
|
1004
|
-
const declaration = read(modelStarterExecutionAssetId(input.taskset));
|
|
1005
|
-
if (declaration.asset.visibility !== "host_private" || declaration.asset.mediaType !== "application/json" || declaration.asset.path !== "environment/execution.json") throw new Error("Taskset execution declaration must be private JSON at environment/execution.json.");
|
|
1006
|
-
execution = ModelStarterExecutionSchema.parse(JSON.parse(declaration.text));
|
|
1007
|
-
}
|
|
1008
|
-
const ids = /* @__PURE__ */ new Set([
|
|
1009
|
-
execution.javascript.module.id,
|
|
1010
|
-
...[execution.environment.actionSchemaRef, execution.environment.observationSchemaRef, execution.environment.stateSchemaRef].flatMap((ref) => ref ? [ref.id] : []),
|
|
1011
|
-
...input.taskset.tasks.flatMap((task) => task.privilegedContextRef ? [task.privilegedContextRef] : [])
|
|
1012
|
-
]);
|
|
1013
|
-
const assets = [...ids].map(read);
|
|
1014
|
-
validateModelStarterExecution(execution, input.taskset, assets);
|
|
1015
|
-
return { execution, assets };
|
|
1016
|
-
}
|
|
1017
|
-
function createTasksetPackageExecutionFile(execution) {
|
|
1018
|
-
const asset = createModelStarterExecutionAsset(execution);
|
|
1019
|
-
const bytes = new TextEncoder().encode(asset.text);
|
|
1020
|
-
let raw = "";
|
|
1021
|
-
for (const byte of bytes) raw += String.fromCharCode(byte);
|
|
1022
|
-
return { asset: asset.asset, base64: btoa(raw) };
|
|
1023
|
-
}
|
|
1024
|
-
|
|
1025
|
-
// src/taskset-calibration-fixtures.ts
|
|
1026
|
-
import { z as z11 } from "zod";
|
|
1027
|
-
import { compileBoundGraders as compileBoundGraders2, createRewardBinding, RewardReleaseRefSchema } from "@openpond/evals/rewards";
|
|
1028
|
-
import { TaskRecordSchema } from "@openpond/evals/tasksets";
|
|
1029
|
-
import {
|
|
1030
|
-
learningRef as learningRef4,
|
|
1031
|
-
LearningRevisionRefSchema,
|
|
1032
|
-
RewardFixtureSchema,
|
|
1033
|
-
verifyRewardCalibrationClosure,
|
|
1034
|
-
rewardCalibrationClosureAsset,
|
|
1035
|
-
MAX_REWARD_CALIBRATION_CLOSURE_BYTES
|
|
1036
|
-
} from "@openpond/evals/learning/reward-calibration-closure";
|
|
1037
|
-
var ProjectionSchema = z11.object({
|
|
1038
|
-
schemaVersion: z11.literal("openpond.connectedRewardCalibrationProjection.v1"),
|
|
1039
|
-
sourceReward: RewardReleaseRefSchema,
|
|
1040
|
-
fixtureSet: ImmutableAssetRefSchema,
|
|
1041
|
-
calibrationCheck: LearningRevisionRefSchema,
|
|
1042
|
-
fixture: RewardFixtureSchema,
|
|
1043
|
-
fixtureHash: z11.string().regex(/^[a-f0-9]{64}$/),
|
|
1044
|
-
checkResultHash: z11.string().regex(/^[a-f0-9]{64}$/),
|
|
1045
|
-
independentCalibrationTask: z11.literal(true),
|
|
1046
|
-
closureRef: ImmutableAssetRefSchema,
|
|
1047
|
-
graderId: z11.string().min(1).max(240).optional()
|
|
1048
|
-
}).strict();
|
|
1049
|
-
function authoredFixtures(value) {
|
|
1050
|
-
const authoring = value.taskset.metadata.ordinaryAuthoring ?? value.taskset.metadata.starterAuthoring;
|
|
1051
|
-
return z11.object({ graderFixtures: GraderFixtureSchema.array().max(1e5).default([]) }).passthrough().parse(authoring ?? {}).graderFixtures;
|
|
1052
|
-
}
|
|
1053
|
-
function isIndependent(fixture) {
|
|
1054
|
-
const projection = fixture.metadata.rewardCalibration;
|
|
1055
|
-
return projection !== void 0;
|
|
1056
|
-
}
|
|
1057
|
-
function assertTasksetCalibrationClosures(value) {
|
|
1058
|
-
for (const fixture of authoredFixtures(value)) {
|
|
1059
|
-
const projection = fixture.metadata.rewardCalibration;
|
|
1060
|
-
if (projection && typeof projection === "object" && Object.hasOwn(projection, "closureRef"))
|
|
1061
|
-
resolveIndependent(value, fixture);
|
|
1062
|
-
}
|
|
1063
|
-
for (const file of value.files) if (file.asset.mediaType === "application/vnd.openpond.reward-calibration-closure+json")
|
|
1064
|
-
readPrivateClosure(value, file.asset);
|
|
1065
|
-
}
|
|
1066
|
-
function resolveTasksetCalibrationFixture(value, fixtureId) {
|
|
1067
|
-
const { contentHash: packageHash, ...packageContent } = value;
|
|
1068
|
-
if (contentHash(packageContent) !== packageHash) fail("calibration_package_changed");
|
|
1069
|
-
const fixtures = authoredFixtures(value).filter((fixture2) => fixture2.id === fixtureId);
|
|
1070
|
-
if (fixtures.length !== 1) fail("calibration_fixture_not_unique");
|
|
1071
|
-
const fixture = fixtures[0];
|
|
1072
|
-
if (!isIndependent(fixture)) fail("calibration_fixture_not_independent");
|
|
1073
|
-
return resolveIndependent(value, fixture);
|
|
1074
|
-
}
|
|
1075
|
-
function resolveIndependent(value, authored) {
|
|
1076
|
-
const projection = ProjectionSchema.parse(authored.metadata.rewardCalibration);
|
|
1077
|
-
const closure = readPrivateClosure(value, projection.closureRef);
|
|
1078
|
-
if (contentHash(projection.sourceReward) !== contentHash(learningRef4(closure.reward)) || contentHash(projection.fixtureSet) !== contentHash(closure.reward.fixtureSetRef) || projection.calibrationCheck.id !== closure.check.id || projection.calibrationCheck.revision !== closure.check.revision || projection.calibrationCheck.contentHash !== contentHash(closure.check)) fail("calibration_source_changed");
|
|
1079
|
-
const fixture = projection.fixture;
|
|
1080
|
-
const id = `calibration-${contentHash([projection.sourceReward, fixture.id])}`;
|
|
1081
|
-
if (authored.id !== id || authored.taskId !== id || value.taskset.tasks.some((task) => task.id === id) || projection.fixtureHash !== contentHash(fixture) || contentHash(authored.output) !== contentHash(fixture.output) || authored.infrastructureError !== fixture.infrastructureError) fail("calibration_fixture_changed");
|
|
1082
|
-
const checks = closure.check.results.filter((result) => result.fixture.id === fixture.id && result.fixture.contentHash === projection.fixtureHash);
|
|
1083
|
-
if (checks.length !== 1 || contentHash(checks[0]) !== projection.checkResultHash) fail("calibration_result_changed");
|
|
1084
|
-
const expected = fixture.expected;
|
|
1085
|
-
if (expected.status !== "scored" || typeof expected.passed !== "boolean" || authored.expectedPassed !== expected.passed || authored.label !== (expected.passed ? "positive" : "negative")) fail("calibration_expectation_unsupported");
|
|
1086
|
-
const graderId = projection.graderId ?? projection.sourceReward.id;
|
|
1087
|
-
const graders = value.taskset.graders.filter((grader2) => grader2.id === graderId);
|
|
1088
|
-
const grader = graders[0];
|
|
1089
|
-
if (graders.length !== 1 || !grader) fail("calibration_grader_not_unique");
|
|
1090
|
-
if (authored.expectedRewardEligible !== grader.rewardEligible) fail("calibration_reward_expectation_changed");
|
|
1091
|
-
const binding = createRewardBinding({
|
|
1092
|
-
schemaVersion: "openpond.rewardBinding.v1",
|
|
1093
|
-
id: `calibration-${closure.contentHash}`,
|
|
1094
|
-
revision: 1,
|
|
1095
|
-
sources: [{
|
|
1096
|
-
graderId,
|
|
1097
|
-
reward: projection.sourceReward,
|
|
1098
|
-
role: "evaluation",
|
|
1099
|
-
normalization: { kind: "identity" },
|
|
1100
|
-
weight: grader.weight,
|
|
1101
|
-
required: true,
|
|
1102
|
-
hardGate: grader.hardGate,
|
|
1103
|
-
privileged: grader.privileged,
|
|
1104
|
-
fixtureRefs: []
|
|
1105
|
-
}],
|
|
1106
|
-
aggregation: "weighted_mean",
|
|
1107
|
-
unscorable: "exclude_optional_require_all_required"
|
|
1108
|
-
}, [closure.reward]);
|
|
1109
|
-
const canonical = compileBoundGraders2(binding, [closure.reward])[0];
|
|
1110
|
-
const implementation = (spec) => {
|
|
1111
|
-
const { rewardEligible: _eligible, taskIds: _taskIds, ...rest } = spec;
|
|
1112
|
-
return rest.kind === "model_judge" ? { ...rest, temperature: rest.temperature ?? 0 } : rest;
|
|
1113
|
-
};
|
|
1114
|
-
if (contentHash(implementation(grader)) !== contentHash(implementation(canonical))) fail("calibration_grader_changed");
|
|
1115
|
-
if (grader.kind === "model_judge") assertJudgeFixtureMembership(value, grader, projection, closure);
|
|
1116
|
-
return {
|
|
1117
|
-
kind: "independent_reward_calibration",
|
|
1118
|
-
fixtureId: id,
|
|
1119
|
-
task: TaskRecordSchema.parse({
|
|
1120
|
-
id,
|
|
1121
|
-
clusterKey: id,
|
|
1122
|
-
split: "validation",
|
|
1123
|
-
input: fixture.input,
|
|
1124
|
-
expectedOutput: fixture.expectedOutput,
|
|
1125
|
-
privilegedContextRef: null
|
|
1126
|
-
}),
|
|
1127
|
-
evaluatorContext: fixture.evaluatorContext,
|
|
1128
|
-
evidence: {
|
|
1129
|
-
output: fixture.output,
|
|
1130
|
-
artifactRefs: fixture.artifactRefs,
|
|
1131
|
-
runtimeEventRefs: fixture.runtimeEventRefs,
|
|
1132
|
-
infrastructureError: fixture.infrastructureError
|
|
1133
|
-
},
|
|
1134
|
-
grader: canonical,
|
|
1135
|
-
reward: closure.reward,
|
|
1136
|
-
binding,
|
|
1137
|
-
expected,
|
|
1138
|
-
source: {
|
|
1139
|
-
reward: projection.sourceReward,
|
|
1140
|
-
fixtureSet: projection.fixtureSet,
|
|
1141
|
-
check: projection.calibrationCheck,
|
|
1142
|
-
closure: projection.closureRef,
|
|
1143
|
-
fixtureHash: projection.fixtureHash,
|
|
1144
|
-
checkResultHash: projection.checkResultHash
|
|
1145
|
-
}
|
|
1146
|
-
};
|
|
1147
|
-
}
|
|
1148
|
-
function readPrivateClosure(value, reference) {
|
|
1149
|
-
if (reference.visibility !== "host_private" || reference.sizeBytes > MAX_REWARD_CALIBRATION_CLOSURE_BYTES || reference.mediaType !== "application/vnd.openpond.reward-calibration-closure+json") fail("calibration_closure_not_private");
|
|
1150
|
-
const files = value.files.filter((file) => file.asset.id === reference.id);
|
|
1151
|
-
if (files.length !== 1 || contentHash(files[0].asset) !== contentHash(reference)) fail("calibration_closure_asset_changed");
|
|
1152
|
-
const bytes = decodeTasksetPackageFile(files[0]);
|
|
1153
|
-
const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
1154
|
-
const closure = verifyRewardCalibrationClosure(JSON.parse(text));
|
|
1155
|
-
if (contentHash(rewardCalibrationClosureAsset(closure).asset) !== contentHash(reference)) fail("calibration_closure_reference_changed");
|
|
1156
|
-
const aliases = value.files.filter((file) => file.asset.contentHash === reference.contentHash || file.asset.id === reference.id || file.asset.path === reference.path);
|
|
1157
|
-
if (aliases.some((file) => file.asset.visibility !== "host_private")) fail("calibration_closure_visible_alias");
|
|
1158
|
-
const isAlias = (ref) => ref.id === reference.id || ref.path === reference.path || ref.contentHash === reference.contentHash;
|
|
1159
|
-
for (const task of value.taskset.tasks) {
|
|
1160
|
-
if (task.artifactRefs.some(isAlias) || (task.requiredOutputs ?? []).some((output) => output.schemaRef && isAlias(output.schemaRef)) || task.privilegedContextRef && value.files.some((file) => file.asset.id === task.privilegedContextRef && isAlias(file.asset)))
|
|
1161
|
-
fail("calibration_closure_task_alias");
|
|
1162
|
-
if (containsClosure(task.input, text, reference.contentHash) || containsClosure(task.policyVisibleContext, text, reference.contentHash))
|
|
1163
|
-
fail("calibration_closure_inline_alias");
|
|
1164
|
-
}
|
|
1165
|
-
for (const ref of [value.environment.actionSchemaRef, value.environment.observationSchemaRef, value.environment.stateSchemaRef])
|
|
1166
|
-
if (ref && isAlias(ref)) fail("calibration_closure_environment_schema_alias");
|
|
1167
|
-
for (const metadata of [value.taskset.metadata.environmentResources, value.environment.metadata.resources]) {
|
|
1168
|
-
if (metadata === void 0) continue;
|
|
1169
|
-
const resources = z11.array(z11.object({ id: z11.string().optional(), path: z11.string(), visibility: z11.string().optional() }).passthrough()).parse(metadata);
|
|
1170
|
-
for (const resource of resources) {
|
|
1171
|
-
const file = value.files.find((item) => item.asset.path === resource.path);
|
|
1172
|
-
if (resource.id === reference.id || resource.path === reference.path || file && isAlias(file.asset))
|
|
1173
|
-
fail("calibration_closure_environment_resource_alias");
|
|
1174
|
-
}
|
|
1175
|
-
}
|
|
1176
|
-
return closure;
|
|
1177
|
-
}
|
|
1178
|
-
function containsClosure(value, text, hash) {
|
|
1179
|
-
if (typeof value === "string") return value === text || sha256(value) === hash;
|
|
1180
|
-
if (!value || typeof value !== "object") return false;
|
|
1181
|
-
if (sha256(JSON.stringify(value)) === hash) return true;
|
|
1182
|
-
return Object.values(value).some((item) => containsClosure(item, text, hash));
|
|
1183
|
-
}
|
|
1184
|
-
function fail(code) {
|
|
1185
|
-
throw new Error(code);
|
|
1186
|
-
}
|
|
1187
|
-
function assertJudgeFixtureMembership(value, grader, selected, closure) {
|
|
1188
|
-
const graderId = grader.id;
|
|
1189
|
-
const authoring = value.taskset.metadata.ordinaryAuthoring ?? value.taskset.metadata.starterAuthoring;
|
|
1190
|
-
const mapping = z11.object({ judgeCalibrationFixtures: z11.record(z11.string(), z11.array(z11.string().min(1)).max(500)) }).passthrough().parse(authoring);
|
|
1191
|
-
const expected = closure.check.fixtureRefs.map((ref) => `calibration-${contentHash([selected.sourceReward, ref.id])}`);
|
|
1192
|
-
if (contentHash(mapping.judgeCalibrationFixtures[graderId] ?? []) !== contentHash(expected)) fail("calibration_judge_membership_changed");
|
|
1193
|
-
const fixtures = authoredFixtures(value);
|
|
1194
|
-
for (let index = 0; index < expected.length; index++) {
|
|
1195
|
-
const matches = fixtures.filter((fixture) => fixture.id === expected[index]);
|
|
1196
|
-
if (matches.length !== 1) fail("calibration_judge_membership_changed");
|
|
1197
|
-
const authored = matches[0];
|
|
1198
|
-
const projected = ProjectionSchema.parse(authored.metadata.rewardCalibration);
|
|
1199
|
-
const expectation = projected.fixture.expected;
|
|
1200
|
-
if (expectation.status !== "scored" || typeof expectation.passed !== "boolean" || authored.expectedPassed !== expectation.passed || authored.expectedRewardEligible !== grader.rewardEligible || authored.label !== (expectation.passed ? "positive" : "negative") || contentHash(authored.output) !== contentHash(projected.fixture.output) || authored.infrastructureError !== projected.fixture.infrastructureError) fail("calibration_judge_membership_changed");
|
|
1201
|
-
const retained = closure.check.fixtureRefs[index];
|
|
1202
|
-
const result = closure.check.results.find((item) => item.fixture.id === retained.id);
|
|
1203
|
-
if (authored.taskId !== authored.id || projected.fixture.id !== retained.id || projected.fixtureHash !== retained.contentHash || contentHash(projected.fixture) !== retained.contentHash || projected.checkResultHash !== contentHash(result) || contentHash(projected.sourceReward) !== contentHash(selected.sourceReward) || contentHash(projected.fixtureSet) !== contentHash(selected.fixtureSet) || contentHash(projected.calibrationCheck) !== contentHash(selected.calibrationCheck) || contentHash(projected.closureRef) !== contentHash(selected.closureRef) || (projected.graderId ?? projected.sourceReward.id) !== graderId) fail("calibration_judge_membership_changed");
|
|
1204
|
-
}
|
|
1205
|
-
}
|
|
1206
|
-
|
|
1207
|
-
// src/taskset-package-contracts.ts
|
|
1208
|
-
import { z as z12 } from "zod";
|
|
1209
|
-
import { EnvironmentReleaseSchema as EnvironmentReleaseSchema3, VerifierSetReleaseSchema as VerifierSetReleaseSchema3, verifyEnvironmentRelease as verifyEnvironmentRelease3, verifyVerifierSetRelease as verifyVerifierSetRelease3 } from "@openpond/evals";
|
|
1210
|
-
import { assertTasksetRelease as assertTasksetRelease2, TasksetReleaseSchema as TasksetReleaseSchema2 } from "@openpond/evals/tasksets";
|
|
1211
|
-
import { assertBoundedTaskJson as assertBoundedTaskJson2 } from "@openpond/evals/task-schema";
|
|
1212
|
-
import { taskBatchPackageMetadata as taskBatchPackageMetadata2 } from "@openpond/evals/learning";
|
|
1213
|
-
|
|
1214
|
-
// src/model-taskset-authoring-lineage.ts
|
|
1215
|
-
import { sameLearningRef as sameLearningRef3 } from "@openpond/evals/learning";
|
|
1216
|
-
function modelAuthoredTasksetId(lineage) {
|
|
1217
|
-
return `model-authored-taskset-${contentHash({ owner: lineage.owner, root: lineage.root })}`;
|
|
1218
|
-
}
|
|
1219
|
-
function assertModelTasksetAuthoring(taskset) {
|
|
1220
|
-
if (taskset.metadata.modelTasksetAuthoring === void 0) return null;
|
|
1221
|
-
const lineage = ModelTasksetAuthoringSchema.parse(taskset.metadata.modelTasksetAuthoring);
|
|
1222
|
-
if (taskset.id !== modelAuthoredTasksetId(lineage) || (taskset.revision === 1 ? !sameLearningRef3(lineage.parent, lineage.root) : lineage.parent.id !== taskset.id || lineage.parent.revision !== taskset.revision - 1)) {
|
|
1223
|
-
throw new Error("Authored Taskset revision differs from its ownership lineage.");
|
|
1224
|
-
}
|
|
1225
|
-
return lineage;
|
|
1226
|
-
}
|
|
1227
|
-
|
|
1228
|
-
// src/taskset-package-contracts.ts
|
|
1229
|
-
var BoundModelResourcesSchema = ModelTasksetPackageSchema.omit({ taskset: true, executionResources: true });
|
|
1230
|
-
var TasksetPackageContentSchema = z12.object({
|
|
1231
|
-
schemaVersion: z12.literal("openpond.tasksetPackage.v1"),
|
|
1232
|
-
taskset: TasksetReleaseSchema2,
|
|
1233
|
-
environment: EnvironmentReleaseSchema3,
|
|
1234
|
-
verifierSet: VerifierSetReleaseSchema3,
|
|
1235
|
-
files: z12.array(TasksetPackageFileSchema).max(1e4),
|
|
1236
|
-
modelResources: BoundModelResourcesSchema.optional(),
|
|
1237
|
-
learningResources: TasksetPackageLearningResourcesSchema.optional()
|
|
1238
|
-
}).strict();
|
|
1239
|
-
var TasksetPackageSchema = TasksetPackageContentSchema.extend({ contentHash: z12.string().regex(/^[a-f0-9]{64}$/) }).strict();
|
|
1240
|
-
function tasksetPackageRewardBinding(value) {
|
|
1241
|
-
return value.learningResources ? taskBatchPackageMetadata2(value.taskset).binding : value.modelResources?.rewardBinding ?? null;
|
|
1242
|
-
}
|
|
1243
|
-
function resolveTasksetPackageInstructions(value) {
|
|
1244
|
-
if (value.learningResources) return taskBatchPackageMetadata2(value.taskset).definition.instructions;
|
|
1245
|
-
if (value.modelResources) return value.modelResources.taskDefinition.instructions;
|
|
1246
|
-
const authoring = z12.object({ instructions: z12.string().max(2e4).optional() }).passthrough().parse(value.taskset.metadata.ordinaryAuthoring ?? {});
|
|
1247
|
-
return authoring.instructions ?? "";
|
|
1248
|
-
}
|
|
1249
|
-
function validateTasksetPackage(value) {
|
|
1250
|
-
assertBoundedTaskJson2(value, MAX_TASKSET_PACKAGE_BYTES);
|
|
1251
|
-
const result = TasksetPackageSchema.parse(value);
|
|
1252
|
-
const { contentHash: hash, ...content } = result;
|
|
1253
|
-
if (contentHash(content) !== hash) throw new Error("Taskset package content hash differs from its bytes.");
|
|
1254
|
-
const { taskset, environment, verifierSet } = result;
|
|
1255
|
-
assertTasksetRelease2(taskset);
|
|
1256
|
-
const authoring = assertModelTasksetAuthoring(taskset);
|
|
1257
|
-
if (authoring && (result.modelResources || result.learningResources)) throw new Error("Taskset package cannot declare competing authoring graphs.");
|
|
1258
|
-
if (!verifyEnvironmentRelease3(environment) || !verifyVerifierSetRelease3(verifierSet) || !same2(taskset.environmentRelease, { id: environment.id, contentHash: environment.contentHash }) || !same2(taskset.verifierSetRelease, { id: verifierSet.id, contentHash: verifierSet.contentHash }) || !same2(taskset.environment, environment.contract) || !same2(taskset.graders, verifierSet.graders)) {
|
|
1259
|
-
throw new Error("Taskset package execution releases do not match its Taskset.");
|
|
1260
|
-
}
|
|
1261
|
-
const files = /* @__PURE__ */ new Map();
|
|
1262
|
-
for (const file of result.files) {
|
|
1263
|
-
if (files.has(file.asset.id)) throw new Error(`Duplicate Taskset asset identity: ${file.asset.id}.`);
|
|
1264
|
-
decodeTasksetPackageFile(file);
|
|
1265
|
-
files.set(file.asset.id, file);
|
|
1266
|
-
}
|
|
1267
|
-
function requireAsset(ref) {
|
|
1268
|
-
const file = files.get(ref.id);
|
|
1269
|
-
if (!file || !same2(file.asset, ref)) throw new Error(`Taskset package asset is missing or mismatched: ${ref.id}.`);
|
|
1270
|
-
}
|
|
1271
|
-
for (const task of taskset.tasks) {
|
|
1272
|
-
for (const asset of task.artifactRefs) requireAsset(asset);
|
|
1273
|
-
for (const output of task.requiredOutputs ?? []) if (output.schemaRef) requireAsset(output.schemaRef);
|
|
1274
|
-
if (task.privilegedContextRef && files.get(task.privilegedContextRef)?.asset.visibility !== "host_private") {
|
|
1275
|
-
throw new Error(`Taskset package private context is missing: ${task.id}.`);
|
|
1276
|
-
}
|
|
1277
|
-
}
|
|
1278
|
-
for (const ref of [environment.actionSchemaRef, environment.observationSchemaRef, environment.stateSchemaRef]) if (ref) requireAsset(ref);
|
|
1279
|
-
for (const grader of taskset.graders) {
|
|
1280
|
-
const ref = "verifierRef" in grader ? grader.verifierRef : "rubricRef" in grader ? grader.rubricRef : null;
|
|
1281
|
-
if (ref) {
|
|
1282
|
-
requireAsset(ref);
|
|
1283
|
-
if (ref.visibility === "policy") throw new Error(`Taskset grader asset must be private: ${ref.id}.`);
|
|
1284
|
-
}
|
|
1285
|
-
}
|
|
1286
|
-
if (taskset.metrics?.customAggregator) {
|
|
1287
|
-
const aggregator = taskset.metrics.customAggregator;
|
|
1288
|
-
const modules = result.files.filter((file) => file.asset.path === aggregator.module);
|
|
1289
|
-
if (modules.length !== 1 || modules[0].asset.contentHash !== aggregator.contentHash || modules[0].asset.sizeBytes > 524288 || modules[0].asset.visibility === "policy") {
|
|
1290
|
-
throw new Error("Taskset metric module is missing or mismatched, public, or exceeds the source limit.");
|
|
1291
|
-
}
|
|
1292
|
-
}
|
|
1293
|
-
for (const declarations of [taskset.metadata.environmentResources, environment.metadata.resources]) {
|
|
1294
|
-
if (declarations === void 0) continue;
|
|
1295
|
-
for (const resource of z12.array(z12.object({ path: z12.string().min(1) }).passthrough()).parse(declarations)) {
|
|
1296
|
-
if (!result.files.some((file) => file.asset.path === resource.path)) throw new Error(`Taskset environment resource is missing: ${resource.path}.`);
|
|
1297
|
-
}
|
|
1298
|
-
}
|
|
1299
|
-
if (result.learningResources) {
|
|
1300
|
-
if (result.modelResources) throw new Error("Taskset package cannot declare competing authoring graphs.");
|
|
1301
|
-
const learning = validateTasksetLearningResources(taskset, result.learningResources);
|
|
1302
|
-
for (const expected of learningPackageContextFiles(learning)) {
|
|
1303
|
-
if (!same2(files.get(expected.asset.id), expected)) throw new Error("Taskset private context differs from its reviewed evidence.");
|
|
1304
|
-
}
|
|
1305
|
-
for (const asset of learning.assets) requireAsset(asset.asset);
|
|
1306
|
-
} else if (result.modelResources) {
|
|
1307
|
-
const model = validateModelTasksetPackage({ ...result.modelResources, taskset, executionResources: { environment, verifierSet } });
|
|
1308
|
-
for (const asset of model.assets) requireAsset(asset.asset);
|
|
1309
|
-
} else if (["starter", "rewardBinding", "rewardExecution", "modelTasksetDerivation", "learning"].some((key) => taskset.metadata[key] !== void 0)) {
|
|
1310
|
-
throw new Error("Bound Taskset publication requires its complete model resources.");
|
|
1311
|
-
}
|
|
1312
|
-
resolveTasksetPackageExecution(result);
|
|
1313
|
-
resolveTasksetPackageInstructions(result);
|
|
1314
|
-
assertTasksetCalibrationClosures(result);
|
|
1315
|
-
return result;
|
|
1316
|
-
}
|
|
1317
|
-
function createTasksetPackage(value) {
|
|
1318
|
-
assertBoundedTaskJson2(value, MAX_TASKSET_PACKAGE_BYTES);
|
|
1319
|
-
const content = TasksetPackageContentSchema.parse(value);
|
|
1320
|
-
return validateTasksetPackage({ ...content, contentHash: contentHash(content) });
|
|
1321
|
-
}
|
|
1322
|
-
function same2(left, right) {
|
|
1323
|
-
return contentHash({ value: left }) === contentHash({ value: right });
|
|
1324
|
-
}
|
|
1325
|
-
|
|
1326
|
-
export {
|
|
1327
|
-
ConnectedDatasetSourceRefSchema,
|
|
1328
|
-
DatasetSplitSchema,
|
|
1329
|
-
DatasetSemanticFieldSchema,
|
|
1330
|
-
DatasetSemanticSchemaSchema,
|
|
1331
|
-
DatasetShardRefSchema,
|
|
1332
|
-
DatasetArtifactManifestSchema,
|
|
1333
|
-
DatasetArtifactSummarySchema,
|
|
1334
|
-
DatasetCatalogItemSchema,
|
|
1335
|
-
DatasetCatalogResponseSchema,
|
|
1336
|
-
DatasetArtifactRegistryEntrySchema,
|
|
1337
|
-
DatasetRowPageRequestSchema,
|
|
1338
|
-
DatasetRowPageSchema,
|
|
1339
|
-
DatasetStorageSettingsSchema,
|
|
1340
|
-
DatasetStorageRootSchema,
|
|
1341
|
-
DatasetStorageStateSchema,
|
|
1342
|
-
UpdateDatasetStorageSettingsRequestSchema,
|
|
1343
|
-
DatasetSourceReviewStatusSchema,
|
|
1344
|
-
HuggingFaceDatasetSourceRefSchema,
|
|
1345
|
-
UploadedFileDatasetSourceRefSchema,
|
|
1346
|
-
GeneratedDatasetSourceRefSchema,
|
|
1347
|
-
LearningBatchDatasetSourceRefSchema,
|
|
1348
|
-
ExternalDatasetSourceRefSchema,
|
|
1349
|
-
HarnessActionBindingSchema,
|
|
1350
|
-
TasksetSplitSchema,
|
|
1351
|
-
TasksetPurposeSchema,
|
|
1352
|
-
TasksetBenchmarkBindingSchema,
|
|
1353
|
-
TasksetPreferenceComparisonBindingSchema,
|
|
1354
|
-
TrainingSourceConsentSchema,
|
|
1355
|
-
TrainingSourceRefSchema,
|
|
1356
|
-
TasksetSourceRefSchema,
|
|
1357
|
-
TaskPolicyBoundarySchema,
|
|
1358
|
-
TaskAssetRefSchema,
|
|
1359
|
-
TaskRequiredOutputSchema,
|
|
1360
|
-
TaskDataRecordSchema,
|
|
1361
|
-
TasksetEnvironmentResourceSchema,
|
|
1362
|
-
DemonstrationSignalSchema,
|
|
1363
|
-
PreferenceSignalSchema,
|
|
1364
|
-
CorrectionSignalSchema,
|
|
1365
|
-
FeedbackSignalSchema,
|
|
1366
|
-
RewardSignalSchema,
|
|
1367
|
-
LabelSignalSchema,
|
|
1368
|
-
LearningSignalRefSchema,
|
|
1369
|
-
LearningSignalInventorySchema,
|
|
1370
|
-
TasksetEnvironmentContractSchema,
|
|
1371
|
-
TasksetCapabilityManifestSchema,
|
|
1372
|
-
DeterministicGraderSpecSchema,
|
|
1373
|
-
RubricGraderSpecSchema,
|
|
1374
|
-
HumanGraderSpecSchema,
|
|
1375
|
-
CustomVerifierGraderSpecSchema,
|
|
1376
|
-
GraderSpecSchema,
|
|
1377
|
-
GraderFixtureLabelSchema,
|
|
1378
|
-
GraderFixtureSchema,
|
|
1379
|
-
modelAuthoredTasksetId,
|
|
1380
|
-
assertModelTasksetAuthoring,
|
|
1381
|
-
ModelStarterEnvironmentAttemptSchema,
|
|
1382
|
-
verifyModelStarterEnvironmentAttempt,
|
|
1383
|
-
ModelStarterExecutionSchema,
|
|
1384
|
-
ModelStarterToolFixtureScriptSchema,
|
|
1385
|
-
modelStarterExecutionAssetId,
|
|
1386
|
-
createModelStarterExecutionAsset,
|
|
1387
|
-
resolveModelStarterExecutionAsset,
|
|
1388
|
-
validateModelStarterExecution,
|
|
1389
|
-
NativeHarnessExperimentAttemptSchema,
|
|
1390
|
-
verifyNativeHarnessExperimentAttempt,
|
|
1391
|
-
ModelTasksetExecutionResourcesSchema,
|
|
1392
|
-
modelTasksetExecutionResourcesAssetId,
|
|
1393
|
-
createModelTasksetExecutionResourcesAsset,
|
|
1394
|
-
resolveModelTasksetExecutionResourcesAsset,
|
|
1395
|
-
ModelTasksetDerivationSchema,
|
|
1396
|
-
ModelTasksetPackageSchema,
|
|
1397
|
-
deriveModelTaskset,
|
|
1398
|
-
compileModelTasksetReward,
|
|
1399
|
-
validateModelTasksetPackage,
|
|
1400
|
-
TasksetPackageLearningResourcesSchema,
|
|
1401
|
-
learningPackageContextFiles,
|
|
1402
|
-
validateTasksetLearningResources,
|
|
1403
|
-
MAX_TASKSET_PACKAGE_BYTES,
|
|
1404
|
-
TasksetPackageFileSchema,
|
|
1405
|
-
decodeTasksetPackageFile,
|
|
1406
|
-
resolveTasksetPackageExecution,
|
|
1407
|
-
createTasksetPackageExecutionFile,
|
|
1408
|
-
assertTasksetCalibrationClosures,
|
|
1409
|
-
resolveTasksetCalibrationFixture,
|
|
1410
|
-
TasksetPackageContentSchema,
|
|
1411
|
-
TasksetPackageSchema,
|
|
1412
|
-
tasksetPackageRewardBinding,
|
|
1413
|
-
resolveTasksetPackageInstructions,
|
|
1414
|
-
validateTasksetPackage,
|
|
1415
|
-
createTasksetPackage
|
|
1416
|
-
};
|
|
1417
|
-
//# sourceMappingURL=chunk-FHCSDXIA.js.map
|