agent-hitch 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -3
- package/README.zh-CN.md +14 -2
- package/dist/bin/hitch.js +25 -3
- package/dist/bin/hitch.js.map +1 -1
- package/dist/scripts/canary-benchmark.js +54 -0
- package/dist/scripts/canary-benchmark.js.map +1 -0
- package/dist/scripts/canary-buildkit-secrets.js +166 -0
- package/dist/scripts/canary-buildkit-secrets.js.map +1 -0
- package/dist/scripts/canary-collision-domains.js +77 -0
- package/dist/scripts/canary-collision-domains.js.map +1 -0
- package/dist/scripts/canary-docker-image-observation.js +181 -0
- package/dist/scripts/canary-docker-image-observation.js.map +1 -0
- package/dist/scripts/canary-eval-scheduler-throughput.js +47 -0
- package/dist/scripts/canary-eval-scheduler-throughput.js.map +1 -0
- package/dist/scripts/canary-gpu-hardware.js +95 -0
- package/dist/scripts/canary-gpu-hardware.js.map +1 -0
- package/dist/scripts/canary-harbor-load.js +365 -0
- package/dist/scripts/canary-harbor-load.js.map +1 -0
- package/dist/scripts/canary-packaged-harness.js +75 -0
- package/dist/scripts/canary-packaged-harness.js.map +1 -0
- package/dist/scripts/canary-resource-load.js +188 -0
- package/dist/scripts/canary-resource-load.js.map +1 -0
- package/dist/scripts/check-architecture.js +10 -5
- package/dist/scripts/check-architecture.js.map +1 -1
- package/dist/scripts/docker-reaper-crash-child.js +21 -0
- package/dist/scripts/docker-reaper-crash-child.js.map +1 -0
- package/dist/src/adapters/catalog.js +3 -1
- package/dist/src/adapters/catalog.js.map +1 -1
- package/dist/src/adapters/discovery.js +1 -1
- package/dist/src/adapters/discovery.js.map +1 -1
- package/dist/src/adapters/providers/claude.js +6 -0
- package/dist/src/adapters/providers/claude.js.map +1 -1
- package/dist/src/adapters/providers/codex-auth.js +29 -0
- package/dist/src/adapters/providers/codex-auth.js.map +1 -0
- package/dist/src/adapters/providers/codex.js +29 -6
- package/dist/src/adapters/providers/codex.js.map +1 -1
- package/dist/src/adapters/providers/deepseek.js +6 -0
- package/dist/src/adapters/providers/deepseek.js.map +1 -1
- package/dist/src/adapters/providers/model-call.js +26 -0
- package/dist/src/adapters/providers/model-call.js.map +1 -0
- package/dist/src/adapters/providers/opencode.js +6 -0
- package/dist/src/adapters/providers/opencode.js.map +1 -1
- package/dist/src/adapters/providers/pi.js +6 -0
- package/dist/src/adapters/providers/pi.js.map +1 -1
- package/dist/src/artifacts/index.js +1 -0
- package/dist/src/artifacts/index.js.map +1 -1
- package/dist/src/artifacts/preparer.js +7 -2
- package/dist/src/artifacts/preparer.js.map +1 -1
- package/dist/src/backends/harbor/agent-budget.js +31 -0
- package/dist/src/backends/harbor/agent-budget.js.map +1 -0
- package/dist/src/backends/harbor/backend.js +59 -107
- package/dist/src/backends/harbor/backend.js.map +1 -1
- package/dist/src/backends/harbor/environment-config.js +92 -0
- package/dist/src/backends/harbor/environment-config.js.map +1 -0
- package/dist/src/backends/harbor/index.js +6 -1
- package/dist/src/backends/harbor/index.js.map +1 -1
- package/dist/src/backends/harbor/model-proxy-config.js +23 -0
- package/dist/src/backends/harbor/model-proxy-config.js.map +1 -0
- package/dist/src/backends/harbor/process.js +177 -0
- package/dist/src/backends/harbor/process.js.map +1 -0
- package/dist/src/backends/harbor/regrade.js +77 -0
- package/dist/src/backends/harbor/regrade.js.map +1 -0
- package/dist/src/backends/harbor/runtime-toolchain.js +30 -0
- package/dist/src/backends/harbor/runtime-toolchain.js.map +1 -0
- package/dist/src/backends/harbor/supervisor.js +74 -0
- package/dist/src/backends/harbor/supervisor.js.map +1 -0
- package/dist/src/backends/harbor/task-resources.js +206 -0
- package/dist/src/backends/harbor/task-resources.js.map +1 -0
- package/dist/src/backends/harbor/tools.js +6 -29
- package/dist/src/backends/harbor/tools.js.map +1 -1
- package/dist/src/backends/index.js +5 -1
- package/dist/src/backends/index.js.map +1 -1
- package/dist/src/benchmarks/index.js +3 -0
- package/dist/src/benchmarks/index.js.map +1 -0
- package/dist/src/benchmarks/loader.js +271 -0
- package/dist/src/benchmarks/loader.js.map +1 -0
- package/dist/src/benchmarks/metrics.js +16 -0
- package/dist/src/benchmarks/metrics.js.map +1 -0
- package/dist/src/benchmarks/toml.js +13 -0
- package/dist/src/benchmarks/toml.js.map +1 -0
- package/dist/src/benchmarks/validation.js +242 -0
- package/dist/src/benchmarks/validation.js.map +1 -0
- package/dist/src/cli/arguments.js +6 -4
- package/dist/src/cli/arguments.js.map +1 -1
- package/dist/src/cli/commands/benchmark.js +22 -0
- package/dist/src/cli/commands/benchmark.js.map +1 -0
- package/dist/src/cli/commands/capabilities.js +18 -0
- package/dist/src/cli/commands/capabilities.js.map +1 -0
- package/dist/src/cli/commands/daemon.js +135 -8
- package/dist/src/cli/commands/daemon.js.map +1 -1
- package/dist/src/cli/commands/eval.js +228 -6
- package/dist/src/cli/commands/eval.js.map +1 -1
- package/dist/src/cli/commands/images.js +40 -0
- package/dist/src/cli/commands/images.js.map +1 -0
- package/dist/src/cli/commands/run.js +42 -17
- package/dist/src/cli/commands/run.js.map +1 -1
- package/dist/src/cli/commands/runs.js +25 -3
- package/dist/src/cli/commands/runs.js.map +1 -1
- package/dist/src/cli/commands/trajectory.js +96 -5
- package/dist/src/cli/commands/trajectory.js.map +1 -1
- package/dist/src/cli/commands/verifier.js +36 -0
- package/dist/src/cli/commands/verifier.js.map +1 -0
- package/dist/src/cli/commands/worker.js +79 -0
- package/dist/src/cli/commands/worker.js.map +1 -0
- package/dist/src/cli/main.js +10 -0
- package/dist/src/cli/main.js.map +1 -1
- package/dist/src/cli/output.js +84 -4
- package/dist/src/cli/output.js.map +1 -1
- package/dist/src/control-plane/build-admission.js +69 -0
- package/dist/src/control-plane/build-admission.js.map +1 -0
- package/dist/src/control-plane/build-records.js +6 -0
- package/dist/src/control-plane/build-records.js.map +1 -0
- package/dist/src/control-plane/collisions.js +40 -0
- package/dist/src/control-plane/collisions.js.map +1 -0
- package/dist/src/control-plane/eval-control-state.js +22 -0
- package/dist/src/control-plane/eval-control-state.js.map +1 -0
- package/dist/src/control-plane/eval-control-work.js +44 -0
- package/dist/src/control-plane/eval-control-work.js.map +1 -0
- package/dist/src/control-plane/eval-image-resolution.js +52 -0
- package/dist/src/control-plane/eval-image-resolution.js.map +1 -0
- package/dist/src/control-plane/eval-image-services.js +30 -0
- package/dist/src/control-plane/eval-image-services.js.map +1 -0
- package/dist/src/control-plane/eval-records.js +252 -0
- package/dist/src/control-plane/eval-records.js.map +1 -0
- package/dist/src/control-plane/eval-recovery.js +181 -0
- package/dist/src/control-plane/eval-recovery.js.map +1 -0
- package/dist/src/control-plane/eval-scheduler.js +450 -0
- package/dist/src/control-plane/eval-scheduler.js.map +1 -0
- package/dist/src/control-plane/index.js +20 -0
- package/dist/src/control-plane/index.js.map +1 -0
- package/dist/src/control-plane/local-worker.js +49 -0
- package/dist/src/control-plane/local-worker.js.map +1 -0
- package/dist/src/control-plane/model-capture-planning.js +12 -0
- package/dist/src/control-plane/model-capture-planning.js.map +1 -0
- package/dist/src/control-plane/remote-result-transport.js +175 -0
- package/dist/src/control-plane/remote-result-transport.js.map +1 -0
- package/dist/src/control-plane/remote-work-coordinator.js +333 -0
- package/dist/src/control-plane/remote-work-coordinator.js.map +1 -0
- package/dist/src/control-plane/remote-work-inputs.js +266 -0
- package/dist/src/control-plane/remote-work-inputs.js.map +1 -0
- package/dist/src/control-plane/remote-work-item.js +62 -0
- package/dist/src/control-plane/remote-work-item.js.map +1 -0
- package/dist/src/control-plane/remote-work-recovery.js +220 -0
- package/dist/src/control-plane/remote-work-recovery.js.map +1 -0
- package/dist/src/control-plane/remote-worker-artifacts.js +135 -0
- package/dist/src/control-plane/remote-worker-artifacts.js.map +1 -0
- package/dist/src/control-plane/remote-worker-client.js +202 -0
- package/dist/src/control-plane/remote-worker-client.js.map +1 -0
- package/dist/src/control-plane/remote-worker-configuration.js +34 -0
- package/dist/src/control-plane/remote-worker-configuration.js.map +1 -0
- package/dist/src/control-plane/remote-worker-credentials.js +58 -0
- package/dist/src/control-plane/remote-worker-credentials.js.map +1 -0
- package/dist/src/control-plane/remote-worker-protocol.js +489 -0
- package/dist/src/control-plane/remote-worker-protocol.js.map +1 -0
- package/dist/src/control-plane/remote-worker-runner.js +214 -0
- package/dist/src/control-plane/remote-worker-runner.js.map +1 -0
- package/dist/src/control-plane/remote-workers.js +377 -0
- package/dist/src/control-plane/remote-workers.js.map +1 -0
- package/dist/src/control-plane/rerun-scheduler.js +448 -0
- package/dist/src/control-plane/rerun-scheduler.js.map +1 -0
- package/dist/src/control-plane/rerun-submission.js +62 -0
- package/dist/src/control-plane/rerun-submission.js.map +1 -0
- package/dist/src/control-plane/resources.js +182 -0
- package/dist/src/control-plane/resources.js.map +1 -0
- package/dist/src/control-plane/scheduler-eval-entry.js +32 -0
- package/dist/src/control-plane/scheduler-eval-entry.js.map +1 -0
- package/dist/src/control-plane/synthetic-result.js +21 -0
- package/dist/src/control-plane/synthetic-result.js.map +1 -0
- package/dist/src/control-plane/work-admission.js +23 -0
- package/dist/src/control-plane/work-admission.js.map +1 -0
- package/dist/src/control-plane/work-dispatcher.js +237 -0
- package/dist/src/control-plane/work-dispatcher.js.map +1 -0
- package/dist/src/controller-runtime/hash.js +19 -3
- package/dist/src/controller-runtime/hash.js.map +1 -1
- package/dist/src/controller-runtime/index.js +1 -1
- package/dist/src/controller-runtime/index.js.map +1 -1
- package/dist/src/controller-runtime/store.js +15 -6
- package/dist/src/controller-runtime/store.js.map +1 -1
- package/dist/src/daemon/auth.js +5 -0
- package/dist/src/daemon/auth.js.map +1 -1
- package/dist/src/daemon/client.js.map +1 -1
- package/dist/src/daemon/event-stream.js +46 -0
- package/dist/src/daemon/event-stream.js.map +1 -0
- package/dist/src/daemon/health.js +66 -0
- package/dist/src/daemon/health.js.map +1 -0
- package/dist/src/daemon/launcher.js +28 -1
- package/dist/src/daemon/launcher.js.map +1 -1
- package/dist/src/daemon/logging.js +23 -0
- package/dist/src/daemon/logging.js.map +1 -0
- package/dist/src/daemon/scheduler.js +26 -5
- package/dist/src/daemon/scheduler.js.map +1 -1
- package/dist/src/daemon/server.js +255 -98
- package/dist/src/daemon/server.js.map +1 -1
- package/dist/src/daemon/telemetry.js +163 -0
- package/dist/src/daemon/telemetry.js.map +1 -0
- package/dist/src/daemon/worker-routes.js +219 -0
- package/dist/src/daemon/worker-routes.js.map +1 -0
- package/dist/src/domain/benchmarks.js +2 -0
- package/dist/src/domain/benchmarks.js.map +1 -0
- package/dist/src/domain/bundles.js +2 -0
- package/dist/src/domain/bundles.js.map +1 -0
- package/dist/src/domain/control-plane.js +2 -0
- package/dist/src/domain/control-plane.js.map +1 -0
- package/dist/src/domain/execution-evidence.js +2 -0
- package/dist/src/domain/execution-evidence.js.map +1 -0
- package/dist/src/domain/execution-plan.js +2 -0
- package/dist/src/domain/execution-plan.js.map +1 -0
- package/dist/src/domain/images.js +2 -0
- package/dist/src/domain/images.js.map +1 -0
- package/dist/src/domain/index.js +13 -0
- package/dist/src/domain/index.js.map +1 -1
- package/dist/src/domain/interactions.js +2 -0
- package/dist/src/domain/interactions.js.map +1 -0
- package/dist/src/domain/providers.js +2 -0
- package/dist/src/domain/providers.js.map +1 -0
- package/dist/src/domain/resources.js +2 -0
- package/dist/src/domain/resources.js.map +1 -0
- package/dist/src/domain/runs.js.map +1 -1
- package/dist/src/domain/validation.js +11 -4
- package/dist/src/domain/validation.js.map +1 -1
- package/dist/src/domain/verifier-evidence-validation.js +214 -0
- package/dist/src/domain/verifier-evidence-validation.js.map +1 -0
- package/dist/src/domain/verifier-evidence.js +3 -0
- package/dist/src/domain/verifier-evidence.js.map +1 -0
- package/dist/src/domain/verifier-score-contract.js +232 -0
- package/dist/src/domain/verifier-score-contract.js.map +1 -0
- package/dist/src/domain/workers.js +2 -0
- package/dist/src/domain/workers.js.map +1 -0
- package/dist/src/evals/benchmark-adapter-manifest.js +219 -0
- package/dist/src/evals/benchmark-adapter-manifest.js.map +1 -0
- package/dist/src/evals/benchmark-candidate.js +36 -0
- package/dist/src/evals/benchmark-candidate.js.map +1 -0
- package/dist/src/evals/benchmark-run.js +166 -0
- package/dist/src/evals/benchmark-run.js.map +1 -0
- package/dist/src/evals/collect-only-rerun.js +183 -0
- package/dist/src/evals/collect-only-rerun.js.map +1 -0
- package/dist/src/evals/directory.js +30 -0
- package/dist/src/evals/directory.js.map +1 -0
- package/dist/src/evals/docker-engine-stats.js +89 -0
- package/dist/src/evals/docker-engine-stats.js.map +1 -0
- package/dist/src/evals/docker-ownership.js +51 -0
- package/dist/src/evals/docker-ownership.js.map +1 -0
- package/dist/src/evals/docker-reaper.js +144 -0
- package/dist/src/evals/docker-reaper.js.map +1 -0
- package/dist/src/evals/docker-resource-observer.js +207 -0
- package/dist/src/evals/docker-resource-observer.js.map +1 -0
- package/dist/src/evals/duration-estimator.js +126 -0
- package/dist/src/evals/duration-estimator.js.map +1 -0
- package/dist/src/evals/environment-image-planning.js +164 -0
- package/dist/src/evals/environment-image-planning.js.map +1 -0
- package/dist/src/evals/eval-artifact-planning.js +56 -0
- package/dist/src/evals/eval-artifact-planning.js.map +1 -0
- package/dist/src/evals/eval-finalization.js +10 -0
- package/dist/src/evals/eval-finalization.js.map +1 -0
- package/dist/src/evals/eval-lifecycle-events.js +29 -0
- package/dist/src/evals/eval-lifecycle-events.js.map +1 -0
- package/dist/src/evals/eval-logical-plan.js +51 -0
- package/dist/src/evals/eval-logical-plan.js.map +1 -0
- package/dist/src/evals/eval-result-builder.js +81 -0
- package/dist/src/evals/eval-result-builder.js.map +1 -0
- package/dist/src/evals/events.js +53 -5
- package/dist/src/evals/events.js.map +1 -1
- package/dist/src/evals/evolution-baseline.js +191 -0
- package/dist/src/evals/evolution-baseline.js.map +1 -0
- package/dist/src/evals/execution-evidence.js +116 -0
- package/dist/src/evals/execution-evidence.js.map +1 -0
- package/dist/src/evals/execution-leases.js +342 -0
- package/dist/src/evals/execution-leases.js.map +1 -0
- package/dist/src/evals/execution-plan-artifacts.js +72 -0
- package/dist/src/evals/execution-plan-artifacts.js.map +1 -0
- package/dist/src/evals/execution-plan-images.js +69 -0
- package/dist/src/evals/execution-plan-images.js.map +1 -0
- package/dist/src/evals/execution-plan-resources.js +174 -0
- package/dist/src/evals/execution-plan-resources.js.map +1 -0
- package/dist/src/evals/execution-plan.js +451 -0
- package/dist/src/evals/execution-plan.js.map +1 -0
- package/dist/src/evals/failure-classifier.js +61 -0
- package/dist/src/evals/failure-classifier.js.map +1 -0
- package/dist/src/evals/harbor-artifact-builder-image.js +99 -0
- package/dist/src/evals/harbor-artifact-builder-image.js.map +1 -0
- package/dist/src/evals/harbor-artifact-builder.js +358 -0
- package/dist/src/evals/harbor-artifact-builder.js.map +1 -0
- package/dist/src/evals/harbor-bridge-error.js +58 -3
- package/dist/src/evals/harbor-bridge-error.js.map +1 -1
- package/dist/src/evals/harbor-node-runtime.js +146 -0
- package/dist/src/evals/harbor-node-runtime.js.map +1 -0
- package/dist/src/evals/harbor-phase-timings.js +60 -0
- package/dist/src/evals/harbor-phase-timings.js.map +1 -0
- package/dist/src/evals/index.js +40 -2
- package/dist/src/evals/index.js.map +1 -1
- package/dist/src/evals/infrastructure-retry.js +208 -84
- package/dist/src/evals/infrastructure-retry.js.map +1 -1
- package/dist/src/evals/interaction-capture-import.js +49 -0
- package/dist/src/evals/interaction-capture-import.js.map +1 -0
- package/dist/src/evals/local-docker-provider.js +372 -0
- package/dist/src/evals/local-docker-provider.js.map +1 -0
- package/dist/src/evals/local-eval-planning.js +80 -0
- package/dist/src/evals/local-eval-planning.js.map +1 -0
- package/dist/src/evals/model-capture-plan.js +22 -0
- package/dist/src/evals/model-capture-plan.js.map +1 -0
- package/dist/src/evals/model-capture-runtime.js +114 -0
- package/dist/src/evals/model-capture-runtime.js.map +1 -0
- package/dist/src/evals/model-proxy-runtime-state.js +53 -0
- package/dist/src/evals/model-proxy-runtime-state.js.map +1 -0
- package/dist/src/evals/native-phase-evidence.js +440 -0
- package/dist/src/evals/native-phase-evidence.js.map +1 -0
- package/dist/src/evals/physical-retry-work.js +31 -0
- package/dist/src/evals/physical-retry-work.js.map +1 -0
- package/dist/src/evals/planned-execution-support.js +66 -0
- package/dist/src/evals/planned-execution-support.js.map +1 -0
- package/dist/src/evals/planned-execution.js +469 -0
- package/dist/src/evals/planned-execution.js.map +1 -0
- package/dist/src/evals/planned-progress-publisher.js +62 -0
- package/dist/src/evals/planned-progress-publisher.js.map +1 -0
- package/dist/src/evals/planned-remote-execution.js +40 -0
- package/dist/src/evals/planned-remote-execution.js.map +1 -0
- package/dist/src/evals/planned-retry-execution.js +174 -0
- package/dist/src/evals/planned-retry-execution.js.map +1 -0
- package/dist/src/evals/planned-retry-lifecycle.js +188 -0
- package/dist/src/evals/planned-retry-lifecycle.js.map +1 -0
- package/dist/src/evals/preparation-rerun.js +225 -0
- package/dist/src/evals/preparation-rerun.js.map +1 -0
- package/dist/src/evals/prepared-harness.js +76 -0
- package/dist/src/evals/prepared-harness.js.map +1 -0
- package/dist/src/evals/progress.js +52 -12
- package/dist/src/evals/progress.js.map +1 -1
- package/dist/src/evals/records.js +20 -3
- package/dist/src/evals/records.js.map +1 -1
- package/dist/src/evals/recovery.js +242 -0
- package/dist/src/evals/recovery.js.map +1 -0
- package/dist/src/evals/regrade-evidence.js +77 -0
- package/dist/src/evals/regrade-evidence.js.map +1 -0
- package/dist/src/evals/remote-infrastructure-retry.js +165 -0
- package/dist/src/evals/remote-infrastructure-retry.js.map +1 -0
- package/dist/src/evals/request.js +21 -3
- package/dist/src/evals/request.js.map +1 -1
- package/dist/src/evals/rerun-inputs.js +108 -0
- package/dist/src/evals/rerun-inputs.js.map +1 -0
- package/dist/src/evals/rerun-slots.js +4 -3
- package/dist/src/evals/rerun-slots.js.map +1 -1
- package/dist/src/evals/rerun-types.js +45 -0
- package/dist/src/evals/rerun-types.js.map +1 -0
- package/dist/src/evals/rerun.js +133 -88
- package/dist/src/evals/rerun.js.map +1 -1
- package/dist/src/evals/result-helpers.js +78 -0
- package/dist/src/evals/result-helpers.js.map +1 -0
- package/dist/src/evals/resume-state.js +36 -0
- package/dist/src/evals/resume-state.js.map +1 -0
- package/dist/src/evals/retry-backoff.js +20 -0
- package/dist/src/evals/retry-backoff.js.map +1 -0
- package/dist/src/evals/retry-state.js +195 -0
- package/dist/src/evals/retry-state.js.map +1 -0
- package/dist/src/evals/scheduler-metrics.js +95 -0
- package/dist/src/evals/scheduler-metrics.js.map +1 -0
- package/dist/src/evals/scheduler-trace-replay.js +107 -0
- package/dist/src/evals/scheduler-trace-replay.js.map +1 -0
- package/dist/src/evals/service-types.js +2 -0
- package/dist/src/evals/service-types.js.map +1 -0
- package/dist/src/evals/service.js +331 -347
- package/dist/src/evals/service.js.map +1 -1
- package/dist/src/evals/task-resources.js +209 -0
- package/dist/src/evals/task-resources.js.map +1 -0
- package/dist/src/evals/trial-environment-evidence.js +87 -0
- package/dist/src/evals/trial-environment-evidence.js.map +1 -0
- package/dist/src/evals/trial-execution-evidence.js +13 -0
- package/dist/src/evals/trial-execution-evidence.js.map +1 -0
- package/dist/src/evals/trial-import-identity.js +31 -0
- package/dist/src/evals/trial-import-identity.js.map +1 -0
- package/dist/src/evals/trial-import.js +183 -188
- package/dist/src/evals/trial-import.js.map +1 -1
- package/dist/src/evals/trial-publication-recovery.js +144 -0
- package/dist/src/evals/trial-publication-recovery.js.map +1 -0
- package/dist/src/evals/trial-publication.js +25 -0
- package/dist/src/evals/trial-publication.js.map +1 -0
- package/dist/src/evals/trial-reference-validation.js +49 -0
- package/dist/src/evals/trial-reference-validation.js.map +1 -0
- package/dist/src/evals/verifier-artifacts.js +317 -0
- package/dist/src/evals/verifier-artifacts.js.map +1 -0
- package/dist/src/evals/verifier-eligibility.js +25 -0
- package/dist/src/evals/verifier-eligibility.js.map +1 -0
- package/dist/src/evals/verifier-only-rerun.js +210 -0
- package/dist/src/evals/verifier-only-rerun.js.map +1 -0
- package/dist/src/evals/verifier-runtime.js +61 -0
- package/dist/src/evals/verifier-runtime.js.map +1 -0
- package/dist/src/evals/verifier-score-artifacts.js +119 -0
- package/dist/src/evals/verifier-score-artifacts.js.map +1 -0
- package/dist/src/evals/work-item-artifacts.js +23 -0
- package/dist/src/evals/work-item-artifacts.js.map +1 -0
- package/dist/src/foundation/config.js +17 -0
- package/dist/src/foundation/config.js.map +1 -1
- package/dist/src/foundation/contained-file.js +85 -0
- package/dist/src/foundation/contained-file.js.map +1 -0
- package/dist/src/foundation/credential-redaction.js +162 -0
- package/dist/src/foundation/credential-redaction.js.map +1 -0
- package/dist/src/foundation/environment-image-references.js +59 -0
- package/dist/src/foundation/environment-image-references.js.map +1 -0
- package/dist/src/foundation/executable.js +45 -5
- package/dist/src/foundation/executable.js.map +1 -1
- package/dist/src/foundation/fs.js +2 -1
- package/dist/src/foundation/fs.js.map +1 -1
- package/dist/src/foundation/index.js +6 -2
- package/dist/src/foundation/index.js.map +1 -1
- package/dist/src/foundation/process-identity.js +69 -0
- package/dist/src/foundation/process-identity.js.map +1 -0
- package/dist/src/images/context.js +47 -0
- package/dist/src/images/context.js.map +1 -0
- package/dist/src/images/docker-buildkit.js +137 -0
- package/dist/src/images/docker-buildkit.js.map +1 -0
- package/dist/src/images/dockerfile.js +67 -0
- package/dist/src/images/dockerfile.js.map +1 -0
- package/dist/src/images/gc.js +261 -0
- package/dist/src/images/gc.js.map +1 -0
- package/dist/src/images/index.js +10 -0
- package/dist/src/images/index.js.map +1 -0
- package/dist/src/images/manifest.js +109 -0
- package/dist/src/images/manifest.js.map +1 -0
- package/dist/src/images/ownership.js +5 -0
- package/dist/src/images/ownership.js.map +1 -0
- package/dist/src/images/records.js +32 -0
- package/dist/src/images/records.js.map +1 -0
- package/dist/src/images/registry.js +154 -0
- package/dist/src/images/registry.js.map +1 -0
- package/dist/src/images/service.js +224 -0
- package/dist/src/images/service.js.map +1 -0
- package/dist/src/model-access/capture.js +255 -0
- package/dist/src/model-access/capture.js.map +1 -0
- package/dist/src/model-access/index.js +5 -0
- package/dist/src/model-access/index.js.map +1 -0
- package/dist/src/model-access/policy.js +49 -0
- package/dist/src/model-access/policy.js.map +1 -0
- package/dist/src/model-access/proxy.js +336 -0
- package/dist/src/model-access/proxy.js.map +1 -0
- package/dist/src/model-access/records.js +167 -0
- package/dist/src/model-access/records.js.map +1 -0
- package/dist/src/runs/adapter-process.js +12 -0
- package/dist/src/runs/adapter-process.js.map +1 -0
- package/dist/src/runs/bundle.js +450 -0
- package/dist/src/runs/bundle.js.map +1 -0
- package/dist/src/runs/events.js +12 -1
- package/dist/src/runs/events.js.map +1 -1
- package/dist/src/runs/executor.js +46 -45
- package/dist/src/runs/executor.js.map +1 -1
- package/dist/src/runs/finalizer.js +17 -0
- package/dist/src/runs/finalizer.js.map +1 -1
- package/dist/src/runs/index.js +8 -1
- package/dist/src/runs/index.js.map +1 -1
- package/dist/src/runs/manifest.js +5 -5
- package/dist/src/runs/manifest.js.map +1 -1
- package/dist/src/runs/phase-bundle.js +86 -0
- package/dist/src/runs/phase-bundle.js.map +1 -0
- package/dist/src/runs/phase-cancellation.js +107 -0
- package/dist/src/runs/phase-cancellation.js.map +1 -0
- package/dist/src/runs/phase-group.js +113 -0
- package/dist/src/runs/phase-group.js.map +1 -0
- package/dist/src/runs/query.js +1 -1
- package/dist/src/runs/query.js.map +1 -1
- package/dist/src/runs/records.js +3 -0
- package/dist/src/runs/records.js.map +1 -1
- package/dist/src/runs/request.js +10 -3
- package/dist/src/runs/request.js.map +1 -1
- package/dist/src/runs/training-candidate.js +161 -0
- package/dist/src/runs/training-candidate.js.map +1 -0
- package/dist/src/runs/verifier-evidence-redaction.js +61 -0
- package/dist/src/runs/verifier-evidence-redaction.js.map +1 -0
- package/dist/src/runs/verifier-evidence.js +459 -0
- package/dist/src/runs/verifier-evidence.js.map +1 -0
- package/dist/src/runs/verifier-structured-evidence.js +83 -0
- package/dist/src/runs/verifier-structured-evidence.js.map +1 -0
- package/dist/src/trajectories/analysis.js +288 -0
- package/dist/src/trajectories/analysis.js.map +1 -0
- package/dist/src/trajectories/chunk-projection.js +148 -0
- package/dist/src/trajectories/chunk-projection.js.map +1 -0
- package/dist/src/trajectories/content-projection.js +389 -0
- package/dist/src/trajectories/content-projection.js.map +1 -0
- package/dist/src/trajectories/dsh-chunk-contract.js +176 -0
- package/dist/src/trajectories/dsh-chunk-contract.js.map +1 -0
- package/dist/src/trajectories/dsh-contract.js +284 -0
- package/dist/src/trajectories/dsh-contract.js.map +1 -0
- package/dist/src/trajectories/events-chunk-drill.js +95 -0
- package/dist/src/trajectories/events-chunk-drill.js.map +1 -0
- package/dist/src/trajectories/events-page.js +367 -0
- package/dist/src/trajectories/events-page.js.map +1 -0
- package/dist/src/trajectories/index.js +3 -0
- package/dist/src/trajectories/index.js.map +1 -1
- package/dist/src/trajectories/provider-capture.js +15 -24
- package/dist/src/trajectories/provider-capture.js.map +1 -1
- package/dist/src/trajectories/providers/deepseek.js +3 -3
- package/dist/src/trajectories/providers/deepseek.js.map +1 -1
- package/dist/src/trajectories/request-attempt.js +47 -0
- package/dist/src/trajectories/request-attempt.js.map +1 -0
- package/dist/src/trajectories/stream-reader.js +311 -0
- package/dist/src/trajectories/stream-reader.js.map +1 -0
- package/dist/src/trajectories/surface-fold.js +126 -0
- package/dist/src/trajectories/surface-fold.js.map +1 -0
- package/dist/src/workers/index.js +3 -0
- package/dist/src/workers/index.js.map +1 -0
- package/dist/src/workers/remote-harbor-work-spec.js +97 -0
- package/dist/src/workers/remote-harbor-work-spec.js.map +1 -0
- package/dist/src/workers/remote-harbor-worker.js +223 -0
- package/dist/src/workers/remote-harbor-worker.js.map +1 -0
- package/docs/schemas/benchmark-hook-request-v1.schema.json +65 -0
- package/docs/schemas/benchmark-hook-response-v1.schema.json +75 -0
- package/docs/schemas/benchmark-hook-v1.schema.json +33 -0
- package/docs/schemas/benchmark-lock-v1.schema.json +351 -0
- package/docs/schemas/benchmark-metric-v1.schema.json +37 -0
- package/docs/schemas/benchmark-package-v1.schema.json +167 -0
- package/docs/schemas/benchmark-phase-group.schema.json +41 -0
- package/docs/schemas/benchmark-profile-v1.schema.json +141 -0
- package/docs/schemas/benchmark-task-v1.schema.json +384 -0
- package/docs/schemas/benchmark-tool-result-v1.schema.json +30 -0
- package/docs/schemas/environment-build-index.schema.json +13 -0
- package/docs/schemas/environment-build-record.schema.json +25 -0
- package/docs/schemas/environment-image-manifest.schema.json +62 -0
- package/docs/schemas/error.schema.json +22 -0
- package/docs/schemas/eval-control.schema.json +36 -0
- package/docs/schemas/eval-execution-plan.schema.json +216 -0
- package/docs/schemas/eval-progress.schema.json +1 -15
- package/docs/schemas/eval-rerun-result.schema.json +58 -0
- package/docs/schemas/eval-rerun-state.schema.json +61 -0
- package/docs/schemas/eval-rerun-submission.schema.json +60 -0
- package/docs/schemas/eval-result.schema.json +32 -13
- package/docs/schemas/eval-submission-input.schema.json +14 -0
- package/docs/schemas/eval-submission.schema.json +68 -0
- package/docs/schemas/eval-trial-publication.schema.json +15 -0
- package/docs/schemas/eval-trial-reference.schema.json +129 -0
- package/docs/schemas/execution-evidence.schema.json +96 -0
- package/docs/schemas/execution-lease.schema.json +50 -0
- package/docs/schemas/execution-provider-status.schema.json +65 -0
- package/docs/schemas/execution-worker.schema.json +53 -0
- package/docs/schemas/harbor-process-exit.schema.json +14 -0
- package/docs/schemas/interaction-capture-ref.schema.json +45 -0
- package/docs/schemas/local-provider-execution.schema.json +33 -0
- package/docs/schemas/model-interaction.schema.json +41 -0
- package/docs/schemas/regrade-assessment-reference.schema.json +11 -0
- package/docs/schemas/remote-result-envelope.schema.json +32 -0
- package/docs/schemas/remote-tree-envelope.schema.json +25 -0
- package/docs/schemas/remote-work-offer.schema.json +58 -0
- package/docs/schemas/remote-work-receipt.schema.json +50 -0
- package/docs/schemas/remote-worker-artifact.schema.json +17 -0
- package/docs/schemas/remote-worker-event.schema.json +18 -0
- package/docs/schemas/remote-worker-heartbeat.schema.json +37 -0
- package/docs/schemas/remote-worker-registration.schema.json +52 -0
- package/docs/schemas/result-bundle-index.schema.json +133 -0
- package/docs/schemas/run-context.schema.json +15 -0
- package/docs/schemas/run-environment-images.schema.json +39 -0
- package/docs/schemas/run-manifest.schema.json +4 -0
- package/docs/schemas/run-request.schema.json +4 -0
- package/docs/schemas/training-data-candidate.schema.json +46 -0
- package/docs/schemas/trajectory-analysis.schema.json +258 -0
- package/docs/schemas/trajectory-events-page.schema.json +40 -0
- package/docs/schemas/verifier-assessment.schema.json +77 -0
- package/docs/schemas/verifier-evidence.schema.json +276 -0
- package/integrations/harbor/hitch_benchmark.py +252 -0
- package/integrations/harbor/hitch_candidate_recycle.py +262 -0
- package/integrations/harbor/hitch_harbor_agent.py +698 -568
- package/integrations/harbor/hitch_harbor_environment.py +469 -0
- package/integrations/harbor/hitch_harbor_task_resources.py +275 -0
- package/integrations/harbor/hitch_harbor_verifier.py +83 -2
- package/integrations/harbor/hitch_phase_supervisor.py +452 -0
- package/integrations/harbor/hitch_tool_client.mjs +91 -0
- package/integrations/model-call/cli.js +57 -0
- package/package.json +30 -6
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { lstat, readFile } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
+
import { redactCredentialText } from "../foundation/index.js";
|
|
4
|
+
import { parseFailureClassification } from "./failure-classifier.js";
|
|
3
5
|
const HITCH_BRIDGE_ERROR_MAX_BYTES = 64 * 1024;
|
|
4
6
|
const HITCH_BRIDGE_ERROR_MESSAGE_MAX_BYTES = 2048;
|
|
5
7
|
const HITCH_BRIDGE_ERROR_CODES = new Set([
|
|
@@ -17,7 +19,7 @@ const HITCH_BRIDGE_ERROR_CODES = new Set([
|
|
|
17
19
|
"hitch_result_artifact_copy_failed",
|
|
18
20
|
]);
|
|
19
21
|
/** Read a Harbor bridge diagnostic as untrusted, bounded evidence. */
|
|
20
|
-
export async function readHarborBridgeError(trialDirectory) {
|
|
22
|
+
export async function readHarborBridgeError(trialDirectory, credentialValues = []) {
|
|
21
23
|
const source = path.join(trialDirectory, "agent", "hitch-bridge-error.json");
|
|
22
24
|
try {
|
|
23
25
|
const info = await lstat(source);
|
|
@@ -27,7 +29,7 @@ export async function readHarborBridgeError(trialDirectory) {
|
|
|
27
29
|
const value = JSON.parse(raw);
|
|
28
30
|
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
29
31
|
return null;
|
|
30
|
-
const diagnostic = value;
|
|
32
|
+
const diagnostic = sanitizeEvidence(value, credentialValues);
|
|
31
33
|
if (diagnostic.schema_version !== "1")
|
|
32
34
|
return null;
|
|
33
35
|
if (typeof diagnostic.code !== "string" || !HITCH_BRIDGE_ERROR_CODES.has(diagnostic.code))
|
|
@@ -36,12 +38,65 @@ export async function readHarborBridgeError(trialDirectory) {
|
|
|
36
38
|
return null;
|
|
37
39
|
if (Buffer.byteLength(diagnostic.message, "utf8") > HITCH_BRIDGE_ERROR_MESSAGE_MAX_BYTES)
|
|
38
40
|
return null;
|
|
39
|
-
|
|
41
|
+
const failureClassification = bridgeFailureClassification(diagnostic);
|
|
42
|
+
return {
|
|
43
|
+
code: diagnostic.code, message: bridgeErrorMessage(diagnostic), raw: `${JSON.stringify(diagnostic)}\n`,
|
|
44
|
+
...(failureClassification ? { failureClassification } : {}),
|
|
45
|
+
};
|
|
40
46
|
}
|
|
41
47
|
catch {
|
|
42
48
|
return null;
|
|
43
49
|
}
|
|
44
50
|
}
|
|
51
|
+
function bridgeFailureClassification(diagnostic) {
|
|
52
|
+
if (diagnostic.failure_classification !== undefined) {
|
|
53
|
+
try {
|
|
54
|
+
return parseFailureClassification(diagnostic.failure_classification);
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return undefined;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (diagnostic.code !== "hitch_process_failed")
|
|
61
|
+
return undefined;
|
|
62
|
+
const text = JSON.stringify(diagnostic);
|
|
63
|
+
if (/(?:insufficient|exhausted|depleted).{0,40}(?:balance|quota|credit)|(?:balance|quota|credit).{0,40}(?:insufficient|exhausted|depleted)|billing.{0,30}(?:limit|hard)/i.test(text)) {
|
|
64
|
+
return providerClassification("provider_quota_exhausted", "never", true);
|
|
65
|
+
}
|
|
66
|
+
if (/(?:invalid|missing|expired|revoked).{0,30}(?:api[_ -]?key|credential|token)|(?:authentication|authorization)\s+(?:failed|required)|\bunauthorized\b|(?:status|http)\s*401\b/i.test(text)) {
|
|
67
|
+
return providerClassification("provider_auth_failed", "operator-required", "unknown");
|
|
68
|
+
}
|
|
69
|
+
if (/(?:model|parameter|request).{0,40}(?:not found|does not exist|unsupported|invalid)|(?:unknown|invalid)\s+model/i.test(text)) {
|
|
70
|
+
return providerClassification("provider_configuration_invalid", "never", "unknown");
|
|
71
|
+
}
|
|
72
|
+
if (/\brate[ _-]?limit(?:ed|ing)?\b|too many requests|(?:status|http)\s*429\b/i.test(text)) {
|
|
73
|
+
return providerClassification("provider_rate_limited", "transient", true);
|
|
74
|
+
}
|
|
75
|
+
if (/\b(?:econnreset|etimedout|eai_again|socket hang up|connection reset|temporary network|network unreachable)\b/i.test(text)) {
|
|
76
|
+
return providerClassification("provider_transport_transient", "transient", true);
|
|
77
|
+
}
|
|
78
|
+
if (/agent run timed out|"status":"timed_out"|"code":"timed_out"/i.test(text)) {
|
|
79
|
+
return { schema_version: "1", phase: "agent", code: "agent_timed_out", candidate_started: true, retryability: "never", source: "bridge-evidence" };
|
|
80
|
+
}
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
function providerClassification(code, retryability, candidateStarted) {
|
|
84
|
+
return { schema_version: "1", phase: "provider", code, candidate_started: candidateStarted, retryability, source: "bridge-evidence" };
|
|
85
|
+
}
|
|
86
|
+
function sanitizeEvidence(value, credentialValues, key = "") {
|
|
87
|
+
if (typeof value === "string") {
|
|
88
|
+
if (/(?:authorization|cookie|api[-_]?key|token|secret|password|credential)/i.test(key))
|
|
89
|
+
return "[REDACTED]";
|
|
90
|
+
return redactCredentialText(value, credentialValues).text;
|
|
91
|
+
}
|
|
92
|
+
if (Array.isArray(value))
|
|
93
|
+
return value.map((entry) => sanitizeEvidence(entry, credentialValues));
|
|
94
|
+
if (value && typeof value === "object") {
|
|
95
|
+
return Object.fromEntries(Object.entries(value)
|
|
96
|
+
.map(([name, entry]) => [name, sanitizeEvidence(entry, credentialValues, name)]));
|
|
97
|
+
}
|
|
98
|
+
return value;
|
|
99
|
+
}
|
|
45
100
|
function bridgeErrorMessage(diagnostic) {
|
|
46
101
|
const message = diagnostic.message;
|
|
47
102
|
if (!message.includes("no diagnostic output"))
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"harbor-bridge-error.js","sourceRoot":"","sources":["../../../src/evals/harbor-bridge-error.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,IAAI,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"harbor-bridge-error.js","sourceRoot":"","sources":["../../../src/evals/harbor-bridge-error.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAE9D,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAErE,MAAM,4BAA4B,GAAG,EAAE,GAAG,IAAI,CAAC;AAC/C,MAAM,oCAAoC,GAAG,IAAI,CAAC;AAClD,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC;IACvC,uBAAuB;IACvB,sBAAsB;IACtB,sBAAsB;IACtB,uBAAuB;IACvB,0BAA0B;IAC1B,oBAAoB;IACpB,2BAA2B;IAC3B,6BAA6B;IAC7B,8BAA8B;IAC9B,kCAAkC;IAClC,gCAAgC;IAChC,mCAAmC;CACpC,CAAC,CAAC;AASH,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,cAAsB,EAAE,mBAAsC,EAAE;IAC1G,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,yBAAyB,CAAC,CAAC;IAC7E,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,4BAA4B;YAAE,OAAO,IAAI,CAAC;QAC9F,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAY,CAAC;QACzC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC7E,MAAM,UAAU,GAAG,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,CAA4B,CAAC;QACxF,IAAI,UAAU,CAAC,cAAc,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC;QACnD,IAAI,OAAO,UAAU,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACvG,IAAI,OAAO,UAAU,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC;QACtF,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,oCAAoC;YAAE,OAAO,IAAI,CAAC;QACtG,MAAM,qBAAqB,GAAG,2BAA2B,CAAC,UAAU,CAAC,CAAC;QACtE,OAAO;YACL,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,OAAO,EAAE,kBAAkB,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI;YACtG,GAAG,CAAC,qBAAqB,CAAC,CAAC,CAAC,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC5D,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CAAC,UAAmC;IACtE,IAAI,UAAU,CAAC,sBAAsB,KAAK,SAAS,EAAE,CAAC;QACpD,IAAI,CAAC;YAAC,OAAO,0BAA0B,CAAC,UAAU,CAAC,sBAAsB,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,SAAS,CAAC;QAAC,CAAC;IAC3G,CAAC;IACD,IAAI,UAAU,CAAC,IAAI,KAAK,sBAAsB;QAAE,OAAO,SAAS,CAAC;IACjE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IACxC,IAAI,qKAAqK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACrL,OAAO,sBAAsB,CAAC,0BAA0B,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,8KAA8K,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9L,OAAO,sBAAsB,CAAC,sBAAsB,EAAE,mBAAmB,EAAE,SAAS,CAAC,CAAC;IACxF,CAAC;IACD,IAAI,iHAAiH,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjI,OAAO,sBAAsB,CAAC,gCAAgC,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtF,CAAC;IACD,IAAI,2EAA2E,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3F,OAAO,sBAAsB,CAAC,uBAAuB,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,+GAA+G,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/H,OAAO,sBAAsB,CAAC,8BAA8B,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;IACnF,CAAC;IACD,IAAI,8DAA8D,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9E,OAAO,EAAE,cAAc,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;IACrJ,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,sBAAsB,CAC7B,IAAY,EACZ,YAAqD,EACrD,gBAA8D;IAE9D,OAAO,EAAE,cAAc,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;AACxI,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAE,gBAAmC,EAAE,GAAG,GAAG,EAAE;IACrF,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,IAAI,wEAAwE,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,YAAY,CAAC;QAC5G,OAAO,oBAAoB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,IAAI,CAAC;IAC5D,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC;IACjG,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC;aACvE,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACtF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,kBAAkB,CAAC,UAAmC;IAC7D,MAAM,OAAO,GAAG,UAAU,CAAC,OAAiB,CAAC;IAC7C,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAC;QAAE,OAAO,OAAO,CAAC;IAC9D,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;IACnC,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,OAAO,CAAC;IACtF,MAAM,QAAQ,GAAG,OAAkC,CAAC;IACpD,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC;SACtD,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAuB,CAAC;IACpF,IAAI,CAAC,IAAI;QAAE,OAAO,OAAO,CAAC;IAC1B,OAAO,eAAe,CACpB,OAAO,CAAC,OAAO,CAAC,uBAAuB,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,EACrD,oCAAoC,CACrC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,KAAa,EAAE,QAAgB;IACtD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC3C,IAAI,OAAO,CAAC,UAAU,IAAI,QAAQ;QAAE,OAAO,KAAK,CAAC;IACjD,OAAO,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,GAAG,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;AAClG,CAAC"}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
2
|
+
import { chmod, cp, lstat, mkdtemp, rename, rm } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { artifactDirectoryIntegrity } from "../artifacts/index.js";
|
|
5
|
+
import { HARBOR_NODE_VERSION_WITH_PREFIX } from "../backends/index.js";
|
|
6
|
+
import { HitchError, atomicWriteJSON, digest, ensureDir, fingerprintExecutable, readJSON, runCommand, statePaths, withFileLock } from "../foundation/index.js";
|
|
7
|
+
export const HARBOR_NODE_RUNTIME_RECIPE = "1";
|
|
8
|
+
export const HARBOR_NODE_RUNTIME_DIRECTORY = ".hitch-node-runtime";
|
|
9
|
+
const ARCHIVE = "node-runtime.tar.gz";
|
|
10
|
+
const MANIFEST = "node-runtime.json";
|
|
11
|
+
const MAX_ARCHIVE_BYTES = 128 * 1024 * 1024;
|
|
12
|
+
/** Export once from the pinned Linux builder; trial setup never downloads Node. */
|
|
13
|
+
export async function prepareHarborNodeRuntime(input) {
|
|
14
|
+
const base = path.join(statePaths(input.root).store, "harbor-artifacts", "node-runtimes");
|
|
15
|
+
const [artifacts, locks, temporary, refs, invalid] = await Promise.all(["artifacts", "locks", "tmp", "refs", "invalid"].map((name) => ensureDir(path.join(base, name))));
|
|
16
|
+
const identity = {
|
|
17
|
+
schema_version: "1",
|
|
18
|
+
recipe_version: HARBOR_NODE_RUNTIME_RECIPE,
|
|
19
|
+
node_version: HARBOR_NODE_VERSION_WITH_PREFIX,
|
|
20
|
+
platform: input.builder.artifactPlatform,
|
|
21
|
+
libc: "glibc",
|
|
22
|
+
builder_image_id: input.builder.id,
|
|
23
|
+
};
|
|
24
|
+
const key = digest(identity).slice("sha256:".length);
|
|
25
|
+
const ref = path.join(refs, `${key}.json`);
|
|
26
|
+
return withFileLock(locks, key, async () => {
|
|
27
|
+
const previous = await readJSON(ref, null).catch(() => null);
|
|
28
|
+
if (typeof previous?.runtime_id === "string" && /^sha256:[a-f0-9]{64}$/.test(previous.runtime_id)) {
|
|
29
|
+
const directory = path.join(artifacts, previous.runtime_id.slice("sha256:".length));
|
|
30
|
+
try {
|
|
31
|
+
const manifest = await verifyHarborNodeRuntime(directory, identity);
|
|
32
|
+
if (manifest.runtime_id !== previous.runtime_id)
|
|
33
|
+
throw runtimeError("cache reference identity mismatch");
|
|
34
|
+
return { directory, manifest };
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
if (await lstat(directory).catch(() => null)) {
|
|
38
|
+
await rename(directory, path.join(invalid, `${path.basename(directory)}-${randomBytes(6).toString("hex")}`));
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
const staging = await mkdtemp(path.join(temporary, "export-"));
|
|
43
|
+
const container = `hitch-node-runtime-${process.pid}-${randomBytes(6).toString("hex")}`;
|
|
44
|
+
const options = {
|
|
45
|
+
env: input.env, ...(input.signal ? { signal: input.signal } : {}),
|
|
46
|
+
timeoutMs: 180_000, failureCode: "harbor_node_runtime_prepare_failed", failureExitCode: 12,
|
|
47
|
+
};
|
|
48
|
+
try {
|
|
49
|
+
await runCommand(input.docker, [
|
|
50
|
+
"run", "--name", container, "--network", "none", "--platform", input.builder.dockerPlatform,
|
|
51
|
+
// Never resolve a mutable tag after the builder's image inspection.
|
|
52
|
+
input.builder.id, "sh", "-ceu", [
|
|
53
|
+
`test "$(node -p process.version)" = "${identity.node_version}"`,
|
|
54
|
+
`test "$(node -p 'process.platform + \"-\" + process.arch')" = "${identity.platform}"`,
|
|
55
|
+
"getconf GNU_LIBC_VERSION",
|
|
56
|
+
"test -x /usr/local/bin/node",
|
|
57
|
+
// Keep npm/npx available as with the former nvm install. Do not ship
|
|
58
|
+
// the builder's pnpm, headers, or system libraries into task images.
|
|
59
|
+
`tar --sort=name --mtime=@0 --owner=0 --group=0 --numeric-owner -czf /tmp/${ARCHIVE} -C /usr/local bin/node bin/npm bin/npx lib/node_modules/npm`,
|
|
60
|
+
].join("\n"),
|
|
61
|
+
], options);
|
|
62
|
+
await runCommand(input.docker, ["cp", `${container}:/tmp/${ARCHIVE}`, path.join(staging, ARCHIVE)], options);
|
|
63
|
+
const archive = path.join(staging, ARCHIVE);
|
|
64
|
+
const info = await lstat(archive);
|
|
65
|
+
if (!info.isFile() || info.size <= 0 || info.size > MAX_ARCHIVE_BYTES)
|
|
66
|
+
throw runtimeError("exported archive is invalid");
|
|
67
|
+
const payload = { ...identity, archive_sha256: await fingerprintExecutable(archive), archive_bytes: info.size };
|
|
68
|
+
const manifest = { ...payload, runtime_id: digest(payload) };
|
|
69
|
+
await atomicWriteJSON(path.join(staging, MANIFEST), manifest);
|
|
70
|
+
await verifyHarborNodeRuntime(staging, identity);
|
|
71
|
+
const destination = path.join(artifacts, manifest.runtime_id.slice("sha256:".length));
|
|
72
|
+
if (await lstat(destination).catch(() => null)) {
|
|
73
|
+
try {
|
|
74
|
+
await verifyHarborNodeRuntime(destination, manifest);
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
await rename(destination, path.join(invalid, `${path.basename(destination)}-${randomBytes(6).toString("hex")}`));
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
if (!await lstat(destination).catch(() => null))
|
|
81
|
+
await rename(staging, destination);
|
|
82
|
+
await atomicWriteJSON(ref, { runtime_id: manifest.runtime_id });
|
|
83
|
+
return { directory: destination, manifest };
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
await runCommand(input.docker, ["rm", "--force", container], { env: input.env, timeoutMs: 30_000 }).catch(() => { });
|
|
87
|
+
await rm(staging, { recursive: true, force: true });
|
|
88
|
+
}
|
|
89
|
+
}, {
|
|
90
|
+
timeoutCode: "harbor_node_runtime_locked", timeoutExitCode: 12,
|
|
91
|
+
...(input.signal ? { signal: input.signal } : {}),
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
export async function verifyHarborNodeRuntime(directory, expected) {
|
|
95
|
+
const info = await lstat(directory);
|
|
96
|
+
const manifestInfo = await lstat(path.join(directory, MANIFEST));
|
|
97
|
+
if (!info.isDirectory() || info.isSymbolicLink() || !manifestInfo.isFile() || manifestInfo.size > 16_384) {
|
|
98
|
+
throw runtimeError("cache entry is not a regular runtime bundle");
|
|
99
|
+
}
|
|
100
|
+
const manifest = await readJSON(path.join(directory, MANIFEST));
|
|
101
|
+
const { runtime_id, ...payload } = manifest;
|
|
102
|
+
const archive = path.join(directory, ARCHIVE);
|
|
103
|
+
const archiveInfo = await lstat(archive);
|
|
104
|
+
if (Object.keys(manifest).sort().join(",") !== "archive_bytes,archive_sha256,builder_image_id,libc,node_version,platform,recipe_version,runtime_id,schema_version"
|
|
105
|
+
|| manifest.schema_version !== "1" || manifest.recipe_version !== HARBOR_NODE_RUNTIME_RECIPE
|
|
106
|
+
|| manifest.node_version !== HARBOR_NODE_VERSION_WITH_PREFIX || manifest.libc !== "glibc"
|
|
107
|
+
|| !/^linux-(x64|arm64)$/.test(manifest.platform)
|
|
108
|
+
|| !/^sha256:[a-f0-9]{64}$/.test(manifest.builder_image_id)
|
|
109
|
+
|| !archiveInfo.isFile() || archiveInfo.size <= 0 || archiveInfo.size > MAX_ARCHIVE_BYTES
|
|
110
|
+
|| archiveInfo.size !== manifest.archive_bytes || await fingerprintExecutable(archive) !== manifest.archive_sha256
|
|
111
|
+
|| digest(payload) !== runtime_id
|
|
112
|
+
|| Object.entries(expected).some(([key, value]) => manifest[key] !== value)) {
|
|
113
|
+
throw runtimeError("cache identity or archive checksum mismatch");
|
|
114
|
+
}
|
|
115
|
+
return manifest;
|
|
116
|
+
}
|
|
117
|
+
/** Compose in staging, then seal a NEW identity; never mutate a published artifact. */
|
|
118
|
+
export async function attachHarborNodeRuntime(directory, manifest, runtime) {
|
|
119
|
+
const destination = path.join(directory, HARBOR_NODE_RUNTIME_DIRECTORY);
|
|
120
|
+
if (await lstat(destination).catch(() => null))
|
|
121
|
+
throw runtimeError("harness uses the reserved offline runtime directory");
|
|
122
|
+
await cp(runtime.directory, destination, { recursive: true, errorOnExist: true, force: false });
|
|
123
|
+
await verifyHarborNodeRuntime(destination, runtime.manifest);
|
|
124
|
+
// mkdtemp cache roots are private. This public runtime payload must remain
|
|
125
|
+
// readable when a task runs Hitch as a non-root user; seal these modes too.
|
|
126
|
+
await chmod(destination, 0o755);
|
|
127
|
+
await chmod(path.join(destination, MANIFEST), 0o644);
|
|
128
|
+
await chmod(path.join(destination, ARCHIVE), 0o644);
|
|
129
|
+
const integrity = await artifactDirectoryIntegrity(directory);
|
|
130
|
+
const artifactId = digest({
|
|
131
|
+
kind: "harbor-offline-node-artifact-v1", harness_artifact_id: manifest.artifact_id,
|
|
132
|
+
node_runtime_id: runtime.manifest.runtime_id, artifact_integrity: integrity,
|
|
133
|
+
});
|
|
134
|
+
const composed = {
|
|
135
|
+
...manifest, artifact_id: artifactId, artifact_integrity: integrity,
|
|
136
|
+
toolchain: { ...manifest.toolchain, node_runtime: runtime.manifest.runtime_id },
|
|
137
|
+
};
|
|
138
|
+
await atomicWriteJSON(path.join(directory, "artifact.json"), composed);
|
|
139
|
+
const renamed = path.join(path.dirname(directory), artifactId.slice("sha256:".length));
|
|
140
|
+
await rename(directory, renamed);
|
|
141
|
+
return { directory: renamed, manifest: composed };
|
|
142
|
+
}
|
|
143
|
+
function runtimeError(message) {
|
|
144
|
+
return new HitchError(`Harbor offline Node runtime: ${message}`, { code: "harbor_node_runtime_invalid", exitCode: 12 });
|
|
145
|
+
}
|
|
146
|
+
//# sourceMappingURL=harbor-node-runtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"harbor-node-runtime.js","sourceRoot":"","sources":["../../../src/evals/harbor-node-runtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACzE,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAC;AAEnE,OAAO,EAAE,+BAA+B,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,EAAE,SAAS,EAAE,qBAAqB,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAG/J,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC;AAC9C,MAAM,CAAC,MAAM,6BAA6B,GAAG,qBAAqB,CAAC;AACnE,MAAM,OAAO,GAAG,qBAAqB,CAAC;AACtC,MAAM,QAAQ,GAAG,mBAAmB,CAAC;AACrC,MAAM,iBAAiB,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAmB5C,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,KAM9C;IACC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,kBAAkB,EAAE,eAAe,CAAC,CAAC;IAC1F,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CACpE,CAAC,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CACrD,CAAC;IAC9C,MAAM,QAAQ,GAAG;QACf,cAAc,EAAE,GAAY;QAC5B,cAAc,EAAE,0BAA0B;QAC1C,YAAY,EAAE,+BAA+B;QAC7C,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,gBAAgB;QACxC,IAAI,EAAE,OAAgB;QACtB,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE;KACnC,CAAC;IACF,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACrD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;IAC3C,OAAO,YAAY,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,IAAI,EAAE;QACzC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAkC,GAAG,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC9F,IAAI,OAAO,QAAQ,EAAE,UAAU,KAAK,QAAQ,IAAI,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAClG,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YACpF,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,uBAAuB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;gBACpE,IAAI,QAAQ,CAAC,UAAU,KAAK,QAAQ,CAAC,UAAU;oBAAE,MAAM,YAAY,CAAC,mCAAmC,CAAC,CAAC;gBACzG,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC7C,MAAM,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC/G,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;QAC/D,MAAM,SAAS,GAAG,sBAAsB,OAAO,CAAC,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACxF,MAAM,OAAO,GAAG;YACd,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,oCAAoC,EAAE,eAAe,EAAE,EAAE;SAC3F,CAAC;QACF,IAAI,CAAC;YACH,MAAM,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE;gBAC7B,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,cAAc;gBAC3F,oEAAoE;gBACpE,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;oBAC9B,wCAAwC,QAAQ,CAAC,YAAY,GAAG;oBAChE,kEAAkE,QAAQ,CAAC,QAAQ,GAAG;oBACtF,0BAA0B;oBAC1B,6BAA6B;oBAC7B,qEAAqE;oBACrE,qEAAqE;oBACrE,4EAA4E,OAAO,8DAA8D;iBAClJ,CAAC,IAAI,CAAC,IAAI,CAAC;aACb,EAAE,OAAO,CAAC,CAAC;YACZ,MAAM,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,GAAG,SAAS,SAAS,OAAO,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAC7G,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,iBAAiB;gBAAE,MAAM,YAAY,CAAC,6BAA6B,CAAC,CAAC;YACzH,MAAM,OAAO,GAAG,EAAE,GAAG,QAAQ,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC,OAAO,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;YAChH,MAAM,QAAQ,GAA8B,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACxF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YAC9D,MAAM,uBAAuB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YACjD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YACtF,IAAI,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/C,IAAI,CAAC;oBAAC,MAAM,uBAAuB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;gBAAC,CAAC;gBAC7D,MAAM,CAAC;oBAAC,MAAM,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBAAC,CAAC;YAC7H,CAAC;YACD,IAAI,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;gBAAE,MAAM,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YACpF,MAAM,eAAe,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAChE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;QAC9C,CAAC;gBAAS,CAAC;YACT,MAAM,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACpH,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC,EAAE;QACD,WAAW,EAAE,4BAA4B,EAAE,eAAe,EAAE,EAAE;QAC9D,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,SAAiB,EACjB,QAA4C;IAE5C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;IACpC,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;IACjE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,YAAY,CAAC,IAAI,GAAG,MAAM,EAAE,CAAC;QACzG,MAAM,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACpE,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAA4B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC3F,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,EAAE,GAAG,QAAQ,CAAC;IAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,mHAAmH;WAC7J,QAAQ,CAAC,cAAc,KAAK,GAAG,IAAI,QAAQ,CAAC,cAAc,KAAK,0BAA0B;WACzF,QAAQ,CAAC,YAAY,KAAK,+BAA+B,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO;WACtF,CAAC,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;WAC9C,CAAC,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC;WACxD,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,WAAW,CAAC,IAAI,IAAI,CAAC,IAAI,WAAW,CAAC,IAAI,GAAG,iBAAiB;WACtF,WAAW,CAAC,IAAI,KAAK,QAAQ,CAAC,aAAa,IAAI,MAAM,qBAAqB,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,cAAc;WAC/G,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU;WAC9B,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAsC,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;QACjH,MAAM,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,uFAAuF;AACvF,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,SAAiB,EAAE,QAA0B,EAAE,OAAoB;IAEnE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,6BAA6B,CAAC,CAAC;IACxE,IAAI,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;QAAE,MAAM,YAAY,CAAC,qDAAqD,CAAC,CAAC;IAC1H,MAAM,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IAChG,MAAM,uBAAuB,CAAC,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7D,2EAA2E;IAC3E,4EAA4E;IAC5E,MAAM,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC;IACrD,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC;IACpD,MAAM,SAAS,GAAG,MAAM,0BAA0B,CAAC,SAAS,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,MAAM,CAAC;QACxB,IAAI,EAAE,iCAAiC,EAAE,mBAAmB,EAAE,QAAQ,CAAC,WAAW;QAClF,eAAe,EAAE,OAAO,CAAC,QAAQ,CAAC,UAAU,EAAE,kBAAkB,EAAE,SAAS;KAC5E,CAAC,CAAC;IACH,MAAM,QAAQ,GAAqB;QACjC,GAAG,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,kBAAkB,EAAE,SAAS;QACnE,SAAS,EAAE,EAAE,GAAG,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,UAAU,EAAE;KAChF,CAAC;IACF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAC;IACvE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACvF,MAAM,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;AACpD,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,OAAO,IAAI,UAAU,CAAC,gCAAgC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,6BAA6B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;AAC1H,CAAC"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { lstat, readFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
const MAX_TIMING_BYTES = 16 * 1024;
|
|
4
|
+
const TRIAL_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,255}$/;
|
|
5
|
+
export async function harborPhaseTimingEvents(jobDirectory, rawResult) {
|
|
6
|
+
const trials = Array.isArray(rawResult?.trial_results) ? rawResult.trial_results : [];
|
|
7
|
+
const events = [];
|
|
8
|
+
for (const value of trials) {
|
|
9
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
10
|
+
continue;
|
|
11
|
+
const trial = value;
|
|
12
|
+
if (typeof trial.trial_name !== "string" || !TRIAL_NAME.test(trial.trial_name) || trial.trial_name === "." || trial.trial_name === "..")
|
|
13
|
+
continue;
|
|
14
|
+
const identity = { trial_id: trial.trial_name, ...(typeof trial.task_name === "string" ? { task_id: trial.task_name } : {}) };
|
|
15
|
+
for (const source of [
|
|
16
|
+
{ directory: "agent", phases: new Set(["setup", "agent"]) },
|
|
17
|
+
{ directory: "verifier", phases: new Set(["verifier"]) },
|
|
18
|
+
]) {
|
|
19
|
+
const file = path.join(jobDirectory, trial.trial_name, source.directory, "hitch-phase-timings.json");
|
|
20
|
+
try {
|
|
21
|
+
const info = await lstat(file);
|
|
22
|
+
if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || info.size > MAX_TIMING_BYTES)
|
|
23
|
+
throw new TypeError("unsafe timing evidence");
|
|
24
|
+
const record = parseTiming(JSON.parse(await readFile(file, "utf8")), source.phases);
|
|
25
|
+
for (const [phase, timing] of Object.entries(record))
|
|
26
|
+
events.push({ type: `eval.${phase}.completed`, ...identity, duration_ms: timing.duration_ms });
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
if (error.code === "ENOENT")
|
|
30
|
+
continue;
|
|
31
|
+
events.push({ type: "eval.phase-timing.invalid", ...identity, phase_source: source.directory, code: "phase_timing_invalid" });
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return events;
|
|
36
|
+
}
|
|
37
|
+
function parseTiming(value, allowed) {
|
|
38
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
39
|
+
throw new TypeError("timing evidence must be an object");
|
|
40
|
+
const record = value;
|
|
41
|
+
if (Object.keys(record).some((key) => key !== "schema_version" && key !== "phases") || record.schema_version !== "1"
|
|
42
|
+
|| !record.phases || typeof record.phases !== "object" || Array.isArray(record.phases))
|
|
43
|
+
throw new TypeError("timing evidence envelope is invalid");
|
|
44
|
+
const phases = record.phases;
|
|
45
|
+
if (Object.keys(phases).some((phase) => !allowed.has(phase)))
|
|
46
|
+
throw new TypeError("timing evidence phase is invalid");
|
|
47
|
+
const result = {};
|
|
48
|
+
for (const [phase, value] of Object.entries(phases)) {
|
|
49
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
50
|
+
throw new TypeError("phase timing is invalid");
|
|
51
|
+
const timing = value;
|
|
52
|
+
if (Object.keys(timing).some((key) => key !== "duration_ms" && key !== "completed_at")
|
|
53
|
+
|| !Number.isSafeInteger(timing.duration_ms) || timing.duration_ms < 0 || timing.duration_ms > 7 * 24 * 60 * 60 * 1_000
|
|
54
|
+
|| typeof timing.completed_at !== "string" || !Number.isFinite(Date.parse(timing.completed_at)))
|
|
55
|
+
throw new TypeError("phase timing fields are invalid");
|
|
56
|
+
result[phase] = { duration_ms: timing.duration_ms };
|
|
57
|
+
}
|
|
58
|
+
return result;
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=harbor-phase-timings.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"harbor-phase-timings.js","sourceRoot":"","sources":["../../../src/evals/harbor-phase-timings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,MAAM,gBAAgB,GAAG,EAAE,GAAG,IAAI,CAAC;AACnC,MAAM,UAAU,GAAG,oCAAoC,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,YAAoB,EAAE,SAAyC;IAC3G,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IACtF,MAAM,MAAM,GAA8B,EAAE,CAAC;IAC7C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,SAAS;QAC1E,MAAM,KAAK,GAAG,KAAgC,CAAC;QAC/C,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,GAAG,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI;YAAE,SAAS;QAClJ,MAAM,QAAQ,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;QAC9H,KAAK,MAAM,MAAM,IAAI;YACnB,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,EAAE;YAC3D,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,EAAE;SACzD,EAAE,CAAC;YACF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,SAAS,EAAE,0BAA0B,CAAC,CAAC;YACrG,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC/B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,gBAAgB;oBAAE,MAAM,IAAI,SAAS,CAAC,wBAAwB,CAAC,CAAC;gBAC/I,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBACpF,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,KAAK,YAAY,EAAE,GAAG,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;YACvJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;oBAAE,SAAS;gBACjE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,GAAG,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,IAAI,EAAE,sBAAsB,EAAE,CAAC,CAAC;YAChI,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,KAAc,EAAE,OAAoB;IACvD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAC;IAC1H,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,gBAAgB,IAAI,GAAG,KAAK,QAAQ,CAAC,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG;WAC/G,CAAC,MAAM,CAAC,MAAM,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IACrJ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAiC,CAAC;IACxD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IACtH,MAAM,MAAM,GAA4C,EAAE,CAAC;IAC3D,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,yBAAyB,CAAC,CAAC;QAChH,MAAM,MAAM,GAAG,KAAgC,CAAC;QAChD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,aAAa,IAAI,GAAG,KAAK,cAAc,CAAC;eACjF,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,IAAK,MAAM,CAAC,WAAsB,GAAG,CAAC,IAAK,MAAM,CAAC,WAAsB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK;eAC5I,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;QAC1J,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,WAAqB,EAAE,CAAC;IAChE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/src/evals/index.js
CHANGED
|
@@ -1,7 +1,45 @@
|
|
|
1
1
|
export { DEFAULT_EVAL_SETUP_TIMEOUT_MS, DEFAULT_EVAL_TIMEOUT_MS, DEFAULT_INFRASTRUCTURE_RETRIES, DEFAULT_INFRASTRUCTURE_RETRY_BACKOFF_MS, newEvalId, resolveBenchmarkReference, resolveLocalDatasetTaskIds, validateEvalId, validateEvalRequest } from "./request.js";
|
|
2
|
+
export { buildBenchmarkAdapterManifest, loadBenchmarkAdapterManifest, scoreWithinRange } from "./benchmark-adapter-manifest.js";
|
|
3
|
+
export { EvalEventSink, MAX_EVAL_EVENT_BYTES } from "./events.js";
|
|
4
|
+
export { buildEvalExecutionPlan, parseEvalExecutionPlan, DEFAULT_EVAL_TRIAL_RESOURCES } from "./execution-plan.js";
|
|
5
|
+
export { workSchedulingPriority } from "./planned-execution-support.js";
|
|
6
|
+
export { planTaskSchedulingHints, recordTaskDuration, schedulingHintsFromPlan } from "./duration-estimator.js";
|
|
7
|
+
export { classifyTrialFailure, parseFailureClassification, physicalRetryAllowed } from "./failure-classifier.js";
|
|
8
|
+
export { ensurePhysicalRetryDecision, ensureTerminalRetryDecision, parseEvalRetryState, readEvalRetryState, resolveRetryWork, transitionRetryDecision } from "./retry-state.js";
|
|
9
|
+
export { runtimeResourcesForTask } from "./execution-plan-resources.js";
|
|
10
|
+
export { deriveTaskResourceRequirement, resolveLocalTaskPlanningInputs, resolveLocalTaskResourceRequirements, HARBOR_EGRESS_SIDECAR_RESOURCES } from "./task-resources.js";
|
|
11
|
+
export { parseExecutionEvidence } from "./execution-evidence.js";
|
|
12
|
+
export { parseDockerMemoryBytes, startDockerResourceObserver } from "./docker-resource-observer.js";
|
|
13
|
+
export { parseDockerEngineContainerStats, readDockerEngineContainerStats } from "./docker-engine-stats.js";
|
|
14
|
+
export { DEFAULT_EXECUTION_LEASE_HEARTBEAT_MS, DEFAULT_EXECUTION_LEASE_TTL_MS, acceptExecutionLease, createExecutionLease, heartbeatExecutionLease, markExecutionLeaseLost, markExecutionLeaseRunning, parseExecutionLease, readExecutionLeases, recoverExecutionLeases, reissueExecutionLease, releaseExecutionLease } from "./execution-leases.js";
|
|
2
15
|
export { runEval } from "./service.js";
|
|
3
|
-
export {
|
|
16
|
+
export { harborArtifactDirectory, loadHarborArtifact, prepareHarborArtifact } from "./harbor-artifact-builder.js";
|
|
17
|
+
export { DEFAULT_HARBOR_ARTIFACT_BUILDER_BASE_IMAGE } from "./harbor-artifact-builder-image.js";
|
|
18
|
+
export { HARBOR_NODE_VERSION, HARBOR_NODE_VERSION_WITH_PREFIX, HARBOR_PNPM_VERSION } from "../backends/index.js";
|
|
19
|
+
export { startEvalModelCaptureRuntime } from "./model-capture-runtime.js";
|
|
20
|
+
export { recoverLocalDockerEvalLeases } from "./recovery.js";
|
|
21
|
+
export { inspectEval, isControlPlaneEval, listEvals } from "./records.js";
|
|
4
22
|
export { importEvalTrialRun, importEvalTrialRuns, validateEvalTrialReferences } from "./trial-import.js";
|
|
5
|
-
export {
|
|
23
|
+
export { captureVerifierDiagnostics, DEFAULT_VERIFIER_DIAGNOSTIC_MAX_BYTES, MAX_VERIFIER_DIAGNOSTIC_MAX_BYTES, persistTrialVerifierDiagnostics, VERIFIER_ARTIFACT_NAMES, VERIFIER_DIAGNOSTICS_INDEX_REF } from "./verifier-artifacts.js";
|
|
24
|
+
export { captureVerifierScoreEvidence } from "./verifier-score-artifacts.js";
|
|
25
|
+
export { MAX_VERIFIER_FEEDBACK_BYTES, MAX_VERIFIER_PROCESS_BYTES } from "../domain/index.js";
|
|
26
|
+
export { createEvalProgress, evalTrialKey, mergeEvalProgressTrial, parseEvalTrialRef, readEvalProgress, replaceInvalidEvalProgressTrial, writeEvalProgress } from "./progress.js";
|
|
6
27
|
export { rerunEval, selectRerunTasks, selectRerunTrialSlots } from "./rerun.js";
|
|
28
|
+
export { EVAL_RERUN_TYPES, assertEvalRerunTypeSupported, evalRerunSemantics, parseEvalRerunType } from "./rerun-types.js";
|
|
29
|
+
export { LocalDockerExecutionProvider, adoptLocalDockerLeaseEpoch, parseExecutionProviderStatus, parseLocalProviderExecutionRecord, readLocalDockerProcessRecord, readLocalDockerProcessRecordByLease, reconcileLocalDockerProcess, recordLocalDockerProcessExit, recordLocalDockerProcessStart, releaseLocalDockerProcessRecord, waitForLocalDockerProcessTerminal } from "./local-docker-provider.js";
|
|
30
|
+
export { loadTrialEnvironmentImages } from "./trial-environment-evidence.js";
|
|
31
|
+
export { resolvedImageMapping } from "./environment-image-planning.js";
|
|
32
|
+
export { readCandidateIneligibleDiagnostic } from "./verifier-eligibility.js";
|
|
33
|
+
export { retryBackoffMs } from "./retry-backoff.js";
|
|
34
|
+
export { physicalRetryWorkItem, physicalRetryWorkItemForTriggerIds } from "./physical-retry-work.js";
|
|
35
|
+
export { ensureEvolutionBaseline, evolutionBaselineFingerprint, parseEvolutionBaseline, readEvolutionBaseline } from "./evolution-baseline.js";
|
|
36
|
+
export { replaySchedulerTrace } from "./scheduler-trace-replay.js";
|
|
37
|
+
export { recoverPromotedEvalTrialPublications } from "./trial-publication-recovery.js";
|
|
38
|
+
export { parseEvalTrialPublication } from "./trial-publication.js";
|
|
39
|
+
export { runRemoteInfrastructureRetries } from "./remote-infrastructure-retry.js";
|
|
40
|
+
export { loadEvalResumeState } from "./resume-state.js";
|
|
41
|
+
export { DOCKER_OWNERSHIP_LABELS, dockerOwnershipLabelMap, dockerResourceOwnership, validateDockerResourceOwnership } from "./docker-ownership.js";
|
|
42
|
+
export { reapOwnedDockerResources } from "./docker-reaper.js";
|
|
43
|
+
export { compileBenchmark, exportStandardBenchmarkDataset, runBenchmarkEval, STANDARD_BENCHMARK_COMPILER } from "./benchmark-run.js";
|
|
44
|
+
export { prepareVerifierEnvironmentRuntime } from "./verifier-runtime.js";
|
|
7
45
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAE,uBAAuB,EAAE,8BAA8B,EAAE,uCAAuC,EAAE,SAAS,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAE,uBAAuB,EAAE,8BAA8B,EAAE,uCAAuC,EAAE,SAAS,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACtQ,OAAO,EAAE,6BAA6B,EAAE,4BAA4B,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AAIhI,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAClE,OAAO,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,4BAA4B,EAAE,MAAM,qBAAqB,CAAC;AACnH,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAC/G,OAAO,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAEjH,OAAO,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAEhL,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,6BAA6B,EAAE,8BAA8B,EAAE,oCAAoC,EAAE,+BAA+B,EAAE,MAAM,qBAAqB,CAAC;AAE3K,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,sBAAsB,EAAE,2BAA2B,EAAE,MAAM,+BAA+B,CAAC;AACpG,OAAO,EAAE,+BAA+B,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AAI3G,OAAO,EAAE,oCAAoC,EAAE,8BAA8B,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,sBAAsB,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAErV,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAClH,OAAO,EAAE,0CAA0C,EAAE,MAAM,oCAAoC,CAAC;AAChG,OAAO,EAAE,mBAAmB,EAAE,+BAA+B,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAEjH,OAAO,EAAE,4BAA4B,EAAE,MAAM,4BAA4B,CAAC;AAE1E,OAAO,EAAE,4BAA4B,EAAE,MAAM,eAAe,CAAC;AAG7D,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE1E,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAEzG,OAAO,EAAE,0BAA0B,EAAE,qCAAqC,EAAE,iCAAiC,EAAE,+BAA+B,EAAE,uBAAuB,EAAE,8BAA8B,EAAE,MAAM,yBAAyB,CAAC;AAEzO,OAAO,EAAE,4BAA4B,EAAE,MAAM,+BAA+B,CAAC;AAC7E,OAAO,EAAE,2BAA2B,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAE7F,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClL,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEhF,OAAO,EAAE,gBAAgB,EAAE,4BAA4B,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE1H,OAAO,EAAE,4BAA4B,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,iCAAiC,EAAE,4BAA4B,EAAE,mCAAmC,EAAE,2BAA2B,EAAE,4BAA4B,EAAE,6BAA6B,EAAE,+BAA+B,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAC;AAGxY,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAC7E,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAAE,iCAAiC,EAAE,MAAM,2BAA2B,CAAC;AAE9E,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,qBAAqB,EAAE,kCAAkC,EAAE,MAAM,0BAA0B,CAAC;AACrG,OAAO,EAAE,uBAAuB,EAAE,4BAA4B,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAE/I,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE,OAAO,EAAE,oCAAoC,EAAE,MAAM,iCAAiC,CAAC;AAEvF,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AAEnE,OAAO,EAAE,8BAA8B,EAAE,MAAM,kCAAkC,CAAC;AAClF,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,+BAA+B,EAAE,MAAM,uBAAuB,CAAC;AACnJ,OAAO,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,EAAE,gBAAgB,EAAE,8BAA8B,EAAE,gBAAgB,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AACrI,OAAO,EAAE,iCAAiC,EAAE,MAAM,uBAAuB,CAAC"}
|