humanish 0.108.0 → 0.109.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +58 -59
- package/CHANGELOG.md +13 -0
- package/CONTEXT.md +44 -44
- package/README.md +3 -3
- package/TELEMETRY.md +2 -2
- package/dist/actors/codex/app-server-trace.js +2 -1
- package/dist/actors/codex/restricted-participant.js +12 -12
- package/dist/actors/codex/restricted-session.d.ts +2 -2
- package/dist/actors/computer-use/actor.d.ts +6 -6
- package/dist/actors/computer-use/actor.js +2 -2
- package/dist/actors/computer-use/admission-limit.d.ts +2 -2
- package/dist/actors/computer-use/admission-limit.js +3 -3
- package/dist/actors/computer-use/executor-error.d.ts +2 -2
- package/dist/actors/computer-use/executor-error.js +3 -3
- package/dist/actors/computer-use/loop/actions.js +5 -5
- package/dist/actors/computer-use/loop/debrief.js +2 -2
- package/dist/actors/computer-use/loop/ending.d.ts +1 -1
- package/dist/actors/computer-use/loop/ending.js +12 -11
- package/dist/actors/computer-use/loop/observation.js +6 -5
- package/dist/actors/computer-use/loop/provider-call.js +11 -11
- package/dist/actors/computer-use/loop/session.d.ts +4 -4
- package/dist/actors/computer-use/loop/session.js +2 -2
- package/dist/actors/computer-use/loop/trace.js +3 -2
- package/dist/actors/computer-use/loop/types.d.ts +24 -24
- package/dist/actors/computer-use/loop/usage.d.ts +2 -2
- package/dist/actors/computer-use/loop.d.ts +6 -1
- package/dist/actors/computer-use/loop.js +26 -4
- package/dist/actors/computer-use/openai-provider.d.ts +2 -2
- package/dist/actors/computer-use/openai-provider.js +8 -8
- package/dist/actors/computer-use/provider-error.d.ts +4 -4
- package/dist/actors/computer-use/provider-error.js +6 -6
- package/dist/actors/contract.d.ts +4 -4
- package/dist/actors/contract.js +1 -1
- package/dist/actors/local-agent/claude-session.js +3 -3
- package/dist/actors/local-agent/cli.d.ts +1 -1
- package/dist/actors/local-agent/cli.js +1 -1
- package/dist/actors/reasoning-effort.js +3 -3
- package/dist/actors/registry.d.ts +2 -2
- package/dist/actors/registry.js +3 -3
- package/dist/actors/scripted-browser/actor.d.ts +2 -2
- package/dist/actors/scripted-browser/actor.js +1 -1
- package/dist/analysis/automatic-completion.d.ts +7 -7
- package/dist/analysis/automatic-completion.js +1 -1
- package/dist/analysis/automatic-config.d.ts +6 -6
- package/dist/analysis/automatic-config.js +1 -1
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +1 -1
- package/dist/analysis/types.d.ts +6 -0
- package/dist/browser-control/client.js +16 -16
- package/dist/browser-control/dispatcher.js +4 -4
- package/dist/browser-control/protocol.js +14 -14
- package/dist/browser-control/recording-transfer.js +4 -4
- package/dist/browser-control/transport.js +7 -7
- package/dist/cli/commands/analysis-signals.d.ts +2 -2
- package/dist/cli/commands/comms.js +14 -22
- package/dist/cli/commands/observe.d.ts +0 -1
- package/dist/cli/commands/observe.js +0 -15
- package/dist/cli/commands/run-command.d.ts +2 -2
- package/dist/cli/commands/run-command.js +3 -3
- package/dist/cli/commands/runs.js +3 -5
- package/dist/cli/commands/setup.js +5 -7
- package/dist/cli/commands/study-format.d.ts +9 -0
- package/dist/cli/commands/{lab-format.js → study-format.js} +5 -5
- package/dist/cli/commands/{lab-route-computer-use.d.ts → study-route-computer-use.d.ts} +6 -6
- package/dist/cli/commands/{lab-route-computer-use.js → study-route-computer-use.js} +12 -12
- package/dist/cli/commands/{lab-route-open.d.ts → study-route-open.d.ts} +4 -4
- package/dist/cli/commands/{lab-route-open.js → study-route-open.js} +2 -2
- package/dist/cli/commands/{lab-route-preview.d.ts → study-route-preview.d.ts} +7 -7
- package/dist/cli/commands/{lab-route-preview.js → study-route-preview.js} +3 -3
- package/dist/cli/commands/study-route-run.d.ts +19 -0
- package/dist/cli/commands/{lab-route-run.js → study-route-run.js} +6 -6
- package/dist/cli/commands/study-route-scripted.d.ts +17 -0
- package/dist/cli/commands/{lab-route-scripted.js → study-route-scripted.js} +4 -4
- package/dist/cli/commands/{lab-route-shared-world.d.ts → study-route-shared-world.d.ts} +5 -5
- package/dist/cli/commands/{lab-route-shared-world.js → study-route-shared-world.js} +7 -7
- package/dist/cli/commands/study-route-terminal.d.ts +17 -0
- package/dist/cli/commands/{lab-route-terminal.js → study-route-terminal.js} +4 -4
- package/dist/cli/commands/study-run.d.ts +9 -0
- package/dist/cli/commands/{lab-run.js → study-run.js} +34 -34
- package/dist/cli/commands/{lab-scorer.d.ts → study-scorer.d.ts} +5 -5
- package/dist/cli/commands/{lab-scorer.js → study-scorer.js} +1 -1
- package/dist/cli/commands/study.js +13 -49
- package/dist/cli/commands/tui.js +12 -12
- package/dist/cli/commands/watch.js +30 -65
- package/dist/cli/deprecations.d.ts +1 -13
- package/dist/cli/deprecations.js +0 -30
- package/dist/cli/doctor.d.ts +3 -3
- package/dist/cli/doctor.js +16 -16
- package/dist/cli/first-run-path.d.ts +4 -4
- package/dist/cli/first-run-path.js +7 -7
- package/dist/cli/io.d.ts +3 -3
- package/dist/cli/io.js +2 -2
- package/dist/cli/observer-follow.d.ts +2 -2
- package/dist/cli/observer-follow.js +2 -2
- package/dist/cli/orientation.d.ts +4 -4
- package/dist/cli/orientation.js +58 -20
- package/dist/cli/program.js +3 -4
- package/dist/cli/telemetry.d.ts +1 -1
- package/dist/cli/telemetry.js +14 -20
- package/dist/comms/catch-host.d.ts +1 -1
- package/dist/comms/catch-host.js +3 -3
- package/dist/comms/connections.d.ts +1 -1
- package/dist/comms/connections.js +2 -2
- package/dist/comms/external-evidence.d.ts +5 -5
- package/dist/comms/external-evidence.js +4 -3
- package/dist/comms/fake-inbox.d.ts +2 -2
- package/dist/comms/fake-inbox.js +2 -2
- package/dist/comms/receiving-runtime.d.ts +4 -4
- package/dist/comms/sandbox-catch.d.ts +2 -2
- package/dist/comms/sandbox-catch.js +2 -2
- package/dist/comms/setup.d.ts +2 -2
- package/dist/comms/setup.js +29 -32
- package/dist/feedback/draft.js +2 -2
- package/dist/feedback/export.js +3 -2
- package/dist/feedback/proof.d.ts +1 -1
- package/dist/feedback/proof.js +2 -2
- package/dist/guest/bootstrap.js +6 -6
- package/dist/guest/browser-tools.js +8 -8
- package/dist/guest/chromium-text.js +17 -17
- package/dist/guest/desktop-executor.d.ts +2 -2
- package/dist/guest/desktop-executor.js +20 -20
- package/dist/guest/desktop-keys.js +4 -4
- package/dist/guest/desktop-media.d.ts +2 -2
- package/dist/guest/desktop-media.js +16 -16
- package/dist/guest/desktop-native.js +10 -10
- package/dist/guest/runtime-desktop.js +2 -2
- package/dist/guest/runtime-processes.d.ts +2 -2
- package/dist/guest/runtime-processes.js +2 -2
- package/dist/guest/runtime.js +5 -5
- package/dist/index.d.ts +13 -15
- package/dist/index.js +6 -8
- package/dist/observer/artifact.d.ts +1 -1
- package/dist/observer/artifact.js +1 -1
- package/dist/observer/data.d.ts +1 -1
- package/dist/observer/library.d.ts +2 -0
- package/dist/observer/library.js +4 -1
- package/dist/observer/render.js +7 -1
- package/dist/observer/run-routes.d.ts +1 -0
- package/dist/observer/run-routes.js +37 -12
- package/dist/observer/serve.d.ts +1 -1
- package/dist/observer-app.html +1 -1
- package/dist/routes/computer-use/bundle-parts.js +3 -2
- package/dist/routes/computer-use/bundle.d.ts +2 -2
- package/dist/routes/computer-use/bundle.js +7 -6
- package/dist/routes/computer-use/e2b-desktop/comms.d.ts +5 -5
- package/dist/routes/computer-use/e2b-desktop/comms.js +5 -4
- package/dist/routes/computer-use/e2b-desktop/fidelity.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.js +3 -3
- package/dist/routes/computer-use/e2b-desktop/start.js +3 -4
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +2 -2
- package/dist/routes/computer-use/e2b-desktop/teardown.js +2 -3
- package/dist/routes/computer-use/external-comms.d.ts +3 -3
- package/dist/routes/computer-use/fanout-bundle.js +11 -10
- package/dist/routes/computer-use/fanout-records.js +2 -2
- package/dist/routes/computer-use/live-flush.d.ts +3 -3
- package/dist/routes/computer-use/live-phase.d.ts +2 -3
- package/dist/routes/computer-use/live-phase.js +3 -3
- package/dist/routes/computer-use/local-tree-pack.d.ts +5 -5
- package/dist/routes/computer-use/local-tree-pack.js +1 -1
- package/dist/routes/computer-use/local-vm.d.ts +8 -8
- package/dist/routes/computer-use/local-vm.js +2 -2
- package/dist/routes/computer-use/participant-desktop.d.ts +3 -3
- package/dist/routes/computer-use/participant-desktop.js +1 -1
- package/dist/routes/computer-use/participant-execution.js +1 -1
- package/dist/routes/computer-use/participant-model.js +3 -3
- package/dist/routes/computer-use/participant-prompt.d.ts +5 -5
- package/dist/routes/computer-use/participant-prompt.js +3 -3
- package/dist/routes/computer-use/participant-runs.d.ts +10 -10
- package/dist/routes/computer-use/participant-runs.js +14 -13
- package/dist/routes/computer-use/plan.d.ts +12 -12
- package/dist/routes/computer-use/plan.js +24 -24
- package/dist/routes/computer-use/preflight.d.ts +5 -5
- package/dist/routes/computer-use/preflight.js +5 -5
- package/dist/routes/computer-use/rerun-selection.d.ts +1 -1
- package/dist/routes/computer-use/rerun-selection.js +3 -2
- package/dist/routes/computer-use/result.d.ts +4 -4
- package/dist/routes/computer-use/result.js +13 -12
- package/dist/routes/computer-use/route.d.ts +12 -12
- package/dist/routes/computer-use/route.js +19 -19
- package/dist/routes/computer-use/self-report.js +1 -1
- package/dist/routes/computer-use/setup.d.ts +22 -22
- package/dist/routes/computer-use/setup.js +15 -15
- package/dist/routes/computer-use/single-bundle.d.ts +4 -4
- package/dist/routes/computer-use/single-bundle.js +14 -13
- package/dist/routes/computer-use/subject-projection.d.ts +3 -3
- package/dist/routes/computer-use/types.d.ts +34 -34
- package/dist/routes/computer-use/types.js +3 -3
- package/dist/routes/preview.d.ts +6 -6
- package/dist/routes/preview.js +5 -5
- package/dist/routes/scripted/bundle.d.ts +6 -6
- package/dist/routes/scripted/bundle.js +10 -9
- package/dist/routes/scripted/plan.d.ts +7 -7
- package/dist/routes/scripted/plan.js +6 -6
- package/dist/routes/scripted/result.d.ts +3 -3
- package/dist/routes/scripted/result.js +9 -9
- package/dist/routes/scripted/route.d.ts +9 -9
- package/dist/routes/scripted/route.js +11 -11
- package/dist/routes/scripted/scenario.d.ts +1 -1
- package/dist/routes/scripted/scenario.js +3 -3
- package/dist/routes/scripted/setup.d.ts +6 -6
- package/dist/routes/scripted/setup.js +4 -4
- package/dist/routes/scripted/subject.d.ts +5 -5
- package/dist/routes/scripted/subject.js +3 -3
- package/dist/routes/scripted/surface-records.d.ts +1 -1
- package/dist/routes/scripted/surface-records.js +2 -2
- package/dist/routes/scripted/surface-sessions.d.ts +3 -3
- package/dist/routes/scripted/types.d.ts +17 -17
- package/dist/routes/scripted/types.js +1 -1
- package/dist/routes/shared-world/bundle-records.js +3 -3
- package/dist/routes/shared-world/bundle.js +15 -14
- package/dist/routes/shared-world/checkpoints.d.ts +3 -3
- package/dist/routes/shared-world/checkpoints.js +1 -1
- package/dist/routes/shared-world/comms.d.ts +9 -9
- package/dist/routes/shared-world/comms.js +6 -5
- package/dist/routes/shared-world/participant-specs.d.ts +3 -3
- package/dist/routes/shared-world/participant-specs.js +7 -7
- package/dist/routes/shared-world/plan.d.ts +9 -9
- package/dist/routes/shared-world/plan.js +9 -9
- package/dist/routes/shared-world/provisioned.d.ts +8 -8
- package/dist/routes/shared-world/provisioned.js +4 -4
- package/dist/routes/shared-world/result.d.ts +6 -6
- package/dist/routes/shared-world/result.js +11 -10
- package/dist/routes/shared-world/route.d.ts +11 -11
- package/dist/routes/shared-world/route.js +19 -19
- package/dist/routes/shared-world/setup.d.ts +13 -13
- package/dist/routes/shared-world/setup.js +19 -19
- package/dist/routes/shared-world/types.d.ts +27 -27
- package/dist/routes/terminal/adapter.d.ts +2 -2
- package/dist/routes/terminal/adapter.js +4 -4
- package/dist/routes/terminal/bundle.d.ts +11 -11
- package/dist/routes/terminal/bundle.js +12 -12
- package/dist/routes/terminal/credentials.d.ts +4 -4
- package/dist/routes/terminal/credentials.js +5 -5
- package/dist/routes/terminal/dry-run.d.ts +4 -4
- package/dist/routes/terminal/dry-run.js +9 -9
- package/dist/routes/terminal/ledger.d.ts +2 -2
- package/dist/routes/terminal/lifetime.d.ts +1 -1
- package/dist/routes/terminal/lifetime.js +1 -1
- package/dist/routes/terminal/live-finish.d.ts +2 -2
- package/dist/routes/terminal/live-finish.js +7 -7
- package/dist/routes/terminal/live-sandbox.d.ts +2 -2
- package/dist/routes/terminal/live-sandbox.js +3 -2
- package/dist/routes/terminal/participant-text.js +2 -1
- package/dist/routes/terminal/persona.d.ts +2 -2
- package/dist/routes/terminal/persona.js +2 -2
- package/dist/routes/terminal/plan.d.ts +7 -7
- package/dist/routes/terminal/plan.js +5 -5
- package/dist/routes/terminal/result.d.ts +5 -5
- package/dist/routes/terminal/result.js +7 -7
- package/dist/routes/terminal/route.d.ts +9 -9
- package/dist/routes/terminal/route.js +14 -14
- package/dist/routes/terminal/session.d.ts +5 -5
- package/dist/routes/terminal/session.js +7 -7
- package/dist/routes/terminal/trace.d.ts +2 -2
- package/dist/routes/terminal/types.d.ts +24 -24
- package/dist/routes/terminal/types.js +2 -2
- package/dist/run/bundle.d.ts +7 -7
- package/dist/run/bundle.js +2 -2
- package/dist/run/costs.d.ts +23 -0
- package/dist/run/costs.js +72 -43
- package/dist/run/dry-run-inputs.d.ts +1 -1
- package/dist/run/dry-run-inputs.js +1 -1
- package/dist/run/dry-run.js +2 -2
- package/dist/run/outcomes.d.ts +1 -1
- package/dist/run/participant.d.ts +1 -1
- package/dist/run/preflight-receipts.js +1 -1
- package/dist/run/pricing.js +2 -2
- package/dist/run/projection.d.ts +32 -32
- package/dist/run/projection.js +44 -44
- package/dist/run/reclaim.d.ts +1 -1
- package/dist/run/reclaim.js +1 -1
- package/dist/run/results.d.ts +2 -2
- package/dist/run/run-cost.d.ts +53 -0
- package/dist/run/run-cost.js +105 -0
- package/dist/run/run-index-analysis.d.ts +15 -0
- package/dist/run/run-index-analysis.js +77 -0
- package/dist/run/run-index.d.ts +11 -1
- package/dist/run/run-index.js +63 -49
- package/dist/run/run.d.ts +3 -3
- package/dist/run/run.js +2 -2
- package/dist/run/shared-world-evidence.d.ts +1 -1
- package/dist/run/shared-world-shape.js +1 -1
- package/dist/run/stats.d.ts +3 -3
- package/dist/run/stats.js +14 -14
- package/dist/run/status.d.ts +10 -2
- package/dist/run/status.js +6 -3
- package/dist/run/streams.d.ts +2 -2
- package/dist/run/study-provenance.d.ts +4 -2
- package/dist/run/study-provenance.js +7 -4
- package/dist/run/study-result.d.ts +0 -5
- package/dist/run/study-result.js +1 -1
- package/dist/run-study.d.ts +112 -0
- package/dist/{run-lab.js → run-study.js} +52 -45
- package/dist/{lab → study}/adapter-extension.d.ts +8 -20
- package/dist/{lab → study}/adapter-extension.js +2 -14
- package/dist/{lab → study}/adapter-scorer-loader.d.ts +10 -10
- package/dist/{lab → study}/adapter-scorer-loader.js +3 -3
- package/dist/study/composition-rules.d.ts +7 -0
- package/dist/{lab → study}/composition-rules.js +12 -12
- package/dist/study/config.d.ts +15 -0
- package/dist/{lab → study}/config.js +37 -20
- package/dist/study/convert.d.ts +1 -1
- package/dist/study/convert.js +15 -15
- package/dist/{lab → study}/device-presets.d.ts +2 -2
- package/dist/{lab → study}/discover.d.ts +29 -23
- package/dist/{lab → study}/discover.js +97 -118
- package/dist/{lab → study}/doctor.d.ts +11 -11
- package/dist/{lab → study}/doctor.js +18 -18
- package/dist/study/files.d.ts +2 -2
- package/dist/study/files.js +6 -5
- package/dist/{lab → study}/init-templates.d.ts +2 -2
- package/dist/{lab → study}/init-templates.js +16 -11
- package/dist/{lab → study}/init.d.ts +1 -1
- package/dist/{lab → study}/init.js +7 -4
- package/dist/{lab → study}/keys.d.ts +2 -2
- package/dist/{lab → study}/keys.js +10 -10
- package/dist/study/migrate.d.ts +1 -1
- package/dist/{lab → study}/parse/actors.d.ts +5 -5
- package/dist/{lab → study}/parse/actors.js +5 -5
- package/dist/study/parse/comms.d.ts +12 -0
- package/dist/study/parse/execution.d.ts +19 -0
- package/dist/{lab → study}/parse/study-v3.d.ts +3 -5
- package/dist/{lab → study}/parse/study-v3.js +3 -3
- package/dist/{lab → study}/parse/subject-state.d.ts +8 -8
- package/dist/{lab → study}/parse/subject-state.js +4 -4
- package/dist/{lab → study}/parse/subject.d.ts +3 -3
- package/dist/{lab → study}/parse/subject.js +8 -8
- package/dist/{lab → study}/parse/values.d.ts +2 -2
- package/dist/{lab → study}/persona-resolve.d.ts +4 -4
- package/dist/{lab → study}/persona-resolve.js +2 -2
- package/dist/{lab → study}/plan-base.d.ts +7 -7
- package/dist/{lab → study}/plan-base.js +1 -1
- package/dist/{lab → study}/plan-participants.d.ts +10 -10
- package/dist/{lab → study}/plan-participants.js +4 -4
- package/dist/{lab → study}/plan-types.d.ts +28 -28
- package/dist/{lab → study}/plan-types.js +2 -2
- package/dist/study/plan.d.ts +13 -0
- package/dist/{lab → study}/plan.js +20 -20
- package/dist/study/preflight-probes.d.ts +3 -0
- package/dist/{lab → study}/preflight-probes.js +9 -8
- package/dist/study/preflight-result.d.ts +8 -0
- package/dist/{lab → study}/preflight-result.js +3 -3
- package/dist/{lab → study}/preflight.d.ts +31 -31
- package/dist/{lab → study}/preflight.js +8 -8
- package/dist/{lab → study}/requirements.d.ts +3 -3
- package/dist/{lab → study}/requirements.js +1 -1
- package/dist/study/route-inputs.d.ts +28 -0
- package/dist/{lab → study}/route-inputs.js +1 -1
- package/dist/{lab → study}/routing.d.ts +12 -12
- package/dist/{lab/run-lab-events.d.ts → study/run-study-events.d.ts} +10 -10
- package/dist/{lab/run-lab-events.js → study/run-study-events.js} +3 -3
- package/dist/{lab/run-lab-homes.d.ts → study/run-study-homes.d.ts} +8 -17
- package/dist/study/run-study-homes.js +3 -0
- package/dist/study/run-study-options.d.ts +28 -0
- package/dist/{lab/run-lab-options.js → study/run-study-options.js} +13 -12
- package/dist/{lab/lab-deps.d.ts → study/study-deps.d.ts} +2 -2
- package/dist/{lab/lab-deps.js → study/study-deps.js} +2 -2
- package/dist/{lab → study}/summary.d.ts +10 -10
- package/dist/{lab → study}/summary.js +15 -15
- package/dist/{lab → study}/tasks.d.ts +3 -3
- package/dist/{lab → study}/tasks.js +1 -1
- package/dist/{lab → study}/types.d.ts +104 -104
- package/dist/{lab → study}/types.js +1 -1
- package/dist/{lab → study}/validation.d.ts +19 -19
- package/dist/{lab → study}/validation.js +13 -13
- package/dist/{lab → study}/warnings.d.ts +4 -4
- package/dist/{lab → study}/warnings.js +1 -1
- package/dist/subject/clone.d.ts +6 -6
- package/dist/subject/clone.js +2 -2
- package/dist/subject/local-tree-archive.d.ts +2 -2
- package/dist/subject/local-tree-archive.js +3 -3
- package/dist/subject/local-tree.d.ts +3 -3
- package/dist/subject/runtime.js +2 -2
- package/dist/subject/serve.d.ts +4 -4
- package/dist/subject/state.d.ts +3 -3
- package/dist/subject/state.js +1 -1
- package/dist/subject/steps.d.ts +1 -1
- package/dist/subject/steps.js +1 -1
- package/dist/substrates/desktop-session.js +2 -2
- package/dist/substrates/detached.d.ts +1 -1
- package/dist/substrates/e2b/desktop-browser.d.ts +4 -4
- package/dist/substrates/e2b/desktop-browser.js +1 -1
- package/dist/substrates/e2b/desktop-cdp.d.ts +1 -1
- package/dist/substrates/e2b/desktop-cdp.js +1 -1
- package/dist/substrates/e2b/desktop-executor.d.ts +1 -1
- package/dist/substrates/e2b/desktop-executor.js +12 -12
- package/dist/substrates/e2b/desktop-geometry.d.ts +1 -1
- package/dist/substrates/e2b/desktop-geometry.js +1 -1
- package/dist/substrates/e2b/desktop-media.d.ts +3 -3
- package/dist/substrates/e2b/lifetime.js +1 -1
- package/dist/substrates/e2b/sandbox.d.ts +3 -3
- package/dist/substrates/e2b/sandbox.js +1 -1
- package/dist/substrates/e2b/sdk.d.ts +1 -1
- package/dist/substrates/e2b/sdk.js +1 -1
- package/dist/substrates/local/runtime-config.d.ts +4 -4
- package/dist/substrates/local/runtime-config.js +4 -4
- package/dist/tui/contract.d.ts +8 -8
- package/dist/tui/contract.js +2 -2
- package/dist/tui/launch.d.ts +3 -3
- package/dist/tui/launch.js +22 -16
- package/dist/tui-app.js +139 -139
- package/dist/verify/subject.d.ts +1 -1
- package/dist/verify/subject.js +1 -1
- package/dist/verify/verify.js +2 -1
- package/docs/architecture/actor-contract.md +5 -5
- package/docs/architecture/browser-control.md +1 -1
- package/docs/architecture/external-public-shared-world.md +8 -7
- package/docs/architecture/github-feedback-loop.md +1 -1
- package/docs/architecture/task-protocol-support.md +1 -1
- package/docs/architecture/terminal-product-route.md +18 -18
- package/docs/contracts/adapter-fixtures.md +2 -2
- package/docs/contracts/errors.md +8 -8
- package/docs/contracts/run-bundle.md +1 -4
- package/docs/contracts/schemas.md +61 -42
- package/docs/contracts/study-analysis.md +4 -1
- package/docs/contracts/study-costs.md +17 -2
- package/examples/README.md +1 -1
- package/examples/scorer/scorer.mjs +3 -3
- package/package.json +1 -1
- package/skills/humanish/SKILL.md +8 -6
- package/skills/humanish/agents/openai.yaml +1 -1
- package/dist/cli/commands/lab-format.d.ts +0 -9
- package/dist/cli/commands/lab-route-run.d.ts +0 -19
- package/dist/cli/commands/lab-route-scripted.d.ts +0 -17
- package/dist/cli/commands/lab-route-terminal.d.ts +0 -17
- package/dist/cli/commands/lab-run.d.ts +0 -9
- package/dist/lab/composition-rules.d.ts +0 -7
- package/dist/lab/config.d.ts +0 -7
- package/dist/lab/deprecated-fields.d.ts +0 -10
- package/dist/lab/deprecated-fields.js +0 -24
- package/dist/lab/parse/comms.d.ts +0 -12
- package/dist/lab/parse/execution.d.ts +0 -19
- package/dist/lab/plan.d.ts +0 -13
- package/dist/lab/preflight-probes.d.ts +0 -3
- package/dist/lab/preflight-result.d.ts +0 -8
- package/dist/lab/route-inputs.d.ts +0 -28
- package/dist/lab/run-lab-homes.js +0 -7
- package/dist/lab/run-lab-options.d.ts +0 -28
- package/dist/library-aliases.d.ts +0 -56
- package/dist/library-aliases.js +0 -22
- package/dist/run-lab.d.ts +0 -112
- /package/dist/{lab → study}/device-presets.js +0 -0
- /package/dist/{lab → study}/init-paths.d.ts +0 -0
- /package/dist/{lab → study}/init-paths.js +0 -0
- /package/dist/{lab → study}/init-plan.d.ts +0 -0
- /package/dist/{lab → study}/init-plan.js +0 -0
- /package/dist/{lab → study}/parse/comms.js +0 -0
- /package/dist/{lab → study}/parse/execution.js +0 -0
- /package/dist/{lab → study}/parse/values.js +0 -0
- /package/dist/{lab → study}/participant-assignment.d.ts +0 -0
- /package/dist/{lab → study}/participant-assignment.js +0 -0
- /package/dist/{lab → study}/persona.d.ts +0 -0
- /package/dist/{lab → study}/persona.js +0 -0
- /package/dist/{lab → study}/routing.js +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { formatCuaDiagnostics, formatCuaStopCause } from "../../routes/computer-use/diagnostics.js";
|
|
2
|
-
/** A
|
|
2
|
+
/** A run's first lines: the command that ran, whether it was a dry run, how it ended, and its route. */
|
|
3
3
|
function runHeader(result, route) {
|
|
4
4
|
const kind = result.dryRun === true ? "dry run" : result.dryRun === false ? "live run" : "run";
|
|
5
5
|
return [
|
|
@@ -19,7 +19,7 @@ function withError(error, lines) {
|
|
|
19
19
|
function participantStatus(status) {
|
|
20
20
|
return status === "contract_proof_only" ? "dry run, nothing ran live" : status;
|
|
21
21
|
}
|
|
22
|
-
export function
|
|
22
|
+
export function formatConcurrentSharedWorldStudyHuman(result) {
|
|
23
23
|
return withError(result.error, [
|
|
24
24
|
...runHeader(result, "shared-world"),
|
|
25
25
|
`run: ${result.runId}`,
|
|
@@ -47,7 +47,7 @@ export function formatConcurrentSharedWorldLabHuman(result) {
|
|
|
47
47
|
...result.warnings.map((warning) => `warning: ${warning}`),
|
|
48
48
|
]);
|
|
49
49
|
}
|
|
50
|
-
export function
|
|
50
|
+
export function formatTerminalStudyHuman(result) {
|
|
51
51
|
return withError(result.error, [
|
|
52
52
|
...runHeader(result, "terminal"),
|
|
53
53
|
`run: ${result.runId}`,
|
|
@@ -60,7 +60,7 @@ export function formatTerminalLabHuman(result) {
|
|
|
60
60
|
...result.warnings.map((warning) => `warning: ${warning}`),
|
|
61
61
|
]);
|
|
62
62
|
}
|
|
63
|
-
export function
|
|
63
|
+
export function formatScriptedStudyHuman(result) {
|
|
64
64
|
return withError(result.error, [
|
|
65
65
|
...runHeader(result, "scripted"),
|
|
66
66
|
`run: ${result.runId}`,
|
|
@@ -79,7 +79,7 @@ export function formatScriptedLabHuman(result) {
|
|
|
79
79
|
...result.warnings.map((warning) => `warning: ${warning}`),
|
|
80
80
|
]);
|
|
81
81
|
}
|
|
82
|
-
export function
|
|
82
|
+
export function formatCuaStudyHuman(result) {
|
|
83
83
|
return withError(result.error, [
|
|
84
84
|
...runHeader(result, "computer-use"),
|
|
85
85
|
`run: ${result.runId}`,
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
|
-
import type {
|
|
3
|
-
import { type CliIo, type
|
|
4
|
-
import type { RouteRun } from "./
|
|
2
|
+
import type { StudyConfig } from "../../study/types.js";
|
|
3
|
+
import { type CliIo, type StudyCommandOptions } from "../io.js";
|
|
4
|
+
import type { RouteRun } from "./study-route-run.js";
|
|
5
5
|
interface ComputerUseRouteArgs {
|
|
6
6
|
command: Command;
|
|
7
7
|
io: CliIo;
|
|
8
|
-
config:
|
|
8
|
+
config: StudyConfig;
|
|
9
9
|
mode: "run" | "watch";
|
|
10
|
-
options:
|
|
10
|
+
options: StudyCommandOptions;
|
|
11
11
|
}
|
|
12
12
|
/**
|
|
13
|
-
* The computer-use route's CLI setup: its settings, the watch and exposure plan, the
|
|
13
|
+
* The computer-use route's CLI setup: its settings, the watch and exposure plan, the runStudyWith options
|
|
14
14
|
* with the live Observer hook, and how it presents the outcome or a run error. Undefined when
|
|
15
15
|
* setup has already written its own result.
|
|
16
16
|
*/
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
import { automaticAnalysisSucceeded } from "../../analysis/automatic-completion.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
|
-
import {} from "../../run-
|
|
4
|
-
import {
|
|
3
|
+
import {} from "../../run-study.js";
|
|
4
|
+
import { resolveStudyDryRun } from "../../study/plan.js";
|
|
5
5
|
import { serveObserver } from "../../observer/render.js";
|
|
6
6
|
import { startExposedObserver, validateExposure } from "../../observer/exposure.js";
|
|
7
7
|
import { ServeTunnelError } from "../../observer/tunnel.js";
|
|
8
8
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
9
9
|
import { onRunShutdown } from "./run-signals.js";
|
|
10
|
-
import {
|
|
10
|
+
import { parseStudyCount, parseParticipantIds, parseObserverPort, wantsJson, writeResult, } from "../io.js";
|
|
11
11
|
import { exposureRequestFromOptions, followObserver, planObserver, showObserver, staticObserverOpen, withObserverServer, } from "../observer-follow.js";
|
|
12
|
-
import {
|
|
13
|
-
import { resolveRouteShouldOpen } from "./
|
|
12
|
+
import { formatCuaStudyHuman } from "./study-format.js";
|
|
13
|
+
import { resolveRouteShouldOpen } from "./study-route-open.js";
|
|
14
14
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
15
15
|
/**
|
|
16
|
-
* The computer-use route's CLI setup: its settings, the watch and exposure plan, the
|
|
16
|
+
* The computer-use route's CLI setup: its settings, the watch and exposure plan, the runStudyWith options
|
|
17
17
|
* with the live Observer hook, and how it presents the outcome or a run error. Undefined when
|
|
18
18
|
* setup has already written its own result.
|
|
19
19
|
*/
|
|
@@ -60,7 +60,7 @@ function refuseCua(args, dryRun, code, message) {
|
|
|
60
60
|
warnings: [],
|
|
61
61
|
error: { code, message },
|
|
62
62
|
};
|
|
63
|
-
writeResult(args.command, args.io, result,
|
|
63
|
+
writeResult(args.command, args.io, result, formatCuaStudyHuman);
|
|
64
64
|
args.io.setExitCode(2);
|
|
65
65
|
}
|
|
66
66
|
/** Parses the options, or writes the refusal and returns undefined. */
|
|
@@ -78,13 +78,13 @@ function resolveCuaSettings(args) {
|
|
|
78
78
|
args.io.setExitCode(2);
|
|
79
79
|
return undefined;
|
|
80
80
|
}
|
|
81
|
-
const count =
|
|
81
|
+
const count = parseStudyCount(args.options.count, args.config.actors[0]?.count ?? 1);
|
|
82
82
|
if (count === null) {
|
|
83
83
|
args.io.writeErr("error: --count must be a positive integer.\n");
|
|
84
84
|
args.io.setExitCode(2);
|
|
85
85
|
return undefined;
|
|
86
86
|
}
|
|
87
|
-
const dryRun =
|
|
87
|
+
const dryRun = resolveStudyDryRun(args.config, args.options.dryRun, true) ?? true;
|
|
88
88
|
const port = parseObserverPort(args.options.port ?? "0");
|
|
89
89
|
const wantsFollow = args.mode === "watch" && !wantsMachine && args.options.detach !== true && dryRun !== true;
|
|
90
90
|
if (port === null) {
|
|
@@ -122,7 +122,7 @@ function prepareCuaWatch(args, settings) {
|
|
|
122
122
|
return undefined;
|
|
123
123
|
return { exposure: exposeValidation.plan, finishedPlan };
|
|
124
124
|
}
|
|
125
|
-
/** The
|
|
125
|
+
/** The runStudyWith options for this invocation, with the live Observer hook for a followed watch. */
|
|
126
126
|
function cuaRunOptions(args, settings, prepared, live) {
|
|
127
127
|
const { finishedPlan } = prepared;
|
|
128
128
|
return {
|
|
@@ -140,7 +140,7 @@ function cuaRunOptions(args, settings, prepared, live) {
|
|
|
140
140
|
dryRun: settings.dryRun,
|
|
141
141
|
...(settings.wantsFollow
|
|
142
142
|
? {
|
|
143
|
-
// Fires inside
|
|
143
|
+
// Fires inside runStudyWith, before the actor loop and before sandbox creation, so a
|
|
144
144
|
// tunnel-auth failure aborts before any spend and leaves no orphaned sandbox.
|
|
145
145
|
onObserverReady: (observer) => attachLiveObserver(args.io, settings, prepared.exposure, live, observer),
|
|
146
146
|
}
|
|
@@ -235,7 +235,7 @@ async function reportCuaRun(args, prepared, result, live) {
|
|
|
235
235
|
],
|
|
236
236
|
};
|
|
237
237
|
}
|
|
238
|
-
writeResult(args.command, args.io, output,
|
|
238
|
+
writeResult(args.command, args.io, output, formatCuaStudyHuman);
|
|
239
239
|
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
240
240
|
if (server && (result.observer?.ok || attachedObserver)) {
|
|
241
241
|
const followResult = output.observer?.ok
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
|
-
import type { CliIo,
|
|
2
|
+
import type { CliIo, StudyCommandOptions } from "../io.js";
|
|
3
3
|
import { type ObserverPlan } from "../observer-follow.js";
|
|
4
4
|
/**
|
|
5
|
-
* Default browser-open policy for a
|
|
5
|
+
* Default browser-open policy for a study route's run. Mirrors the observe/watch gate:
|
|
6
6
|
* an explicit --open/--no-open wins; --json (machine mode) never auto-opens; otherwise a
|
|
7
|
-
*
|
|
7
|
+
* study-file `defaults.open` wins, and the final fallback opens only for an interactive
|
|
8
8
|
* `watch` on a real TTY. Extracted so every route's CLI setup shares one gate (and one test).
|
|
9
9
|
*/
|
|
10
10
|
export declare function resolveRouteShouldOpen(args: {
|
|
@@ -21,7 +21,7 @@ export declare function watchFinishedPlan(args: {
|
|
|
21
21
|
command: Command;
|
|
22
22
|
io: CliIo;
|
|
23
23
|
mode: "run" | "watch";
|
|
24
|
-
options:
|
|
24
|
+
options: StudyCommandOptions;
|
|
25
25
|
}, wantsMachine: boolean, shouldOpen: boolean): ObserverPlan | null | undefined;
|
|
26
26
|
/** The route's static render opens only when nothing else will open the Observer. */
|
|
27
27
|
export declare function observerOpen(mode: "run" | "watch", finishedPlan: ObserverPlan | undefined, shouldOpen: boolean): boolean;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
2
|
import { planObserver, staticObserverOpen } from "../observer-follow.js";
|
|
3
3
|
/**
|
|
4
|
-
* Default browser-open policy for a
|
|
4
|
+
* Default browser-open policy for a study route's run. Mirrors the observe/watch gate:
|
|
5
5
|
* an explicit --open/--no-open wins; --json (machine mode) never auto-opens; otherwise a
|
|
6
|
-
*
|
|
6
|
+
* study-file `defaults.open` wins, and the final fallback opens only for an interactive
|
|
7
7
|
* `watch` on a real TTY. Extracted so every route's CLI setup shares one gate (and one test).
|
|
8
8
|
*/
|
|
9
9
|
export function resolveRouteShouldOpen(args) {
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
|
-
import type {
|
|
3
|
-
import { type CliIo, type
|
|
4
|
-
import type { RouteRun } from "./
|
|
2
|
+
import type { StudyConfig } from "../../study/types.js";
|
|
3
|
+
import { type CliIo, type StudyCommandOptions } from "../io.js";
|
|
4
|
+
import type { RouteRun } from "./study-route-run.js";
|
|
5
5
|
interface PreviewRouteArgs {
|
|
6
6
|
command: Command;
|
|
7
7
|
io: CliIo;
|
|
8
|
-
|
|
9
|
-
config:
|
|
8
|
+
study: string;
|
|
9
|
+
config: StudyConfig;
|
|
10
10
|
mode: "run" | "watch";
|
|
11
|
-
options:
|
|
11
|
+
options: StudyCommandOptions;
|
|
12
12
|
}
|
|
13
13
|
/**
|
|
14
|
-
* The preview route's CLI setup: the participant count and Observer plan, its
|
|
14
|
+
* The preview route's CLI setup: the participant count and Observer plan, its runStudyWith options, and how it
|
|
15
15
|
* presents the outcome. Undefined when setup has already written its own result.
|
|
16
16
|
*/
|
|
17
17
|
export declare function previewRouteRun(args: PreviewRouteArgs): RouteRun | undefined;
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
2
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
3
|
-
import { formatRunHuman,
|
|
3
|
+
import { formatRunHuman, parseStudyCount, writeResult, } from "../io.js";
|
|
4
4
|
import { planObserver, showObserver, staticObserverOpen } from "../observer-follow.js";
|
|
5
5
|
/**
|
|
6
|
-
* The preview route's CLI setup: the participant count and Observer plan, its
|
|
6
|
+
* The preview route's CLI setup: the participant count and Observer plan, its runStudyWith options, and how it
|
|
7
7
|
* presents the outcome. Undefined when setup has already written its own result.
|
|
8
8
|
*/
|
|
9
9
|
export function previewRouteRun(args) {
|
|
10
|
-
const participantCount =
|
|
10
|
+
const participantCount = parseStudyCount(args.options.count, args.config.actors[0]?.count ?? 4);
|
|
11
11
|
if (participantCount === null) {
|
|
12
12
|
const result = {
|
|
13
13
|
...studyResultIdentity("preview", args.config.id),
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { type InternalRunStudyOptions, type StudyOutcome } from "../../run-study.js";
|
|
2
|
+
import type { StudyConfig } from "../../study/types.js";
|
|
3
|
+
import type { LoadedAdapterScorer } from "./study-scorer.js";
|
|
4
|
+
export interface RouteRun {
|
|
5
|
+
readonly options: InternalRunStudyOptions;
|
|
6
|
+
present(outcome: StudyOutcome): Promise<void>;
|
|
7
|
+
/** Handles an error runStudyWith threw, and rethrows any it does not handle. */
|
|
8
|
+
onRunError?(error: unknown): Promise<void>;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Plans the study with a route's options, then presents the refusal or the run. `beforeRun` runs
|
|
12
|
+
* only for a plan that will run: it loads the scorer, or writes its own refusal and returns
|
|
13
|
+
* undefined, so a refused study never imports the scorer's host code. `afterRun` runs once runStudyWith
|
|
14
|
+
* has returned or thrown, before presentation, which may hold its own signal handlers (watch's
|
|
15
|
+
* Observer and tunnel).
|
|
16
|
+
*/
|
|
17
|
+
export declare function runRoute(config: StudyConfig, run: RouteRun, beforeRun?: () => Promise<{
|
|
18
|
+
scorer?: LoadedAdapterScorer;
|
|
19
|
+
} | undefined>, afterRun?: () => void): Promise<void>;
|
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
// What a route's CLI setup hands the
|
|
2
|
-
// the outcome. runRoute plans the
|
|
3
|
-
import {
|
|
1
|
+
// What a route's CLI setup hands the run command: the runStudyWith options it chose and how it presents
|
|
2
|
+
// the outcome. runRoute plans the study once and runs it between the two.
|
|
3
|
+
import { prepareStudy } from "../../run-study.js";
|
|
4
4
|
/**
|
|
5
|
-
* Plans the
|
|
5
|
+
* Plans the study with a route's options, then presents the refusal or the run. `beforeRun` runs
|
|
6
6
|
* only for a plan that will run: it loads the scorer, or writes its own refusal and returns
|
|
7
|
-
* undefined, so a refused
|
|
7
|
+
* undefined, so a refused study never imports the scorer's host code. `afterRun` runs once runStudyWith
|
|
8
8
|
* has returned or thrown, before presentation, which may hold its own signal handlers (watch's
|
|
9
9
|
* Observer and tunnel).
|
|
10
10
|
*/
|
|
11
11
|
export async function runRoute(config, run, beforeRun = () => Promise.resolve({}), afterRun = () => undefined) {
|
|
12
12
|
let outcome;
|
|
13
13
|
try {
|
|
14
|
-
const prepared = await
|
|
14
|
+
const prepared = await prepareStudy(config, run.options);
|
|
15
15
|
if (prepared.ok) {
|
|
16
16
|
const ready = await beforeRun();
|
|
17
17
|
if (ready === undefined)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import type { StudyConfig } from "../../study/types.js";
|
|
3
|
+
import { type CliIo, type StudyCommandOptions } from "../io.js";
|
|
4
|
+
import type { RouteRun } from "./study-route-run.js";
|
|
5
|
+
interface ScriptedRouteArgs {
|
|
6
|
+
command: Command;
|
|
7
|
+
io: CliIo;
|
|
8
|
+
config: StudyConfig;
|
|
9
|
+
mode: "run" | "watch";
|
|
10
|
+
options: StudyCommandOptions;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* The scripted route's CLI setup: its open semantics and runStudyWith options, and how it presents the
|
|
14
|
+
* outcome. Undefined when watch-mode setup has already written its own result.
|
|
15
|
+
*/
|
|
16
|
+
export declare function scriptedRouteRun(args: ScriptedRouteArgs): RouteRun | undefined;
|
|
17
|
+
export {};
|
|
@@ -3,10 +3,10 @@ import { Command } from "commander";
|
|
|
3
3
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
4
4
|
import { wantsJson, writeResult } from "../io.js";
|
|
5
5
|
import { showObserver } from "../observer-follow.js";
|
|
6
|
-
import {
|
|
7
|
-
import { observerOpen, resolveRouteShouldOpen, watchFinishedPlan } from "./
|
|
6
|
+
import { formatScriptedStudyHuman } from "./study-format.js";
|
|
7
|
+
import { observerOpen, resolveRouteShouldOpen, watchFinishedPlan } from "./study-route-open.js";
|
|
8
8
|
/**
|
|
9
|
-
* The scripted route's CLI setup: its open semantics and
|
|
9
|
+
* The scripted route's CLI setup: its open semantics and runStudyWith options, and how it presents the
|
|
10
10
|
* outcome. Undefined when watch-mode setup has already written its own result.
|
|
11
11
|
*/
|
|
12
12
|
export function scriptedRouteRun(args) {
|
|
@@ -33,7 +33,7 @@ export function scriptedRouteRun(args) {
|
|
|
33
33
|
throw new Error(`Expected the scripted route, got ${outcome.route}.`);
|
|
34
34
|
}
|
|
35
35
|
const result = outcome.result;
|
|
36
|
-
writeResult(args.command, args.io, result,
|
|
36
|
+
writeResult(args.command, args.io, result, formatScriptedStudyHuman);
|
|
37
37
|
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
38
38
|
// Watch mode serves the Observer the route rendered through its finished run.
|
|
39
39
|
if (finishedPlan !== undefined && result.ok && result.observer !== undefined) {
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
|
-
import type {
|
|
3
|
-
import { type CliIo, type
|
|
4
|
-
import type { RouteRun } from "./
|
|
2
|
+
import type { StudyConfig } from "../../study/types.js";
|
|
3
|
+
import { type CliIo, type StudyCommandOptions } from "../io.js";
|
|
4
|
+
import type { RouteRun } from "./study-route-run.js";
|
|
5
5
|
interface SharedWorldRouteArgs {
|
|
6
6
|
command: Command;
|
|
7
7
|
io: CliIo;
|
|
8
|
-
config:
|
|
8
|
+
config: StudyConfig;
|
|
9
9
|
mode: "run" | "watch";
|
|
10
|
-
options:
|
|
10
|
+
options: StudyCommandOptions;
|
|
11
11
|
}
|
|
12
12
|
/**
|
|
13
13
|
* The shared-world route's CLI setup: dry run, open and follow semantics, the live Observer server
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { automaticAnalysisSucceeded } from "../../analysis/automatic-completion.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
|
-
import {
|
|
3
|
+
import { resolveStudyDryRun } from "../../study/plan.js";
|
|
4
4
|
import { serveObserver } from "../../observer/render.js";
|
|
5
5
|
import { redactText } from "../../evidence/redaction.js";
|
|
6
6
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
7
7
|
import { parseObserverPort, wantsJson, writeResult, } from "../io.js";
|
|
8
8
|
import { followObserver, showObserver, staticObserverOpen, withObserverServer, } from "../observer-follow.js";
|
|
9
|
-
import {
|
|
10
|
-
import { resolveRouteShouldOpen, watchFinishedPlan } from "./
|
|
11
|
-
import { rosterOf } from "../../
|
|
9
|
+
import { formatConcurrentSharedWorldStudyHuman } from "./study-format.js";
|
|
10
|
+
import { resolveRouteShouldOpen, watchFinishedPlan } from "./study-route-open.js";
|
|
11
|
+
import { rosterOf } from "../../study/parse/actors.js";
|
|
12
12
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
13
13
|
/**
|
|
14
14
|
* The shared-world route's CLI setup: dry run, open and follow semantics, the live Observer server
|
|
@@ -17,7 +17,7 @@ import { studyResultIdentity } from "../../run/study-result.js";
|
|
|
17
17
|
*/
|
|
18
18
|
export function sharedWorldRouteRun(args) {
|
|
19
19
|
const wantsMachine = wantsJson(args.command);
|
|
20
|
-
const dryRun =
|
|
20
|
+
const dryRun = resolveStudyDryRun(args.config, args.options.dryRun, true) ?? true;
|
|
21
21
|
const shouldOpen = resolveRouteShouldOpen({
|
|
22
22
|
optionOpen: args.options.open,
|
|
23
23
|
defaultsOpen: args.config.defaults?.open,
|
|
@@ -42,7 +42,7 @@ export function sharedWorldRouteRun(args) {
|
|
|
42
42
|
warnings: [],
|
|
43
43
|
error: { code: "HUMANISH_SHARED_WORLD_FAILED", message },
|
|
44
44
|
};
|
|
45
|
-
writeResult(args.command, args.io, result,
|
|
45
|
+
writeResult(args.command, args.io, result, formatConcurrentSharedWorldStudyHuman);
|
|
46
46
|
args.io.setExitCode(2);
|
|
47
47
|
};
|
|
48
48
|
if (wantsFollow && port === null) {
|
|
@@ -116,7 +116,7 @@ export function sharedWorldRouteRun(args) {
|
|
|
116
116
|
],
|
|
117
117
|
};
|
|
118
118
|
}
|
|
119
|
-
writeResult(args.command, args.io, output,
|
|
119
|
+
writeResult(args.command, args.io, output, formatConcurrentSharedWorldStudyHuman);
|
|
120
120
|
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
121
121
|
if (server && output.observer?.ok) {
|
|
122
122
|
await followObserver(args.io, output.observer, server);
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import type { StudyConfig } from "../../study/types.js";
|
|
3
|
+
import { type CliIo, type StudyCommandOptions } from "../io.js";
|
|
4
|
+
import type { RouteRun } from "./study-route-run.js";
|
|
5
|
+
interface TerminalRouteArgs {
|
|
6
|
+
command: Command;
|
|
7
|
+
io: CliIo;
|
|
8
|
+
config: StudyConfig;
|
|
9
|
+
mode: "run" | "watch";
|
|
10
|
+
options: StudyCommandOptions;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* The terminal route's CLI setup: its open semantics and runStudyWith options, and how it presents the
|
|
14
|
+
* outcome. Undefined when watch-mode setup has already written its own result.
|
|
15
|
+
*/
|
|
16
|
+
export declare function terminalRouteRun(args: TerminalRouteArgs): RouteRun | undefined;
|
|
17
|
+
export {};
|
|
@@ -3,10 +3,10 @@ import { Command } from "commander";
|
|
|
3
3
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
4
4
|
import { wantsJson, writeResult } from "../io.js";
|
|
5
5
|
import { showObserver } from "../observer-follow.js";
|
|
6
|
-
import {
|
|
7
|
-
import { observerOpen, resolveRouteShouldOpen, watchFinishedPlan } from "./
|
|
6
|
+
import { formatTerminalStudyHuman } from "./study-format.js";
|
|
7
|
+
import { observerOpen, resolveRouteShouldOpen, watchFinishedPlan } from "./study-route-open.js";
|
|
8
8
|
/**
|
|
9
|
-
* The terminal route's CLI setup: its open semantics and
|
|
9
|
+
* The terminal route's CLI setup: its open semantics and runStudyWith options, and how it presents the
|
|
10
10
|
* outcome. Undefined when watch-mode setup has already written its own result.
|
|
11
11
|
*/
|
|
12
12
|
export function terminalRouteRun(args) {
|
|
@@ -33,7 +33,7 @@ export function terminalRouteRun(args) {
|
|
|
33
33
|
throw new Error(`Expected the terminal route, got ${outcome.route}.`);
|
|
34
34
|
}
|
|
35
35
|
const result = outcome.result;
|
|
36
|
-
writeResult(args.command, args.io, result,
|
|
36
|
+
writeResult(args.command, args.io, result, formatTerminalStudyHuman);
|
|
37
37
|
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
38
38
|
if (finishedPlan !== undefined && result.ok && result.observer !== undefined) {
|
|
39
39
|
await showObserver({
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import { type CliIo, type StudyCommandOptions } from "../io.js";
|
|
3
|
+
export declare function runStudyCommand(args: {
|
|
4
|
+
command: Command;
|
|
5
|
+
io: CliIo;
|
|
6
|
+
study: string;
|
|
7
|
+
mode: "run" | "watch";
|
|
8
|
+
options: StudyCommandOptions;
|
|
9
|
+
}): Promise<void>;
|
|
@@ -2,26 +2,26 @@ import { automaticAnalysisBudget, formatAutomaticAnalysisBudget, } from "../../a
|
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
3
|
import { Command } from "commander";
|
|
4
4
|
import { deriveRunFacts } from "../telemetry.js";
|
|
5
|
-
import {
|
|
5
|
+
import { resolveStudyManifest } from "../../study/discover.js";
|
|
6
6
|
import { warnAndQueue } from "../deprecations.js";
|
|
7
|
-
import {
|
|
8
|
-
import { keyNamesOf } from "../../
|
|
9
|
-
import { routeOf } from "../../
|
|
10
|
-
import { computerUseRouteRun } from "./
|
|
11
|
-
import { runRoute } from "./
|
|
12
|
-
import { scriptedRouteRun } from "./
|
|
13
|
-
import { sharedWorldRouteRun } from "./
|
|
14
|
-
import { previewRouteRun } from "./
|
|
15
|
-
import { terminalRouteRun } from "./
|
|
16
|
-
import { maybeLoadAdapterScorer } from "./
|
|
7
|
+
import { planStudy, resolveStudyDryRun } from "../../study/plan.js";
|
|
8
|
+
import { keyNamesOf } from "../../study/requirements.js";
|
|
9
|
+
import { routeOf } from "../../study/plan.js";
|
|
10
|
+
import { computerUseRouteRun } from "./study-route-computer-use.js";
|
|
11
|
+
import { runRoute } from "./study-route-run.js";
|
|
12
|
+
import { scriptedRouteRun } from "./study-route-scripted.js";
|
|
13
|
+
import { sharedWorldRouteRun } from "./study-route-shared-world.js";
|
|
14
|
+
import { previewRouteRun } from "./study-route-preview.js";
|
|
15
|
+
import { terminalRouteRun } from "./study-route-terminal.js";
|
|
16
|
+
import { maybeLoadAdapterScorer } from "./study-scorer.js";
|
|
17
17
|
import { discoverCliKeys, formatRunHuman, noteRunFacts, writeResult, } from "../io.js";
|
|
18
18
|
import { watchExposeRequested } from "../observer-follow.js";
|
|
19
19
|
import { beginRunSignalPhase } from "./run-signals.js";
|
|
20
20
|
import { WATCH_SAFE_NOT_APPLICABLE_MESSAGE } from "../../observer/exposure.js";
|
|
21
|
-
export async function
|
|
22
|
-
const resolved = await
|
|
21
|
+
export async function runStudyCommand(args) {
|
|
22
|
+
const resolved = await resolveStudyManifest(args.options.cwd, args.study);
|
|
23
23
|
if (!resolved.ok) {
|
|
24
|
-
writeResult(args.command, args.io, resolved,
|
|
24
|
+
writeResult(args.command, args.io, resolved, formatStudyResolveFailureHuman);
|
|
25
25
|
args.io.setExitCode(2);
|
|
26
26
|
return;
|
|
27
27
|
}
|
|
@@ -33,14 +33,14 @@ export async function runLabCommand(args) {
|
|
|
33
33
|
const config = resolved.config;
|
|
34
34
|
// The run's identity: which manifest, where it lives, and whether it is committed, a
|
|
35
35
|
// local overlay, or an explicit path. Resolved once here (the only place that knows all three)
|
|
36
|
-
// and carried into the run's status record and bundle so the filesystem can answer "which
|
|
36
|
+
// and carried into the run's status record and bundle so the filesystem can answer "which study
|
|
37
37
|
// produced this run" without the old `persona.source = "lab:<id>"` string convention.
|
|
38
|
-
const
|
|
39
|
-
// Named here, once, for every route
|
|
40
|
-
//
|
|
41
|
-
noteRunFacts(args.command, deriveRunFacts({
|
|
38
|
+
const study = { id: config.id, path: resolved.path, origin: resolved.origin };
|
|
39
|
+
// Named here, once, for every route, so the starter study `first-run` is named in telemetry
|
|
40
|
+
// whatever its route's result carries.
|
|
41
|
+
noteRunFacts(args.command, deriveRunFacts({ studyId: config.id }));
|
|
42
42
|
const route = routeOf(config);
|
|
43
|
-
if (route !== "computer-use" &&
|
|
43
|
+
if (route !== "computer-use" && studyRerunFlagsRequested(args.options)) {
|
|
44
44
|
writeUnsupportedRerunFlagsResult(args, route);
|
|
45
45
|
return;
|
|
46
46
|
}
|
|
@@ -54,7 +54,7 @@ export async function runLabCommand(args) {
|
|
|
54
54
|
warnings: [],
|
|
55
55
|
error: {
|
|
56
56
|
code: "HUMANISH_WATCH_OPTION_CONFLICT",
|
|
57
|
-
message: `--expose/--tunnel/--oauth stream a live desktop and apply only to computer-use
|
|
57
|
+
message: `--expose/--tunnel/--oauth stream a live desktop and apply only to computer-use studies; this study runs on the ${route} route.`,
|
|
58
58
|
},
|
|
59
59
|
};
|
|
60
60
|
writeResult(args.command, args.io, result, formatRunHuman);
|
|
@@ -81,8 +81,8 @@ export async function runLabCommand(args) {
|
|
|
81
81
|
// Only a live run reads provider keys, so a dry run looks none up: no `gh auth token`, no e2b
|
|
82
82
|
// login, overlay or key store. A live run fills every key it finds and prints a line only for
|
|
83
83
|
// the keys its plan reads. This comes after the option refusals above, which read no key, and
|
|
84
|
-
// before the route's CLI setup and
|
|
85
|
-
if (
|
|
84
|
+
// before the route's CLI setup and prepareStudy, which do.
|
|
85
|
+
if (resolveStudyDryRun(config, args.options.dryRun, true) === false) {
|
|
86
86
|
const announced = announcedKeyNames(config, args.options);
|
|
87
87
|
await discoverCliKeys({
|
|
88
88
|
io: args.io,
|
|
@@ -90,15 +90,15 @@ export async function runLabCommand(args) {
|
|
|
90
90
|
...(announced === undefined ? {} : { announced }),
|
|
91
91
|
});
|
|
92
92
|
}
|
|
93
|
-
// The route's CLI setup refuses bad options,
|
|
93
|
+
// The route's CLI setup refuses bad options, runStudyWith's plan refuses bad labs, and the route's local
|
|
94
94
|
// checks (keys, runtime auth, subject env, the local agent, caps) refuse this machine, all before
|
|
95
95
|
// the scorer loads, so none imports the scorer's host code. The scripted route's checks and the
|
|
96
96
|
// run-id claim still come after it; the scripted route takes no scorer.
|
|
97
97
|
const routeRun = routeRunFor(route, { ...args, config });
|
|
98
98
|
if (routeRun === undefined)
|
|
99
99
|
return;
|
|
100
|
-
// Every route's
|
|
101
|
-
const run = { ...routeRun, options: { ...routeRun.options,
|
|
100
|
+
// Every route's runStudyWith options carry the study's provenance, from here only.
|
|
101
|
+
const run = { ...routeRun, options: { ...routeRun.options, study } };
|
|
102
102
|
// From here a signal marks the run interrupted and reclaims its sandboxes (run-signals.ts).
|
|
103
103
|
const signals = beginRunSignalPhase(args.io);
|
|
104
104
|
try {
|
|
@@ -126,12 +126,12 @@ export async function runLabCommand(args) {
|
|
|
126
126
|
}
|
|
127
127
|
const scorer = scorerLoad.scorer;
|
|
128
128
|
if (scorer) {
|
|
129
|
-
// Cross-repo guardrail: `humanish
|
|
129
|
+
// Cross-repo guardrail: `humanish run` import()s host JS named in the manifest.
|
|
130
130
|
// Surface it so the invoker (who may not be the manifest author) knows executable code ran.
|
|
131
131
|
args.io.writeErr(`warning: review scorer ${scorer.provenance.ref} (${scorer.provenance.source}) is code that humanish loaded and ran in this process. Review it as you would any code you run.\n`);
|
|
132
132
|
}
|
|
133
133
|
const analysisBudget = automaticAnalysisBudget(config.review?.analysis, route);
|
|
134
|
-
if (analysisBudget &&
|
|
134
|
+
if (analysisBudget && resolveStudyDryRun(config, args.options.dryRun, true) === false) {
|
|
135
135
|
args.io.writeErr(`${formatAutomaticAnalysisBudget(analysisBudget)}\n`);
|
|
136
136
|
}
|
|
137
137
|
return scorer === undefined ? {} : { scorer };
|
|
@@ -147,12 +147,12 @@ export async function runLabCommand(args) {
|
|
|
147
147
|
/**
|
|
148
148
|
* The filled keys a live run prints a line for: the ones its plan reads. Count and rerun options
|
|
149
149
|
* change who runs, not which keys. Undefined, so every fill is printed, when a scorer is declared,
|
|
150
|
-
* since its host code may read any key, and when the
|
|
150
|
+
* since its host code may read any key, and when the study does not plan.
|
|
151
151
|
*/
|
|
152
152
|
function announcedKeyNames(config, options) {
|
|
153
153
|
if (options.scorer !== undefined || config.review?.scorer !== undefined)
|
|
154
154
|
return undefined;
|
|
155
|
-
const planned =
|
|
155
|
+
const planned = planStudy(config, { cwd: options.cwd, dryRun: false });
|
|
156
156
|
return planned.ok ? keyNamesOf(planned.planned.plan) : undefined;
|
|
157
157
|
}
|
|
158
158
|
/** The route's CLI setup. Undefined when the setup has already written its own result. */
|
|
@@ -170,10 +170,10 @@ function routeRunFor(route, args) {
|
|
|
170
170
|
return sharedWorldRouteRun(args);
|
|
171
171
|
default:
|
|
172
172
|
// Compile-time exhaustiveness: a future route must be handled here, not silently no-op.
|
|
173
|
-
throw new Error(`Unhandled
|
|
173
|
+
throw new Error(`Unhandled study route: ${String(route)}`);
|
|
174
174
|
}
|
|
175
175
|
}
|
|
176
|
-
function
|
|
176
|
+
function studyRerunFlagsRequested(options) {
|
|
177
177
|
return options.rerunFailedFrom !== undefined || options.participants !== undefined;
|
|
178
178
|
}
|
|
179
179
|
function writeUnsupportedRerunFlagsResult(args, route) {
|
|
@@ -184,13 +184,13 @@ function writeUnsupportedRerunFlagsResult(args, route) {
|
|
|
184
184
|
warnings: [],
|
|
185
185
|
error: {
|
|
186
186
|
code: "HUMANISH_UNSUPPORTED_RERUN_FLAGS",
|
|
187
|
-
message: `--rerun-failed-from/--participants apply only to computer-use
|
|
187
|
+
message: `--rerun-failed-from/--participants apply only to computer-use studies with more than one participant; this study runs on the ${route} route.`,
|
|
188
188
|
},
|
|
189
189
|
};
|
|
190
190
|
writeResult(args.command, args.io, result, formatRunHuman);
|
|
191
191
|
args.io.setExitCode(2);
|
|
192
192
|
}
|
|
193
|
-
function
|
|
193
|
+
function formatStudyResolveFailureHuman(result) {
|
|
194
194
|
const warnings = result.warnings.map((warning) => `warning: ${warning}\n`).join("");
|
|
195
195
|
return { ...(warnings ? { stdout: warnings } : {}), error: result.error };
|
|
196
196
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { type AdapterScorerModule } from "../../
|
|
2
|
-
import type {
|
|
1
|
+
import { type AdapterScorerModule } from "../../study/adapter-scorer-loader.js";
|
|
2
|
+
import type { StudyRoute } from "../../study/plan.js";
|
|
3
3
|
import type { RunScorerProvenance } from "../../run/bundle.js";
|
|
4
|
-
import type {
|
|
4
|
+
import type { StudyConfig } from "../../study/types.js";
|
|
5
5
|
import type { RunResult } from "../../run/results.js";
|
|
6
6
|
/** A config-declared scorer that resolved + loaded fail-closed, ready to thread into a backend. */
|
|
7
7
|
export interface LoadedAdapterScorer {
|
|
@@ -16,8 +16,8 @@ export interface LoadedAdapterScorer {
|
|
|
16
16
|
*/
|
|
17
17
|
export declare function maybeLoadAdapterScorer(args: {
|
|
18
18
|
cwd: string;
|
|
19
|
-
config:
|
|
20
|
-
route:
|
|
19
|
+
config: StudyConfig;
|
|
20
|
+
route: StudyRoute;
|
|
21
21
|
flag: string | undefined;
|
|
22
22
|
}): Promise<{
|
|
23
23
|
ok: true;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { loadAdapterScorer } from "../../
|
|
1
|
+
import { loadAdapterScorer } from "../../study/adapter-scorer-loader.js";
|
|
2
2
|
/**
|
|
3
3
|
* Resolve `review.scorer.ref` (or the `--scorer` override) to a loaded adopter scorer, fail-closed
|
|
4
4
|
* (typed error) pre-spend. Precedence: CLI `--scorer` overrides the manifest; `source` records which
|