humanish 0.108.0 → 0.109.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +58 -59
- package/CHANGELOG.md +22 -0
- package/CONTEXT.md +44 -44
- package/README.md +3 -3
- package/TELEMETRY.md +2 -2
- package/dist/actors/codex/app-server-trace.js +2 -1
- package/dist/actors/codex/restricted-participant.js +12 -12
- package/dist/actors/codex/restricted-session.d.ts +2 -2
- package/dist/actors/computer-use/actor.d.ts +6 -6
- package/dist/actors/computer-use/actor.js +2 -2
- package/dist/actors/computer-use/admission-limit.d.ts +2 -2
- package/dist/actors/computer-use/admission-limit.js +3 -3
- package/dist/actors/computer-use/executor-error.d.ts +2 -2
- package/dist/actors/computer-use/executor-error.js +3 -3
- package/dist/actors/computer-use/loop/actions.js +5 -5
- package/dist/actors/computer-use/loop/debrief.js +2 -2
- package/dist/actors/computer-use/loop/ending.d.ts +1 -1
- package/dist/actors/computer-use/loop/ending.js +12 -11
- package/dist/actors/computer-use/loop/observation.js +6 -5
- package/dist/actors/computer-use/loop/provider-call.js +11 -11
- package/dist/actors/computer-use/loop/session.d.ts +4 -4
- package/dist/actors/computer-use/loop/session.js +2 -2
- package/dist/actors/computer-use/loop/trace.js +3 -2
- package/dist/actors/computer-use/loop/types.d.ts +24 -24
- package/dist/actors/computer-use/loop/usage.d.ts +2 -2
- package/dist/actors/computer-use/loop.d.ts +6 -1
- package/dist/actors/computer-use/loop.js +26 -4
- package/dist/actors/computer-use/openai-provider.d.ts +2 -2
- package/dist/actors/computer-use/openai-provider.js +8 -8
- package/dist/actors/computer-use/provider-error.d.ts +4 -4
- package/dist/actors/computer-use/provider-error.js +6 -6
- package/dist/actors/contract.d.ts +4 -4
- package/dist/actors/contract.js +1 -1
- package/dist/actors/local-agent/claude-session.js +3 -3
- package/dist/actors/local-agent/cli.d.ts +1 -1
- package/dist/actors/local-agent/cli.js +1 -1
- package/dist/actors/reasoning-effort.js +3 -3
- package/dist/actors/registry.d.ts +2 -2
- package/dist/actors/registry.js +3 -3
- package/dist/actors/scripted-browser/actor.d.ts +2 -2
- package/dist/actors/scripted-browser/actor.js +1 -1
- package/dist/analysis/automatic-completion.d.ts +7 -7
- package/dist/analysis/automatic-completion.js +1 -1
- package/dist/analysis/automatic-config.d.ts +6 -6
- package/dist/analysis/automatic-config.js +1 -1
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +1 -1
- package/dist/analysis/types.d.ts +6 -0
- package/dist/browser-control/client.js +16 -16
- package/dist/browser-control/dispatcher.js +4 -4
- package/dist/browser-control/protocol.js +14 -14
- package/dist/browser-control/recording-transfer.js +4 -4
- package/dist/browser-control/transport.js +7 -7
- package/dist/cli/commands/analysis-signals.d.ts +2 -2
- package/dist/cli/commands/comms.js +14 -22
- package/dist/cli/commands/migrate.js +15 -4
- package/dist/cli/commands/observe.d.ts +0 -1
- package/dist/cli/commands/observe.js +0 -15
- package/dist/cli/commands/run-command.d.ts +2 -2
- package/dist/cli/commands/run-command.js +3 -3
- package/dist/cli/commands/runs.js +9 -7
- package/dist/cli/commands/setup.js +5 -7
- package/dist/cli/commands/study-format.d.ts +9 -0
- package/dist/cli/commands/{lab-format.js → study-format.js} +5 -5
- package/dist/cli/commands/{lab-route-computer-use.d.ts → study-route-computer-use.d.ts} +6 -6
- package/dist/cli/commands/{lab-route-computer-use.js → study-route-computer-use.js} +12 -12
- package/dist/cli/commands/{lab-route-open.d.ts → study-route-open.d.ts} +4 -4
- package/dist/cli/commands/{lab-route-open.js → study-route-open.js} +2 -2
- package/dist/cli/commands/{lab-route-preview.d.ts → study-route-preview.d.ts} +7 -7
- package/dist/cli/commands/{lab-route-preview.js → study-route-preview.js} +3 -3
- package/dist/cli/commands/study-route-run.d.ts +19 -0
- package/dist/cli/commands/{lab-route-run.js → study-route-run.js} +6 -6
- package/dist/cli/commands/study-route-scripted.d.ts +17 -0
- package/dist/cli/commands/{lab-route-scripted.js → study-route-scripted.js} +4 -4
- package/dist/cli/commands/{lab-route-shared-world.d.ts → study-route-shared-world.d.ts} +5 -5
- package/dist/cli/commands/{lab-route-shared-world.js → study-route-shared-world.js} +7 -7
- package/dist/cli/commands/study-route-terminal.d.ts +17 -0
- package/dist/cli/commands/{lab-route-terminal.js → study-route-terminal.js} +4 -4
- package/dist/cli/commands/study-run.d.ts +9 -0
- package/dist/cli/commands/{lab-run.js → study-run.js} +34 -34
- package/dist/cli/commands/{lab-scorer.d.ts → study-scorer.d.ts} +5 -5
- package/dist/cli/commands/{lab-scorer.js → study-scorer.js} +1 -1
- package/dist/cli/commands/study.js +13 -49
- package/dist/cli/commands/tui.js +12 -12
- package/dist/cli/commands/watch.js +30 -65
- package/dist/cli/deprecations.d.ts +1 -13
- package/dist/cli/deprecations.js +0 -30
- package/dist/cli/doctor.d.ts +3 -3
- package/dist/cli/doctor.js +16 -16
- package/dist/cli/first-run-path.d.ts +4 -4
- package/dist/cli/first-run-path.js +7 -7
- package/dist/cli/io.d.ts +3 -3
- package/dist/cli/io.js +2 -2
- package/dist/cli/observer-follow.d.ts +2 -2
- package/dist/cli/observer-follow.js +2 -2
- package/dist/cli/orientation.d.ts +4 -4
- package/dist/cli/orientation.js +58 -20
- package/dist/cli/program.js +3 -4
- package/dist/cli/telemetry.d.ts +1 -1
- package/dist/cli/telemetry.js +14 -20
- package/dist/comms/catch-host.d.ts +1 -1
- package/dist/comms/catch-host.js +3 -3
- package/dist/comms/connections.d.ts +1 -1
- package/dist/comms/connections.js +2 -2
- package/dist/comms/external-evidence.d.ts +5 -5
- package/dist/comms/external-evidence.js +4 -3
- package/dist/comms/fake-inbox.d.ts +2 -2
- package/dist/comms/fake-inbox.js +2 -2
- package/dist/comms/receiving-runtime.d.ts +4 -4
- package/dist/comms/sandbox-catch.d.ts +2 -2
- package/dist/comms/sandbox-catch.js +2 -2
- package/dist/comms/setup.d.ts +2 -2
- package/dist/comms/setup.js +29 -32
- package/dist/evidence/redaction.d.ts +11 -0
- package/dist/evidence/redaction.js +43 -0
- package/dist/feedback/draft.js +2 -2
- package/dist/feedback/export-bundle.js +5 -4
- package/dist/feedback/export.js +3 -2
- package/dist/feedback/proof.d.ts +1 -1
- package/dist/feedback/proof.js +2 -2
- package/dist/guest/bootstrap.js +6 -6
- package/dist/guest/browser-tools.js +8 -8
- package/dist/guest/chromium-text.js +17 -17
- package/dist/guest/desktop-executor.d.ts +2 -2
- package/dist/guest/desktop-executor.js +20 -20
- package/dist/guest/desktop-keys.js +4 -4
- package/dist/guest/desktop-media.d.ts +2 -2
- package/dist/guest/desktop-media.js +16 -16
- package/dist/guest/desktop-native.js +10 -10
- package/dist/guest/runtime-desktop.js +2 -2
- package/dist/guest/runtime-processes.d.ts +2 -2
- package/dist/guest/runtime-processes.js +2 -2
- package/dist/guest/runtime.js +5 -5
- package/dist/index.d.ts +13 -15
- package/dist/index.js +6 -8
- package/dist/observer/artifact.d.ts +1 -1
- package/dist/observer/artifact.js +1 -1
- package/dist/observer/data.d.ts +1 -1
- package/dist/observer/library.d.ts +2 -0
- package/dist/observer/library.js +4 -1
- package/dist/observer/render.js +7 -1
- package/dist/observer/run-routes.d.ts +1 -0
- package/dist/observer/run-routes.js +37 -12
- package/dist/observer/serve.d.ts +1 -1
- package/dist/observer-app.html +1 -1
- package/dist/routes/computer-use/bundle-parts.js +3 -2
- package/dist/routes/computer-use/bundle.d.ts +2 -2
- package/dist/routes/computer-use/bundle.js +7 -6
- package/dist/routes/computer-use/e2b-desktop/comms.d.ts +5 -5
- package/dist/routes/computer-use/e2b-desktop/comms.js +5 -4
- package/dist/routes/computer-use/e2b-desktop/fidelity.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.js +3 -3
- package/dist/routes/computer-use/e2b-desktop/start.js +3 -4
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +2 -2
- package/dist/routes/computer-use/e2b-desktop/teardown.js +2 -3
- package/dist/routes/computer-use/external-comms.d.ts +3 -3
- package/dist/routes/computer-use/fanout-bundle.js +11 -10
- package/dist/routes/computer-use/fanout-records.js +2 -2
- package/dist/routes/computer-use/in-process-desktop.js +1 -1
- package/dist/routes/computer-use/live-flush.d.ts +3 -3
- package/dist/routes/computer-use/live-phase.d.ts +2 -3
- package/dist/routes/computer-use/live-phase.js +3 -3
- package/dist/routes/computer-use/local-tree-pack.d.ts +5 -5
- package/dist/routes/computer-use/local-tree-pack.js +1 -1
- package/dist/routes/computer-use/local-vm.d.ts +8 -8
- package/dist/routes/computer-use/local-vm.js +2 -2
- package/dist/routes/computer-use/participant-desktop.d.ts +3 -3
- package/dist/routes/computer-use/participant-desktop.js +1 -1
- package/dist/routes/computer-use/participant-execution.js +1 -1
- package/dist/routes/computer-use/participant-model.js +3 -3
- package/dist/routes/computer-use/participant-prompt.d.ts +5 -5
- package/dist/routes/computer-use/participant-prompt.js +3 -3
- package/dist/routes/computer-use/participant-runs.d.ts +10 -10
- package/dist/routes/computer-use/participant-runs.js +14 -13
- package/dist/routes/computer-use/plan.d.ts +12 -12
- package/dist/routes/computer-use/plan.js +27 -27
- package/dist/routes/computer-use/preflight.d.ts +5 -5
- package/dist/routes/computer-use/preflight.js +5 -5
- package/dist/routes/computer-use/rerun-selection.d.ts +1 -1
- package/dist/routes/computer-use/rerun-selection.js +3 -2
- package/dist/routes/computer-use/result.d.ts +4 -4
- package/dist/routes/computer-use/result.js +13 -12
- package/dist/routes/computer-use/route.d.ts +12 -12
- package/dist/routes/computer-use/route.js +19 -19
- package/dist/routes/computer-use/self-report.js +1 -1
- package/dist/routes/computer-use/setup.d.ts +22 -22
- package/dist/routes/computer-use/setup.js +15 -15
- package/dist/routes/computer-use/single-bundle.d.ts +4 -4
- package/dist/routes/computer-use/single-bundle.js +14 -13
- package/dist/routes/computer-use/subject-projection.d.ts +3 -3
- package/dist/routes/computer-use/types.d.ts +34 -34
- package/dist/routes/computer-use/types.js +3 -3
- package/dist/routes/preview.d.ts +6 -6
- package/dist/routes/preview.js +5 -5
- package/dist/routes/scripted/bundle.d.ts +6 -6
- package/dist/routes/scripted/bundle.js +10 -9
- package/dist/routes/scripted/plan.d.ts +7 -7
- package/dist/routes/scripted/plan.js +6 -6
- package/dist/routes/scripted/result.d.ts +3 -3
- package/dist/routes/scripted/result.js +9 -9
- package/dist/routes/scripted/route.d.ts +9 -9
- package/dist/routes/scripted/route.js +11 -11
- package/dist/routes/scripted/scenario.d.ts +1 -1
- package/dist/routes/scripted/scenario.js +3 -3
- package/dist/routes/scripted/setup.d.ts +6 -6
- package/dist/routes/scripted/setup.js +4 -4
- package/dist/routes/scripted/subject.d.ts +5 -5
- package/dist/routes/scripted/subject.js +3 -3
- package/dist/routes/scripted/surface-records.d.ts +1 -1
- package/dist/routes/scripted/surface-records.js +2 -2
- package/dist/routes/scripted/surface-sessions.d.ts +3 -3
- package/dist/routes/scripted/types.d.ts +17 -17
- package/dist/routes/scripted/types.js +1 -1
- package/dist/routes/shared-world/bundle-records.js +3 -3
- package/dist/routes/shared-world/bundle.js +15 -14
- package/dist/routes/shared-world/checkpoints.d.ts +3 -3
- package/dist/routes/shared-world/checkpoints.js +1 -1
- package/dist/routes/shared-world/comms.d.ts +9 -9
- package/dist/routes/shared-world/comms.js +6 -5
- package/dist/routes/shared-world/participant-specs.d.ts +3 -3
- package/dist/routes/shared-world/participant-specs.js +7 -7
- package/dist/routes/shared-world/plan.d.ts +9 -9
- package/dist/routes/shared-world/plan.js +9 -9
- package/dist/routes/shared-world/provisioned.d.ts +8 -8
- package/dist/routes/shared-world/provisioned.js +4 -4
- package/dist/routes/shared-world/result.d.ts +6 -6
- package/dist/routes/shared-world/result.js +11 -10
- package/dist/routes/shared-world/route.d.ts +11 -11
- package/dist/routes/shared-world/route.js +19 -19
- package/dist/routes/shared-world/setup.d.ts +13 -13
- package/dist/routes/shared-world/setup.js +19 -19
- package/dist/routes/shared-world/types.d.ts +27 -27
- package/dist/routes/terminal/adapter.d.ts +2 -2
- package/dist/routes/terminal/adapter.js +4 -4
- package/dist/routes/terminal/bundle.d.ts +11 -11
- package/dist/routes/terminal/bundle.js +12 -12
- package/dist/routes/terminal/credentials.d.ts +4 -4
- package/dist/routes/terminal/credentials.js +5 -5
- package/dist/routes/terminal/dry-run.d.ts +4 -4
- package/dist/routes/terminal/dry-run.js +9 -9
- package/dist/routes/terminal/ledger.d.ts +2 -2
- package/dist/routes/terminal/lifetime.d.ts +1 -1
- package/dist/routes/terminal/lifetime.js +1 -1
- package/dist/routes/terminal/live-finish.d.ts +2 -2
- package/dist/routes/terminal/live-finish.js +7 -7
- package/dist/routes/terminal/live-sandbox.d.ts +2 -2
- package/dist/routes/terminal/live-sandbox.js +3 -2
- package/dist/routes/terminal/participant-text.js +2 -1
- package/dist/routes/terminal/persona.d.ts +2 -2
- package/dist/routes/terminal/persona.js +2 -2
- package/dist/routes/terminal/plan.d.ts +7 -7
- package/dist/routes/terminal/plan.js +5 -5
- package/dist/routes/terminal/result.d.ts +5 -5
- package/dist/routes/terminal/result.js +7 -7
- package/dist/routes/terminal/route.d.ts +9 -9
- package/dist/routes/terminal/route.js +14 -14
- package/dist/routes/terminal/session.d.ts +5 -5
- package/dist/routes/terminal/session.js +7 -7
- package/dist/routes/terminal/trace.d.ts +2 -2
- package/dist/routes/terminal/types.d.ts +24 -24
- package/dist/routes/terminal/types.js +2 -2
- package/dist/run/bundle.d.ts +7 -7
- package/dist/run/bundle.js +2 -2
- package/dist/run/costs.d.ts +23 -0
- package/dist/run/costs.js +72 -43
- package/dist/run/dry-run-inputs.d.ts +1 -1
- package/dist/run/dry-run-inputs.js +1 -1
- package/dist/run/dry-run.js +2 -2
- package/dist/run/outcomes.d.ts +1 -1
- package/dist/run/participant.d.ts +1 -1
- package/dist/run/preflight-receipts.js +1 -1
- package/dist/run/pricing.js +2 -2
- package/dist/run/projection.d.ts +32 -32
- package/dist/run/projection.js +44 -44
- package/dist/run/reclaim.d.ts +1 -1
- package/dist/run/reclaim.js +1 -1
- package/dist/run/results.d.ts +2 -2
- package/dist/run/run-cost.d.ts +53 -0
- package/dist/run/run-cost.js +105 -0
- package/dist/run/run-index-analysis.d.ts +15 -0
- package/dist/run/run-index-analysis.js +77 -0
- package/dist/run/run-index.d.ts +11 -1
- package/dist/run/run-index.js +63 -49
- package/dist/run/run.d.ts +3 -3
- package/dist/run/run.js +2 -2
- package/dist/run/shared-world-evidence.d.ts +1 -1
- package/dist/run/shared-world-shape.js +1 -1
- package/dist/run/stats.d.ts +3 -3
- package/dist/run/stats.js +14 -14
- package/dist/run/status.d.ts +10 -2
- package/dist/run/status.js +6 -3
- package/dist/run/streams.d.ts +2 -2
- package/dist/run/study-provenance.d.ts +4 -2
- package/dist/run/study-provenance.js +7 -4
- package/dist/run/study-result.d.ts +0 -5
- package/dist/run/study-result.js +1 -1
- package/dist/run-study.d.ts +112 -0
- package/dist/{run-lab.js → run-study.js} +52 -45
- package/dist/{lab → study}/adapter-extension.d.ts +8 -20
- package/dist/{lab → study}/adapter-extension.js +2 -14
- package/dist/{lab → study}/adapter-scorer-loader.d.ts +10 -10
- package/dist/{lab → study}/adapter-scorer-loader.js +3 -3
- package/dist/study/composition-rules.d.ts +7 -0
- package/dist/{lab → study}/composition-rules.js +12 -12
- package/dist/study/config.d.ts +15 -0
- package/dist/{lab → study}/config.js +37 -20
- package/dist/study/convert.d.ts +1 -1
- package/dist/study/convert.js +15 -15
- package/dist/{lab → study}/device-presets.d.ts +2 -2
- package/dist/{lab → study}/discover.d.ts +29 -23
- package/dist/{lab → study}/discover.js +97 -118
- package/dist/{lab → study}/doctor.d.ts +11 -11
- package/dist/{lab → study}/doctor.js +18 -18
- package/dist/study/files.d.ts +2 -2
- package/dist/study/files.js +6 -5
- package/dist/{lab → study}/init-templates.d.ts +2 -2
- package/dist/{lab → study}/init-templates.js +16 -11
- package/dist/{lab → study}/init.d.ts +1 -1
- package/dist/{lab → study}/init.js +7 -4
- package/dist/{lab → study}/keys.d.ts +2 -2
- package/dist/{lab → study}/keys.js +10 -10
- package/dist/study/migrate.d.ts +8 -3
- package/dist/study/migrate.js +60 -4
- package/dist/{lab → study}/parse/actors.d.ts +5 -5
- package/dist/{lab → study}/parse/actors.js +5 -5
- package/dist/study/parse/comms.d.ts +12 -0
- package/dist/study/parse/execution.d.ts +19 -0
- package/dist/{lab → study}/parse/study-v3.d.ts +3 -5
- package/dist/{lab → study}/parse/study-v3.js +3 -3
- package/dist/{lab → study}/parse/subject-state.d.ts +8 -8
- package/dist/{lab → study}/parse/subject-state.js +4 -4
- package/dist/{lab → study}/parse/subject.d.ts +3 -3
- package/dist/{lab → study}/parse/subject.js +8 -8
- package/dist/{lab → study}/parse/values.d.ts +2 -2
- package/dist/{lab → study}/persona-resolve.d.ts +4 -4
- package/dist/{lab → study}/persona-resolve.js +2 -2
- package/dist/{lab → study}/plan-base.d.ts +7 -7
- package/dist/{lab → study}/plan-base.js +1 -1
- package/dist/{lab → study}/plan-participants.d.ts +10 -10
- package/dist/{lab → study}/plan-participants.js +4 -4
- package/dist/{lab → study}/plan-types.d.ts +28 -28
- package/dist/{lab → study}/plan-types.js +2 -2
- package/dist/study/plan.d.ts +13 -0
- package/dist/{lab → study}/plan.js +20 -20
- package/dist/study/preflight-probes.d.ts +3 -0
- package/dist/{lab → study}/preflight-probes.js +9 -8
- package/dist/study/preflight-result.d.ts +8 -0
- package/dist/{lab → study}/preflight-result.js +3 -3
- package/dist/{lab → study}/preflight.d.ts +31 -31
- package/dist/{lab → study}/preflight.js +8 -8
- package/dist/{lab → study}/requirements.d.ts +3 -3
- package/dist/{lab → study}/requirements.js +1 -1
- package/dist/study/route-inputs.d.ts +28 -0
- package/dist/{lab → study}/route-inputs.js +1 -1
- package/dist/{lab → study}/routing.d.ts +12 -12
- package/dist/{lab/run-lab-events.d.ts → study/run-study-events.d.ts} +10 -10
- package/dist/{lab/run-lab-events.js → study/run-study-events.js} +4 -4
- package/dist/{lab/run-lab-homes.d.ts → study/run-study-homes.d.ts} +8 -17
- package/dist/study/run-study-homes.js +3 -0
- package/dist/study/run-study-options.d.ts +28 -0
- package/dist/{lab/run-lab-options.js → study/run-study-options.js} +16 -15
- package/dist/{lab/lab-deps.d.ts → study/study-deps.d.ts} +2 -2
- package/dist/{lab/lab-deps.js → study/study-deps.js} +2 -2
- package/dist/{lab → study}/summary.d.ts +10 -10
- package/dist/{lab → study}/summary.js +15 -15
- package/dist/{lab → study}/tasks.d.ts +3 -3
- package/dist/{lab → study}/tasks.js +1 -1
- package/dist/{lab → study}/types.d.ts +104 -104
- package/dist/{lab → study}/types.js +1 -1
- package/dist/{lab → study}/validation.d.ts +19 -19
- package/dist/{lab → study}/validation.js +13 -13
- package/dist/{lab → study}/warnings.d.ts +4 -4
- package/dist/{lab → study}/warnings.js +1 -1
- package/dist/subject/clone.d.ts +6 -6
- package/dist/subject/clone.js +2 -2
- package/dist/subject/local-tree-archive.d.ts +2 -2
- package/dist/subject/local-tree-archive.js +3 -3
- package/dist/subject/local-tree.d.ts +3 -3
- package/dist/subject/runtime.js +2 -2
- package/dist/subject/serve.d.ts +4 -4
- package/dist/subject/state.d.ts +3 -3
- package/dist/subject/state.js +1 -1
- package/dist/subject/steps.d.ts +1 -1
- package/dist/subject/steps.js +1 -1
- package/dist/substrates/desktop-session.js +2 -2
- package/dist/substrates/detached.d.ts +1 -1
- package/dist/substrates/e2b/desktop-browser.d.ts +4 -4
- package/dist/substrates/e2b/desktop-browser.js +1 -1
- package/dist/substrates/e2b/desktop-cdp.d.ts +1 -1
- package/dist/substrates/e2b/desktop-cdp.js +1 -1
- package/dist/substrates/e2b/desktop-executor.d.ts +1 -1
- package/dist/substrates/e2b/desktop-executor.js +12 -12
- package/dist/substrates/e2b/desktop-geometry.d.ts +1 -1
- package/dist/substrates/e2b/desktop-geometry.js +1 -1
- package/dist/substrates/e2b/desktop-media.d.ts +3 -3
- package/dist/substrates/e2b/lifetime.js +1 -1
- package/dist/substrates/e2b/sandbox.d.ts +3 -3
- package/dist/substrates/e2b/sandbox.js +1 -1
- package/dist/substrates/e2b/sdk.d.ts +1 -1
- package/dist/substrates/e2b/sdk.js +1 -1
- package/dist/substrates/local/runtime-config.d.ts +4 -4
- package/dist/substrates/local/runtime-config.js +4 -4
- package/dist/tui/contract.d.ts +8 -8
- package/dist/tui/contract.js +2 -2
- package/dist/tui/launch.d.ts +3 -3
- package/dist/tui/launch.js +22 -16
- package/dist/tui-app.js +139 -139
- package/dist/verify/subject.d.ts +1 -1
- package/dist/verify/subject.js +1 -1
- package/dist/verify/verify.js +2 -1
- package/docs/architecture/actor-contract.md +5 -5
- package/docs/architecture/browser-control.md +1 -1
- package/docs/architecture/external-public-shared-world.md +8 -7
- package/docs/architecture/github-feedback-loop.md +1 -1
- package/docs/architecture/task-protocol-support.md +1 -1
- package/docs/architecture/terminal-product-route.md +18 -18
- package/docs/contracts/adapter-fixtures.md +2 -2
- package/docs/contracts/errors.md +8 -8
- package/docs/contracts/run-bundle.md +6 -4
- package/docs/contracts/schemas.md +61 -42
- package/docs/contracts/study-analysis.md +4 -1
- package/docs/contracts/study-costs.md +17 -2
- package/examples/README.md +1 -1
- package/examples/scorer/scorer.mjs +3 -3
- package/package.json +2 -1
- package/skills/humanish/SKILL.md +8 -6
- package/skills/humanish/agents/openai.yaml +1 -1
- package/dist/cli/commands/lab-format.d.ts +0 -9
- package/dist/cli/commands/lab-route-run.d.ts +0 -19
- package/dist/cli/commands/lab-route-scripted.d.ts +0 -17
- package/dist/cli/commands/lab-route-terminal.d.ts +0 -17
- package/dist/cli/commands/lab-run.d.ts +0 -9
- package/dist/lab/composition-rules.d.ts +0 -7
- package/dist/lab/config.d.ts +0 -7
- package/dist/lab/deprecated-fields.d.ts +0 -10
- package/dist/lab/deprecated-fields.js +0 -24
- package/dist/lab/parse/comms.d.ts +0 -12
- package/dist/lab/parse/execution.d.ts +0 -19
- package/dist/lab/plan.d.ts +0 -13
- package/dist/lab/preflight-probes.d.ts +0 -3
- package/dist/lab/preflight-result.d.ts +0 -8
- package/dist/lab/route-inputs.d.ts +0 -28
- package/dist/lab/run-lab-homes.js +0 -7
- package/dist/lab/run-lab-options.d.ts +0 -28
- package/dist/library-aliases.d.ts +0 -56
- package/dist/library-aliases.js +0 -22
- package/dist/run-lab.d.ts +0 -112
- /package/dist/{lab → study}/device-presets.js +0 -0
- /package/dist/{lab → study}/init-paths.d.ts +0 -0
- /package/dist/{lab → study}/init-paths.js +0 -0
- /package/dist/{lab → study}/init-plan.d.ts +0 -0
- /package/dist/{lab → study}/init-plan.js +0 -0
- /package/dist/{lab → study}/parse/comms.js +0 -0
- /package/dist/{lab → study}/parse/execution.js +0 -0
- /package/dist/{lab → study}/parse/values.js +0 -0
- /package/dist/{lab → study}/participant-assignment.d.ts +0 -0
- /package/dist/{lab → study}/participant-assignment.js +0 -0
- /package/dist/{lab → study}/persona.d.ts +0 -0
- /package/dist/{lab → study}/persona.js +0 -0
- /package/dist/{lab → study}/routing.js +0 -0
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// finished session. The desktop side stays with the participant runner.
|
|
4
4
|
import { startClaudeSession } from "../../actors/local-agent/claude-session.js";
|
|
5
5
|
import { createLocalAgentProvider } from "../../actors/local-agent/cli.js";
|
|
6
|
-
import { pricedModel } from "../../
|
|
6
|
+
import { pricedModel } from "../../study/plan-base.js";
|
|
7
7
|
import { estimateActorCostForExecution, round6 } from "../../run/pricing.js";
|
|
8
8
|
import { redactText } from "../../evidence/redaction.js";
|
|
9
9
|
import { createRestrictedCodexParticipant, } from "../../actors/codex/restricted-participant.js";
|
|
@@ -54,7 +54,7 @@ export async function startParticipantModel(spec, deps, executor) {
|
|
|
54
54
|
// started with no memory of the last. HUMANISH_LOCAL_AGENT_ONE_SHOT=1 keeps that path
|
|
55
55
|
// reachable as a measurement switch: MemTrapBench (2026-08) reports memory frameworks
|
|
56
56
|
// degrading agent performance by 10-40% on some tasks, so "remembers" has to be measured
|
|
57
|
-
// against "does not" on the same
|
|
57
|
+
// against "does not" on the same study, not assumed. The trace records which one ran.
|
|
58
58
|
const oneShot = env.HUMANISH_LOCAL_AGENT_ONE_SHOT !== undefined &&
|
|
59
59
|
env.HUMANISH_LOCAL_AGENT_ONE_SHOT !== "" &&
|
|
60
60
|
env.HUMANISH_LOCAL_AGENT_ONE_SHOT !== "0";
|
|
@@ -208,7 +208,7 @@ export async function closeParticipantModel(model, warnings) {
|
|
|
208
208
|
/** Prices the finished session's tokens onto its trace, writes the trace, and warns on raw screenshots. */
|
|
209
209
|
export async function recordParticipantTrace(spec, deps, session, warnings) {
|
|
210
210
|
// Per-participant model-token cost estimate, attached to the trace before it is persisted (the model
|
|
211
|
-
// id is authoritative here: provider.version). Kept at the
|
|
211
|
+
// id is authoritative here: provider.version). Kept at the study boundary so the pure loop
|
|
212
212
|
// never depends on the operator rate table. estimateActorCost declares absent (null) for an
|
|
213
213
|
// unknown rate / missing usage rather than guessing.
|
|
214
214
|
session.trace.estimatedCost = estimateActorCostForExecution(session.trace.tokenUsage, session.trace.ids.model, session.trace.executionProfile);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ActorPersonaRef } from "../../actors/contract.js";
|
|
2
|
-
import type { DevicePreset } from "../../
|
|
3
|
-
import { type ResolvedPersona } from "../../
|
|
4
|
-
import { type
|
|
2
|
+
import type { DevicePreset } from "../../study/device-presets.js";
|
|
3
|
+
import { type ResolvedPersona } from "../../study/persona.js";
|
|
4
|
+
import { type StudyTask } from "../../study/tasks.js";
|
|
5
5
|
import type { DesktopParticipantRun } from "./types.js";
|
|
6
6
|
export declare const DEFAULT_MISSION = "You are testing a web application. The browser is already open at the subject URL. Explore it, accomplish what the scenario asks, and stop when done.";
|
|
7
7
|
/**
|
|
@@ -21,9 +21,9 @@ export declare function composeParticipantInstructions(args: {
|
|
|
21
21
|
mission: string;
|
|
22
22
|
persona?: string;
|
|
23
23
|
instruction?: string;
|
|
24
|
-
/** The
|
|
24
|
+
/** The study's declared protocol. Only the participant-facing `goal` halves are rendered
|
|
25
25
|
* into the prompt; the `success` criteria never appear here. */
|
|
26
|
-
tasks?: readonly
|
|
26
|
+
tasks?: readonly StudyTask[];
|
|
27
27
|
device: {
|
|
28
28
|
name: string;
|
|
29
29
|
preset: DevicePreset;
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
// the email inbox at run time. Shared by the computer-use and concurrent shared-world routes.
|
|
4
4
|
import { recipientInboxUrl } from "../../comms/capture-surface.js";
|
|
5
5
|
import { digestText } from "../../evidence/redaction.js";
|
|
6
|
-
import { personaBrief, personaToDirectives, renderPersonaPromptSection, } from "../../
|
|
7
|
-
import { renderTaskPrompt } from "../../
|
|
6
|
+
import { personaBrief, personaToDirectives, renderPersonaPromptSection, } from "../../study/persona.js";
|
|
7
|
+
import { renderTaskPrompt } from "../../study/tasks.js";
|
|
8
8
|
export const DEFAULT_MISSION = "You are testing a web application. The browser is already open at the subject URL. Explore it, accomplish what the scenario asks, and stop when done.";
|
|
9
9
|
/**
|
|
10
10
|
* The participant's outcome as one fixed first line of its last message. The
|
|
@@ -28,7 +28,7 @@ export function composeParticipantInstructions(args) {
|
|
|
28
28
|
: `You are a desktop user (${name}, ${preset.width}x${preset.height}).`;
|
|
29
29
|
// The protocol as the participant reads it: numbered goals, nothing else. The success criteria
|
|
30
30
|
// are the researcher's instrument and must never reach this prompt: a persona told how it will
|
|
31
|
-
// be measured optimizes for the measurement instead of using the product (src/
|
|
31
|
+
// be measured optimizes for the measurement instead of using the product (src/study/tasks.ts).
|
|
32
32
|
const taskLines = renderTaskPrompt(args.tasks ?? []);
|
|
33
33
|
// A resolved persona contributes its compiled directives (friction tolerance, skill bias,
|
|
34
34
|
// accessibility behavior, constraints) through the same persona.ts compiler the terminal route
|
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
import type { ComputerUsePlan } from "../../
|
|
2
|
-
import { type
|
|
3
|
-
import { type ResolvedPersona } from "../../
|
|
1
|
+
import type { ComputerUsePlan } from "../../study/plan-types.js";
|
|
2
|
+
import { type StudyConfig } from "../../study/types.js";
|
|
3
|
+
import { type ResolvedPersona } from "../../study/persona.js";
|
|
4
4
|
import { type RunRerunLineage } from "../../run/bundle.js";
|
|
5
5
|
import type { PreparedSelectedOutputDirectory } from "../../run/contained-output.js";
|
|
6
|
-
import { type
|
|
7
|
-
export declare function defaultSessionTimeoutMs(config:
|
|
6
|
+
import { type CuaActorStudyErrorCode, type CuaParticipantPlan, type CuaParticipantPlanEntry, type DesktopParticipantRun } from "./types.js";
|
|
7
|
+
export declare function defaultSessionTimeoutMs(config: StudyConfig): number;
|
|
8
8
|
/** Per-participant sandbox deadline (each one owns its own desktop). Mirrors the
|
|
9
9
|
* single-participant formula verbatim so N=1 stays byte-stable: explicit sandboxTimeoutMs, else session budget + (clone
|
|
10
10
|
* or local-tree: provision budget + Σ state-step budgets) + the server-side
|
|
11
11
|
* reclamation buffer. Local-tree shares the clone route's provisioning budget: it swaps a
|
|
12
12
|
* git clone for an upload+extract, but the shared install/build/state/start/probe pipeline
|
|
13
13
|
* costs the same wall-clock room either way. */
|
|
14
|
-
export declare function resolveParticipantSandboxMs(config:
|
|
14
|
+
export declare function resolveParticipantSandboxMs(config: StudyConfig): number;
|
|
15
15
|
/**
|
|
16
|
-
* The in-flight participant bound a
|
|
16
|
+
* The in-flight participant bound a study declares. Defaults to the participant count: every
|
|
17
17
|
* declared participant runs at once, because a throttle nobody asked for silently turns "N actors
|
|
18
18
|
* live" into waves; total session count and spend are the same either way, only wall-clock
|
|
19
19
|
* and simultaneity differ. A declared execution.concurrency is a cap, clamped to [1, participants].
|
|
@@ -26,7 +26,7 @@ export declare function boundedConcurrency(declared: number | undefined, partici
|
|
|
26
26
|
* sandbox-minutes, before any sandbox or provider call. The same plan appears in dry-run,
|
|
27
27
|
* marked $0 (dryRun: true).
|
|
28
28
|
*/
|
|
29
|
-
export declare function resolveCuaParticipantPlan(config:
|
|
29
|
+
export declare function resolveCuaParticipantPlan(config: StudyConfig, opts?: {
|
|
30
30
|
countOverride?: number;
|
|
31
31
|
env?: Record<string, string | undefined>;
|
|
32
32
|
dryRun?: boolean;
|
|
@@ -34,7 +34,7 @@ export declare function resolveCuaParticipantPlan(config: LabConfig, opts?: {
|
|
|
34
34
|
}): CuaParticipantPlan;
|
|
35
35
|
/** Print the participant plan to stderr before any sandbox/provider call (public-safe: ids, devices,
|
|
36
36
|
* digests, and budgets only; no prompt text, no secrets). */
|
|
37
|
-
export declare function emitPreflightPlan(participantPlan: CuaParticipantPlan,
|
|
37
|
+
export declare function emitPreflightPlan(participantPlan: CuaParticipantPlan, studyId: string): void;
|
|
38
38
|
export declare function formatParticipantPlanEntry(entry: CuaParticipantPlanEntry): string;
|
|
39
39
|
/**
|
|
40
40
|
* Compile the committed persona files the participants use, printing a warning for each one that
|
|
@@ -60,7 +60,7 @@ export declare function loadCuaParticipants(args: {
|
|
|
60
60
|
rerunLineage?: RunRerunLineage;
|
|
61
61
|
} | {
|
|
62
62
|
ok: false;
|
|
63
|
-
code:
|
|
63
|
+
code: CuaActorStudyErrorCode;
|
|
64
64
|
message: string;
|
|
65
65
|
}>;
|
|
66
66
|
/** Scrub known secret values from each participant's declarative snapshot before any bundle uses it. */
|
|
@@ -1,23 +1,24 @@
|
|
|
1
|
-
// The participant runs a computer-use
|
|
1
|
+
// The participant runs a computer-use study drives, built before anything is created: the sandbox
|
|
2
2
|
// and session budgets, the concurrency bound, each participant's prompt, bundle ids and artifact
|
|
3
3
|
// paths, and the public plan printed before any sandbox or provider call. loadCuaParticipants
|
|
4
4
|
// compiles the committed personas first and narrows a rerun to its selected participants.
|
|
5
|
-
import {
|
|
5
|
+
import { isLocalBrowserStudy, LOCAL_BROWSER_LIFETIME_MS, } from "../../substrates/local/runtime-config.js";
|
|
6
6
|
import { DEFAULT_STATE_STEP_TIMEOUT_MS } from "../../subject/state.js";
|
|
7
|
-
import { computerUseParticipants, } from "../../
|
|
7
|
+
import { computerUseParticipants, } from "../../study/plan-participants.js";
|
|
8
8
|
import { resolveParticipant } from "../../run/participant.js";
|
|
9
|
-
import {} from "../../
|
|
10
|
-
import { scrubPersonaBrief } from "../../
|
|
9
|
+
import {} from "../../study/types.js";
|
|
10
|
+
import { scrubPersonaBrief } from "../../study/persona.js";
|
|
11
11
|
import { redactText } from "../../evidence/redaction.js";
|
|
12
12
|
import {} from "../../run/bundle.js";
|
|
13
|
-
import { participantAssignment } from "../../
|
|
14
|
-
import { resolveCommittedPersonas } from "../../
|
|
13
|
+
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
14
|
+
import { resolveCommittedPersonas } from "../../study/persona-resolve.js";
|
|
15
15
|
import { CUA_FANOUT_STRATEGY, CUA_MAX_CONCURRENCY_ENV, DEFAULT_APP_URL_SESSION_TIMEOUT_MS, MIN_DERIVED_SESSION_TIMEOUT_MS, } from "./types.js";
|
|
16
16
|
import { MAX_SANDBOX_MS, SANDBOX_TIMEOUT_BUFFER_MS, SUBJECT_PROVISION_BUDGET_MS, } from "../../substrates/e2b/lifetime.js";
|
|
17
|
-
import { readPositiveInt } from "../../
|
|
17
|
+
import { readPositiveInt } from "../../study/parse/values.js";
|
|
18
18
|
import { digestUrl } from "./bundle-parts.js";
|
|
19
19
|
import { composeParticipantInstructions, DEFAULT_MISSION } from "./participant-prompt.js";
|
|
20
20
|
import { resolveCuaRerunSelection } from "./rerun-selection.js";
|
|
21
|
+
import { plural } from "../../run/text.js";
|
|
21
22
|
export function defaultSessionTimeoutMs(config) {
|
|
22
23
|
const provisionedRoute = config.subject.source === "clone" || config.subject.source === "local-tree";
|
|
23
24
|
if (!provisionedRoute)
|
|
@@ -33,7 +34,7 @@ export function defaultSessionTimeoutMs(config) {
|
|
|
33
34
|
* git clone for an upload+extract, but the shared install/build/state/start/probe pipeline
|
|
34
35
|
* costs the same wall-clock room either way. */
|
|
35
36
|
export function resolveParticipantSandboxMs(config) {
|
|
36
|
-
if (
|
|
37
|
+
if (isLocalBrowserStudy(config))
|
|
37
38
|
return LOCAL_BROWSER_LIFETIME_MS;
|
|
38
39
|
const timeoutMs = config.execution?.timeoutMs ?? defaultSessionTimeoutMs(config);
|
|
39
40
|
const provisionedRoute = config.subject.source === "clone" || config.subject.source === "local-tree";
|
|
@@ -46,7 +47,7 @@ export function resolveParticipantSandboxMs(config) {
|
|
|
46
47
|
SANDBOX_TIMEOUT_BUFFER_MS);
|
|
47
48
|
}
|
|
48
49
|
/**
|
|
49
|
-
* The in-flight participant bound a
|
|
50
|
+
* The in-flight participant bound a study declares. Defaults to the participant count: every
|
|
50
51
|
* declared participant runs at once, because a throttle nobody asked for silently turns "N actors
|
|
51
52
|
* live" into waves; total session count and spend are the same either way, only wall-clock
|
|
52
53
|
* and simultaneity differ. A declared execution.concurrency is a cap, clamped to [1, participants].
|
|
@@ -173,9 +174,9 @@ export function resolveCuaParticipantPlan(config, opts = {}) {
|
|
|
173
174
|
}
|
|
174
175
|
/** Print the participant plan to stderr before any sandbox/provider call (public-safe: ids, devices,
|
|
175
176
|
* digests, and budgets only; no prompt text, no secrets). */
|
|
176
|
-
export function emitPreflightPlan(participantPlan,
|
|
177
|
+
export function emitPreflightPlan(participantPlan, studyId) {
|
|
177
178
|
const lines = [];
|
|
178
|
-
lines.push(`humanish computer-use fan-out plan (${
|
|
179
|
+
lines.push(`humanish computer-use fan-out plan (${studyId}): ${plural(participantPlan.laneCount, "participant")}, strategy ${participantPlan.strategy}, concurrency ${participantPlan.concurrency}${participantPlan.envLoweredConcurrencyFrom === undefined ? "" : ` (lowered from ${participantPlan.envLoweredConcurrencyFrom} by ${CUA_MAX_CONCURRENCY_ENV})`}, ${plural(participantPlan.waves, "wave")}.`);
|
|
179
180
|
lines.push(` session budget ${Math.round(participantPlan.perLaneSessionBudgetMs / 1000)}s per participant; worst-case ~${participantPlan.worstCaseSandboxMinutes} sandbox-minutes total${participantPlan.dryRun ? " (dry-run: $0)" : ""}.`);
|
|
180
181
|
for (const entry of participantPlan.lanes) {
|
|
181
182
|
lines.push(` - ${formatParticipantPlanEntry(entry)}`);
|
|
@@ -227,7 +228,7 @@ export async function loadCuaParticipants(args) {
|
|
|
227
228
|
return { ok: true, participantRuns, participantPlan };
|
|
228
229
|
const selected = await resolveCuaRerunSelection({
|
|
229
230
|
cwd: args.cwd,
|
|
230
|
-
|
|
231
|
+
studyId: plan.studyId,
|
|
231
232
|
sandboxMs: plan.sandboxMs,
|
|
232
233
|
sourceRunId: rerun.sourceRunId,
|
|
233
234
|
...(rerun.participantIds === undefined ? {} : { participantIds: [...rerun.participantIds] }),
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { type CuaActorDescriptor } from "../../actors/registry.js";
|
|
2
|
-
import type { ComputerUsePlan, ComputerUseRunner, RoutePlanResult, RouteRefusal } from "../../
|
|
3
|
-
import type {
|
|
4
|
-
import { type
|
|
5
|
-
/** The error a computer-use
|
|
6
|
-
export interface ComputerUseRefusal extends RouteRefusal<"computer-use",
|
|
2
|
+
import type { ComputerUsePlan, ComputerUseRunner, RoutePlanResult, RouteRefusal } from "../../study/plan-types.js";
|
|
3
|
+
import type { StudyConfig, StudySubjectState } from "../../study/types.js";
|
|
4
|
+
import { type CuaActorStudyErrorCode, type RunCuaActorStudyOptions } from "./types.js";
|
|
5
|
+
/** The error a computer-use study returns before a run starts. */
|
|
6
|
+
export interface ComputerUseRefusal extends RouteRefusal<"computer-use", CuaActorStudyErrorCode> {
|
|
7
7
|
/**
|
|
8
8
|
* Where the route returns it. "before-scope": analysis and tasks, returned before the run scope
|
|
9
9
|
* with no automatic-analysis record. "in-scope": after the cwd checks. "after-personas": after
|
|
@@ -15,7 +15,7 @@ export interface ComputerUseRefusal extends RouteRefusal<"computer-use", CuaActo
|
|
|
15
15
|
readonly actor?: string;
|
|
16
16
|
}
|
|
17
17
|
export type ComputerUsePlanResult = RoutePlanResult<ComputerUsePlan, ComputerUseRefusal>;
|
|
18
|
-
/** The registered descriptor for a planned actor id;
|
|
18
|
+
/** The registered descriptor for a planned actor id; planComputerUseStudy checked the registry. */
|
|
19
19
|
export declare function cuaDescriptorOf(actor: string): CuaActorDescriptor;
|
|
20
20
|
/** Which of the caller's driving homes, createProvider and inProcess, a run has. */
|
|
21
21
|
export interface CallerDriving {
|
|
@@ -28,21 +28,21 @@ export declare function callerDrivingOf(options: {
|
|
|
28
28
|
readonly createProvider?: unknown;
|
|
29
29
|
}): CallerDriving;
|
|
30
30
|
/** The state a planned run's provisioned subject declares. Other subjects declare none. */
|
|
31
|
-
export declare function cuaDeclaredState(plan: ComputerUsePlan):
|
|
32
|
-
/** The URL a refused
|
|
33
|
-
export declare function declaredAppUrl(config:
|
|
31
|
+
export declare function cuaDeclaredState(plan: ComputerUsePlan): StudySubjectState | undefined;
|
|
32
|
+
/** The URL a refused study's result names, from its declaration: a provisioned subject's serve URL. */
|
|
33
|
+
export declare function declaredAppUrl(config: StudyConfig): string;
|
|
34
34
|
/** The URL a planned run's participants open: a provisioned subject's served URL, else its own. */
|
|
35
35
|
export declare function plannedAppUrl(subject: ComputerUseRunner["subject"]): string;
|
|
36
36
|
/**
|
|
37
|
-
* Plan a computer-use
|
|
37
|
+
* Plan a computer-use study. It is called for any config handed to the computer-use runner, not only
|
|
38
38
|
* one routeOf sends here, so a config for another route gets this route's refusal.
|
|
39
39
|
*/
|
|
40
|
-
export declare function
|
|
40
|
+
export declare function planComputerUseStudy(config: StudyConfig, input: {
|
|
41
41
|
readonly dryRun: boolean;
|
|
42
42
|
/** Whether deps.runSession is set: a caller's session runner cannot enforce maxOutputTokens. */
|
|
43
43
|
readonly hasRunSession?: boolean;
|
|
44
44
|
/** Which of the caller's driving homes are set; neither when absent. */
|
|
45
45
|
readonly driving?: CallerDriving;
|
|
46
46
|
readonly countOverride?: number;
|
|
47
|
-
readonly rerun?:
|
|
47
|
+
readonly rerun?: RunCuaActorStudyOptions["rerun"];
|
|
48
48
|
}): ComputerUsePlanResult;
|
|
@@ -4,18 +4,18 @@
|
|
|
4
4
|
// values, spend-cap prices and the comms catch.
|
|
5
5
|
import { resolveAutomaticAnalysis } from "../../analysis/automatic-config.js";
|
|
6
6
|
import { actorRegistry, isCuaActorDescriptor, } from "../../actors/registry.js";
|
|
7
|
-
import { isHttpUrl, isLoopbackUrl } from "../../
|
|
8
|
-
import { subjectStateInvalidReason } from "../../
|
|
9
|
-
import { brainOf, callerBrainOf, capsOf, desktopRequirements, planBase, provisionedSubject, } from "../../
|
|
10
|
-
import { computerUseParticipants, declaredTargets } from "../../
|
|
11
|
-
import { MAX_COMPUTER_USE_PARTICIPANTS } from "../../
|
|
12
|
-
import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, scenarioCapsValidationReason, taskProtocolValidationReason, } from "../../
|
|
13
|
-
import { desktopCliProductReason } from "../../
|
|
14
|
-
import {
|
|
7
|
+
import { isHttpUrl, isLoopbackUrl } from "../../study/parse/subject.js";
|
|
8
|
+
import { subjectStateInvalidReason } from "../../study/parse/subject-state.js";
|
|
9
|
+
import { brainOf, callerBrainOf, capsOf, desktopRequirements, planBase, provisionedSubject, } from "../../study/plan-base.js";
|
|
10
|
+
import { computerUseParticipants, declaredTargets } from "../../study/plan-participants.js";
|
|
11
|
+
import { MAX_COMPUTER_USE_PARTICIPANTS } from "../../study/routing.js";
|
|
12
|
+
import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, scenarioCapsValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
13
|
+
import { desktopCliProductReason } from "../../study/composition-rules.js";
|
|
14
|
+
import { isLocalBrowserStudy } from "../../substrates/local/runtime-config.js";
|
|
15
15
|
import { boundedConcurrency, defaultSessionTimeoutMs, resolveParticipantSandboxMs, } from "./participant-runs.js";
|
|
16
16
|
import { MAX_SANDBOX_MS } from "../../substrates/e2b/lifetime.js";
|
|
17
17
|
import {} from "./types.js";
|
|
18
|
-
/** The registered descriptor for a planned actor id;
|
|
18
|
+
/** The registered descriptor for a planned actor id; planComputerUseStudy checked the registry. */
|
|
19
19
|
export function cuaDescriptorOf(actor) {
|
|
20
20
|
const descriptor = actorRegistry[actor];
|
|
21
21
|
if (!descriptor || !isCuaActorDescriptor(descriptor))
|
|
@@ -52,7 +52,7 @@ export function cuaDeclaredState(plan) {
|
|
|
52
52
|
const { subject } = plan.runner;
|
|
53
53
|
return subject.kind === "clone" || subject.kind === "local-tree" ? subject.state : undefined;
|
|
54
54
|
}
|
|
55
|
-
/** The URL a refused
|
|
55
|
+
/** The URL a refused study's result names, from its declaration: a provisioned subject's serve URL. */
|
|
56
56
|
export function declaredAppUrl(config) {
|
|
57
57
|
const { subject } = config;
|
|
58
58
|
const provisioned = subject.source === "clone" || subject.source === "local-tree";
|
|
@@ -69,11 +69,11 @@ const invalid = (message) => ({
|
|
|
69
69
|
message,
|
|
70
70
|
});
|
|
71
71
|
/**
|
|
72
|
-
* The first reason a computer-use
|
|
72
|
+
* The first reason a computer-use study cannot start, checked before any sandbox, key or provider
|
|
73
73
|
* is touched. The parser enforces most of these too; the engine repeats them for library callers
|
|
74
74
|
* that hand it a config directly. The groups run in this order, and each returns its first reason.
|
|
75
75
|
*/
|
|
76
|
-
function
|
|
76
|
+
function cuaStudyRejection(config, hasRunSession, driving, subjectRoute) {
|
|
77
77
|
return (unsupportedDeclarationReason(config, hasRunSession, driving, subjectRoute) ??
|
|
78
78
|
subjectStructureReason(config, subjectRoute) ??
|
|
79
79
|
entryTargetReason(config, subjectRoute) ??
|
|
@@ -112,15 +112,15 @@ function subjectStructureReason(config, subjectRoute) {
|
|
|
112
112
|
return invalid(!serve
|
|
113
113
|
? "A clone subject on the computer-use route needs `subject.serve` (start and url): humanish starts the app in the sandbox before the participant opens it."
|
|
114
114
|
: `subject.repos[0] must be an owner/repo slug (got "${subjectRepo ?? ""}").`);
|
|
115
|
-
// A library caller that skips
|
|
115
|
+
// A library caller that skips parseStudy gets the same fail-closed shape the parser
|
|
116
116
|
// enforces, naming which requirement is missing.
|
|
117
117
|
if (localTreeRoute && (!serve || config.execution?.target !== "e2b-desktop"))
|
|
118
118
|
return invalid(!serve
|
|
119
|
-
? "local-tree subjects on the computer-use route require `subject.serve` (start + url):
|
|
119
|
+
? "local-tree subjects on the computer-use route require `subject.serve` (start + url): humanish packs and serves the working tree in the sandbox."
|
|
120
120
|
: "local-tree subjects require `execution.target: e2b-desktop`: the packed working tree is provisioned and served inside a hosted desktop sandbox.");
|
|
121
121
|
if (config.subject.state) {
|
|
122
122
|
const stateReason = !provisionedRoute
|
|
123
|
-
? "`subject.state` applies only to clone subjects or local-tree subjects (
|
|
123
|
+
? "`subject.state` applies only to clone subjects or local-tree subjects (humanish seeds the state it serves)."
|
|
124
124
|
: subjectStateInvalidReason(config.subject.state, config.subject.env);
|
|
125
125
|
if (stateReason)
|
|
126
126
|
return invalid(stateReason);
|
|
@@ -159,13 +159,13 @@ function driverReason(driving, { localAppSubject, inProcessRoute }) {
|
|
|
159
159
|
if (driving.inProcess && !driving.createProvider)
|
|
160
160
|
return {
|
|
161
161
|
code: "HUMANISH_COMPUTER_USE_EXECUTOR_NO_PROVIDER",
|
|
162
|
-
message: "
|
|
162
|
+
message: "RunStudyOptions.inProcess needs RunStudyOptions.createProvider: an in-process executor returns no screenshot, so it needs a provider that does not read images. The default OpenAI computer-use provider reads screenshots and would stop the session.",
|
|
163
163
|
};
|
|
164
164
|
// There is no built-in in-process driver for a local app.
|
|
165
165
|
if (localAppSubject && !inProcessRoute)
|
|
166
166
|
return {
|
|
167
167
|
code: "HUMANISH_COMPUTER_USE_LOCAL_APP_NO_EXECUTOR",
|
|
168
|
-
message: "subject.source: local-app has no built-in driver. Supply one through
|
|
168
|
+
message: "subject.source: local-app has no built-in driver. Supply one through runStudy(config, { inProcess: { executor }, createProvider }); a state-driven executor needs a non-vision provider.",
|
|
169
169
|
};
|
|
170
170
|
return undefined;
|
|
171
171
|
}
|
|
@@ -176,7 +176,7 @@ function rosterShapeReason(config) {
|
|
|
176
176
|
const fanoutReason = computerUseValidationReason(config);
|
|
177
177
|
if (fanoutReason)
|
|
178
178
|
return { code: "HUMANISH_COMPUTER_USE_FANOUT_INVALID", message: fanoutReason };
|
|
179
|
-
// The sandbox deadline is derived from the session budget, so a
|
|
179
|
+
// The sandbox deadline is derived from the session budget, so a study can ask for a session that
|
|
180
180
|
// cannot legally be provisioned. Show the arithmetic: the provider's own error names a limit
|
|
181
181
|
// but not which knob produced it.
|
|
182
182
|
const derivedSandboxMs = resolveParticipantSandboxMs(config);
|
|
@@ -192,10 +192,10 @@ function rerunPlan({ sourceRunId, participantIds, }) {
|
|
|
192
192
|
return { sourceRunId, ...(participantIds === undefined ? {} : { participantIds }) };
|
|
193
193
|
}
|
|
194
194
|
/**
|
|
195
|
-
* Plan a computer-use
|
|
195
|
+
* Plan a computer-use study. It is called for any config handed to the computer-use runner, not only
|
|
196
196
|
* one routeOf sends here, so a config for another route gets this route's refusal.
|
|
197
197
|
*/
|
|
198
|
-
export function
|
|
198
|
+
export function planComputerUseStudy(config, input) {
|
|
199
199
|
const hasRunSession = input.hasRunSession === true;
|
|
200
200
|
const driving = input.driving ?? { inProcess: false, createProvider: false };
|
|
201
201
|
const refuse = (stage, code, message, actor) => ({
|
|
@@ -220,20 +220,20 @@ export function planComputerUseLab(config, input) {
|
|
|
220
220
|
if (!descriptor || !isCuaActorDescriptor(descriptor))
|
|
221
221
|
return refuse("in-scope", "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED", `actors[0].type "${actorType}" is not a registered computer-use actor.`);
|
|
222
222
|
const actor = descriptor.id;
|
|
223
|
-
const rejection =
|
|
223
|
+
const rejection = cuaStudyRejection(config, hasRunSession, driving, declaredSubjectRoute(config, driving));
|
|
224
224
|
if (rejection)
|
|
225
225
|
return refuse("in-scope", rejection.code, rejection.message, actor);
|
|
226
226
|
// A shared world runs every participant against one app; this route would run them as separate
|
|
227
227
|
// participants. It comes after the rules above, so a shared-world config that breaks one of them,
|
|
228
|
-
// which
|
|
228
|
+
// which runStudyWith sends here, still gets that rule's message.
|
|
229
229
|
if (config.subject.topology === "shared-world")
|
|
230
|
-
return refuse("in-scope", "HUMANISH_COMPUTER_USE_SUBJECT_INVALID", "subject.topology: shared-world
|
|
230
|
+
return refuse("in-scope", "HUMANISH_COMPUTER_USE_SUBJECT_INVALID", "subject.topology: shared-world studies run every participant against one shared app on the shared-world route; this route runs independent participants. Run the study with runStudy or runConcurrentSharedWorld.", actor);
|
|
231
231
|
// The in-process route drives subject.appUrl on this machine and creates no desktop, so it would
|
|
232
|
-
// skip the subject a clone, local-tree or desktop-cli
|
|
232
|
+
// skip the subject a clone, local-tree or desktop-cli study declares.
|
|
233
233
|
const source = config.subject.source;
|
|
234
234
|
if (driving.inProcess &&
|
|
235
235
|
(source === "clone" || source === "local-tree" || source === "desktop-cli"))
|
|
236
|
-
return refuse("in-scope", "HUMANISH_COMPUTER_USE_SUBJECT_INVALID", `
|
|
236
|
+
return refuse("in-scope", "HUMANISH_COMPUTER_USE_SUBJECT_INVALID", `RunStudyOptions.inProcess drives subject.appUrl in this process, and a ${source} subject needs the hosted desktop the in-process route never creates. Use an app-url or local-app subject with inProcess, or remove inProcess to run on a hosted desktop.`, actor);
|
|
237
237
|
// The parser's rule; without a product the desktop study fails later.
|
|
238
238
|
const productReason = desktopCliProductReason(config);
|
|
239
239
|
if (productReason)
|
|
@@ -243,7 +243,7 @@ export function planComputerUseLab(config, input) {
|
|
|
243
243
|
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", `Computer-use fan-out is capped at ${MAX_COMPUTER_USE_PARTICIPANTS} participants (resolved ${participants.length}); N concurrent paid desktops is real spend.`, actor);
|
|
244
244
|
const [first, ...rest] = participants;
|
|
245
245
|
if (driving.inProcess && rest.length > 0)
|
|
246
|
-
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", "Fan-out to more than one participant is not supported on the in-process route (
|
|
246
|
+
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", "Fan-out to more than one participant is not supported on the in-process route (RunStudyOptions.inProcess): fan-out provisions one independent E2B desktop per participant, which the in-process route deliberately skips. Run a single in-process participant, or fan out on the E2B route.", actor);
|
|
247
247
|
if (first === undefined)
|
|
248
248
|
throw new Error("computerUseParticipants returned no participant");
|
|
249
249
|
const appUrl = config.subject.appUrl ?? "";
|
|
@@ -267,7 +267,7 @@ export function planComputerUseLab(config, input) {
|
|
|
267
267
|
participants: [first],
|
|
268
268
|
subject: source === "local-app" ? { kind: "local-app", appUrl } : appUrlSubject,
|
|
269
269
|
}
|
|
270
|
-
:
|
|
270
|
+
: isLocalBrowserStudy(config)
|
|
271
271
|
? { desktop: "local-vm", brain, participants: [first, ...rest], subject: appUrlSubject }
|
|
272
272
|
: { desktop: "e2b-desktop", brain, participants: [first, ...rest], subject: hosted };
|
|
273
273
|
const declared = config.execution?.concurrency;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { Brain, ComputerUsePlan } from "../../
|
|
2
|
-
import type {
|
|
3
|
-
import type {
|
|
1
|
+
import type { Brain, ComputerUsePlan } from "../../study/plan-types.js";
|
|
2
|
+
import type { StudyCommsExternal } from "../../study/types.js";
|
|
3
|
+
import type { CuaActorStudyErrorCode } from "./types.js";
|
|
4
4
|
/**
|
|
5
5
|
* The first reason a live computer-use run cannot start on this machine: missing keys, a missing
|
|
6
6
|
* or signed-out local agent, missing subject env, a spend cap with no price, or an unreachable
|
|
@@ -14,8 +14,8 @@ export declare function liveCuaRejection(args: {
|
|
|
14
14
|
env: Record<string, string | undefined>;
|
|
15
15
|
/** The plan's requirements: which keys and subject env names this run needs. */
|
|
16
16
|
requirements: ComputerUsePlan["requirements"];
|
|
17
|
-
externalCommsConfig:
|
|
17
|
+
externalCommsConfig: StudyCommsExternal | undefined;
|
|
18
18
|
}): Promise<{
|
|
19
|
-
code:
|
|
19
|
+
code: CuaActorStudyErrorCode;
|
|
20
20
|
message: string;
|
|
21
21
|
} | undefined>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { pricedModel } from "../../
|
|
2
|
-
import { missingKeys, missingSubjectEnv } from "../../
|
|
1
|
+
import { pricedModel } from "../../study/plan-base.js";
|
|
2
|
+
import { missingKeys, missingSubjectEnv } from "../../study/requirements.js";
|
|
3
3
|
import { detectLocalAgents } from "../../actors/local-agent/cli.js";
|
|
4
4
|
import { localAgentRefusal } from "../../actors/local-agent/readiness.js";
|
|
5
5
|
import { describeMissingKeys } from "../../keys/key-resolution.js";
|
|
@@ -40,10 +40,10 @@ export async function liveCuaRejection(args) {
|
|
|
40
40
|
: "";
|
|
41
41
|
return {
|
|
42
42
|
code: "HUMANISH_COMPUTER_USE_KEYS_MISSING",
|
|
43
|
-
message: `Live computer-use
|
|
43
|
+
message: `Live computer-use studies need ${missing.join(" and ")} in the environment (values are never persisted). ${describeMissingKeys(missing, env)}${suggestion}`,
|
|
44
44
|
};
|
|
45
45
|
}
|
|
46
|
-
// A caller's createProvider makes the brain `caller`, so only the
|
|
46
|
+
// A caller's createProvider makes the brain `caller`, so only the study's own local agent is checked.
|
|
47
47
|
if (localAgent) {
|
|
48
48
|
// Refuse here, before a sandbox exists. "codex is not installed" discovered after the
|
|
49
49
|
// machine is paid for is the same information delivered at the worst possible moment.
|
|
@@ -74,7 +74,7 @@ export async function liveCuaRejection(args) {
|
|
|
74
74
|
};
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
|
-
// Adopter-hosted comms catch: fail closed before any sandbox is created, because a comms
|
|
77
|
+
// Adopter-hosted comms catch: fail closed before any sandbox is created, because a comms study
|
|
78
78
|
// whose catch is unreachable collects nothing while every participant still spends. The probe asserts
|
|
79
79
|
// humanish's own service marker in /health, so an adopter's proxy answering 200 for everything cannot
|
|
80
80
|
// pass for a catch.
|
|
@@ -2,7 +2,7 @@ import { type RunRerunLineage } from "../../run/bundle.js";
|
|
|
2
2
|
import type { CuaParticipantPlan, DesktopParticipantRun } from "./types.js";
|
|
3
3
|
export declare function resolveCuaRerunSelection(args: {
|
|
4
4
|
cwd: string;
|
|
5
|
-
|
|
5
|
+
studyId: string;
|
|
6
6
|
sandboxMs: number;
|
|
7
7
|
sourceRunId: string;
|
|
8
8
|
participantIds?: string[];
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import {} from "../../run/bundle.js";
|
|
4
4
|
import { loadRunBundle } from "../../run/locate.js";
|
|
5
5
|
import {} from "../../run/streams.js";
|
|
6
|
+
import { plural } from "../../run/text.js";
|
|
6
7
|
export async function resolveCuaRerunSelection(args) {
|
|
7
8
|
const source = await loadRunBundle(args.cwd, args.sourceRunId);
|
|
8
9
|
if (!source) {
|
|
@@ -43,7 +44,7 @@ export async function resolveCuaRerunSelection(args) {
|
|
|
43
44
|
if (missingPrior.length > 0) {
|
|
44
45
|
return {
|
|
45
46
|
ok: false,
|
|
46
|
-
message:
|
|
47
|
+
message: `${plural(missingPrior.length, "selected participant id")} ${missingPrior.length === 1 ? "was" : "were"} not present in source run ${bundle.runId}: ${missingPrior.join(", ")}`,
|
|
47
48
|
};
|
|
48
49
|
}
|
|
49
50
|
const specsById = new Map(args.participantRuns.map((spec) => [spec.planned.id, spec]));
|
|
@@ -51,7 +52,7 @@ export async function resolveCuaRerunSelection(args) {
|
|
|
51
52
|
if (missingCurrent.length > 0) {
|
|
52
53
|
return {
|
|
53
54
|
ok: false,
|
|
54
|
-
message:
|
|
55
|
+
message: `${plural(missingCurrent.length, "selected participant id")} ${missingCurrent.length === 1 ? "is" : "are"} not present in the current study file ${args.studyId}: ${missingCurrent.join(", ")}`,
|
|
55
56
|
};
|
|
56
57
|
}
|
|
57
58
|
const selectedSpecs = selectedIds.map((id) => specsById.get(id));
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { runStudyParticipants } from "./live-phase.js";
|
|
2
2
|
import type { CuaFinishFacts, CuaRunSetup } from "./setup.js";
|
|
3
|
-
import { type
|
|
3
|
+
import { type CuaActorStudyResult } from "./types.js";
|
|
4
4
|
/** Builds and publishes the final bundle, runs the adapter hooks, renders the Observer and returns the result. */
|
|
5
|
-
export declare function finishCuaRun(setup: CuaRunSetup, finish: CuaFinishFacts, ran: Extract<Awaited<ReturnType<typeof
|
|
5
|
+
export declare function finishCuaRun(setup: CuaRunSetup, finish: CuaFinishFacts, ran: Extract<Awaited<ReturnType<typeof runStudyParticipants>>, {
|
|
6
6
|
ok: true;
|
|
7
|
-
}>): Promise<
|
|
7
|
+
}>): Promise<CuaActorStudyResult>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { adapterScoreFailureMessage, applyBrowserScorer } from "../../
|
|
1
|
+
import { adapterScoreFailureMessage, applyBrowserScorer } from "../../study/adapter-extension.js";
|
|
2
2
|
import { redactText } from "../../evidence/redaction.js";
|
|
3
3
|
import { round6 } from "../../run/pricing.js";
|
|
4
4
|
import { foldScorerFailures, judgeExecution, OUTCOME_POLICIES, participantHarnessFailed, resultOk, } from "../../run/judge.js";
|
|
@@ -10,6 +10,7 @@ import { aggregateCuaSubject, projectParticipantSubjects, subjectProvenanceArg,
|
|
|
10
10
|
import { CUA_FANOUT_STRATEGY, participantSubjectEnv, } from "./types.js";
|
|
11
11
|
import { plannedAppUrl } from "./plan.js";
|
|
12
12
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
13
|
+
import { plural } from "../../run/text.js";
|
|
13
14
|
/** Aggregate participant counts for the result projection. */
|
|
14
15
|
function buildParticipantSummary(outcomes, participantCount, participantPlan, dryRun) {
|
|
15
16
|
if (dryRun || !outcomes) {
|
|
@@ -52,12 +53,12 @@ function buildParticipantSummary(outcomes, participantCount, participantPlan, dr
|
|
|
52
53
|
};
|
|
53
54
|
}
|
|
54
55
|
/**
|
|
55
|
-
* The computer-use
|
|
56
|
+
* The computer-use study result for a finished run. The run passes only when the Observer rendered,
|
|
56
57
|
* every participant passed (dry-run participants pass as contracts), and no adapter score or declared scorer
|
|
57
58
|
* verdict failed; otherwise the error names the first reason.
|
|
58
59
|
*/
|
|
59
|
-
function
|
|
60
|
-
const {
|
|
60
|
+
function cuaStudyResult(args) {
|
|
61
|
+
const { studyId, cwd, runId, appUrl, dryRun, participantRuns, outcomes, subjects, aggregateSubject, participantPlan, rerunLineage, bundle, observer, receivingWarnings, aggregateWarnings, adapterWarnings, } = args;
|
|
61
62
|
const participantCount = participantRuns.length;
|
|
62
63
|
// Participant-level pass: dry-run participants are contract-ok; live ones need a passed,
|
|
63
64
|
// engaged session.
|
|
@@ -104,8 +105,8 @@ function cuaLabResult(args) {
|
|
|
104
105
|
? `Computer-use session ended with a harness error: ${outcome.session.reason}`
|
|
105
106
|
: outcome?.session?.status !== "passed"
|
|
106
107
|
? `Computer-use session ended with ${outcome?.session?.status ?? "unknown"}: ${outcome?.session?.reason ?? "no terminal reason"}`
|
|
107
|
-
: "
|
|
108
|
-
: (observer.error?.message ?? "Observer failed for the computer-use
|
|
108
|
+
: "The computer-use run did not produce a terminal session."
|
|
109
|
+
: (observer.error?.message ?? "Observer failed for the computer-use run.")),
|
|
109
110
|
};
|
|
110
111
|
}
|
|
111
112
|
const failing = (outcomes ?? []).find((outcome) => !participantOk(outcome));
|
|
@@ -114,12 +115,12 @@ function cuaLabResult(args) {
|
|
|
114
115
|
return {
|
|
115
116
|
code,
|
|
116
117
|
message: observer.ok
|
|
117
|
-
? `Fan-out run failed: ${summary.passed}/${participantCount
|
|
118
|
+
? `Fan-out run failed: ${summary.passed}/${plural(participantCount, "participant")} passed (${summary.skipped} skipped, ${plural(summary.harnessErrors, "harness error")}, ${summary.hollow} without engagement)${failing?.sessionError !== undefined ? `; first failure: ${failing.sessionError}` : failing === undefined && args.execution.failures[0] !== undefined ? `; ${args.execution.failures[0].message}` : ""}.`
|
|
118
119
|
: (observer.error?.message ?? "Observer failed for the computer-use fan-out run."),
|
|
119
120
|
};
|
|
120
121
|
})();
|
|
121
122
|
return {
|
|
122
|
-
...studyResultIdentity("computer-use",
|
|
123
|
+
...studyResultIdentity("computer-use", studyId),
|
|
123
124
|
ok,
|
|
124
125
|
cwd,
|
|
125
126
|
actor: args.actorId,
|
|
@@ -198,7 +199,7 @@ function computerUseExecutionFailures(outcomes, observer, runId) {
|
|
|
198
199
|
: [
|
|
199
200
|
{
|
|
200
201
|
kind: "evidence",
|
|
201
|
-
message: observer.error?.message ?? "Observer failed for the computer-use
|
|
202
|
+
message: observer.error?.message ?? "Observer failed for the computer-use run.",
|
|
202
203
|
},
|
|
203
204
|
]),
|
|
204
205
|
];
|
|
@@ -256,7 +257,7 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
256
257
|
context: {
|
|
257
258
|
bundle,
|
|
258
259
|
runDir: run.paths.physicalRunRoot,
|
|
259
|
-
|
|
260
|
+
studyId: plan.studyId,
|
|
260
261
|
runId,
|
|
261
262
|
actor: descriptor.id,
|
|
262
263
|
route: "computer-use",
|
|
@@ -275,8 +276,8 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
275
276
|
const observer = await finished.renderObserver();
|
|
276
277
|
streams.attachFinal(observer);
|
|
277
278
|
const execution = judgeExecution(computerUseExecutionFailures(outcomes, observer, runId), OUTCOME_POLICIES["computer-use"]);
|
|
278
|
-
const result =
|
|
279
|
-
|
|
279
|
+
const result = cuaStudyResult({
|
|
280
|
+
studyId: plan.studyId,
|
|
280
281
|
cwd,
|
|
281
282
|
runId,
|
|
282
283
|
actorId: descriptor.id,
|