humanish 0.108.0 → 0.109.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +58 -59
- package/CHANGELOG.md +13 -0
- package/CONTEXT.md +44 -44
- package/README.md +3 -3
- package/TELEMETRY.md +2 -2
- package/dist/actors/codex/app-server-trace.js +2 -1
- package/dist/actors/codex/restricted-participant.js +12 -12
- package/dist/actors/codex/restricted-session.d.ts +2 -2
- package/dist/actors/computer-use/actor.d.ts +6 -6
- package/dist/actors/computer-use/actor.js +2 -2
- package/dist/actors/computer-use/admission-limit.d.ts +2 -2
- package/dist/actors/computer-use/admission-limit.js +3 -3
- package/dist/actors/computer-use/executor-error.d.ts +2 -2
- package/dist/actors/computer-use/executor-error.js +3 -3
- package/dist/actors/computer-use/loop/actions.js +5 -5
- package/dist/actors/computer-use/loop/debrief.js +2 -2
- package/dist/actors/computer-use/loop/ending.d.ts +1 -1
- package/dist/actors/computer-use/loop/ending.js +12 -11
- package/dist/actors/computer-use/loop/observation.js +6 -5
- package/dist/actors/computer-use/loop/provider-call.js +11 -11
- package/dist/actors/computer-use/loop/session.d.ts +4 -4
- package/dist/actors/computer-use/loop/session.js +2 -2
- package/dist/actors/computer-use/loop/trace.js +3 -2
- package/dist/actors/computer-use/loop/types.d.ts +24 -24
- package/dist/actors/computer-use/loop/usage.d.ts +2 -2
- package/dist/actors/computer-use/loop.d.ts +6 -1
- package/dist/actors/computer-use/loop.js +26 -4
- package/dist/actors/computer-use/openai-provider.d.ts +2 -2
- package/dist/actors/computer-use/openai-provider.js +8 -8
- package/dist/actors/computer-use/provider-error.d.ts +4 -4
- package/dist/actors/computer-use/provider-error.js +6 -6
- package/dist/actors/contract.d.ts +4 -4
- package/dist/actors/contract.js +1 -1
- package/dist/actors/local-agent/claude-session.js +3 -3
- package/dist/actors/local-agent/cli.d.ts +1 -1
- package/dist/actors/local-agent/cli.js +1 -1
- package/dist/actors/reasoning-effort.js +3 -3
- package/dist/actors/registry.d.ts +2 -2
- package/dist/actors/registry.js +3 -3
- package/dist/actors/scripted-browser/actor.d.ts +2 -2
- package/dist/actors/scripted-browser/actor.js +1 -1
- package/dist/analysis/automatic-completion.d.ts +7 -7
- package/dist/analysis/automatic-completion.js +1 -1
- package/dist/analysis/automatic-config.d.ts +6 -6
- package/dist/analysis/automatic-config.js +1 -1
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +1 -1
- package/dist/analysis/types.d.ts +6 -0
- package/dist/browser-control/client.js +16 -16
- package/dist/browser-control/dispatcher.js +4 -4
- package/dist/browser-control/protocol.js +14 -14
- package/dist/browser-control/recording-transfer.js +4 -4
- package/dist/browser-control/transport.js +7 -7
- package/dist/cli/commands/analysis-signals.d.ts +2 -2
- package/dist/cli/commands/comms.js +14 -22
- package/dist/cli/commands/observe.d.ts +0 -1
- package/dist/cli/commands/observe.js +0 -15
- package/dist/cli/commands/run-command.d.ts +2 -2
- package/dist/cli/commands/run-command.js +3 -3
- package/dist/cli/commands/runs.js +3 -5
- package/dist/cli/commands/setup.js +5 -7
- package/dist/cli/commands/study-format.d.ts +9 -0
- package/dist/cli/commands/{lab-format.js → study-format.js} +5 -5
- package/dist/cli/commands/{lab-route-computer-use.d.ts → study-route-computer-use.d.ts} +6 -6
- package/dist/cli/commands/{lab-route-computer-use.js → study-route-computer-use.js} +12 -12
- package/dist/cli/commands/{lab-route-open.d.ts → study-route-open.d.ts} +4 -4
- package/dist/cli/commands/{lab-route-open.js → study-route-open.js} +2 -2
- package/dist/cli/commands/{lab-route-preview.d.ts → study-route-preview.d.ts} +7 -7
- package/dist/cli/commands/{lab-route-preview.js → study-route-preview.js} +3 -3
- package/dist/cli/commands/study-route-run.d.ts +19 -0
- package/dist/cli/commands/{lab-route-run.js → study-route-run.js} +6 -6
- package/dist/cli/commands/study-route-scripted.d.ts +17 -0
- package/dist/cli/commands/{lab-route-scripted.js → study-route-scripted.js} +4 -4
- package/dist/cli/commands/{lab-route-shared-world.d.ts → study-route-shared-world.d.ts} +5 -5
- package/dist/cli/commands/{lab-route-shared-world.js → study-route-shared-world.js} +7 -7
- package/dist/cli/commands/study-route-terminal.d.ts +17 -0
- package/dist/cli/commands/{lab-route-terminal.js → study-route-terminal.js} +4 -4
- package/dist/cli/commands/study-run.d.ts +9 -0
- package/dist/cli/commands/{lab-run.js → study-run.js} +34 -34
- package/dist/cli/commands/{lab-scorer.d.ts → study-scorer.d.ts} +5 -5
- package/dist/cli/commands/{lab-scorer.js → study-scorer.js} +1 -1
- package/dist/cli/commands/study.js +13 -49
- package/dist/cli/commands/tui.js +12 -12
- package/dist/cli/commands/watch.js +30 -65
- package/dist/cli/deprecations.d.ts +1 -13
- package/dist/cli/deprecations.js +0 -30
- package/dist/cli/doctor.d.ts +3 -3
- package/dist/cli/doctor.js +16 -16
- package/dist/cli/first-run-path.d.ts +4 -4
- package/dist/cli/first-run-path.js +7 -7
- package/dist/cli/io.d.ts +3 -3
- package/dist/cli/io.js +2 -2
- package/dist/cli/observer-follow.d.ts +2 -2
- package/dist/cli/observer-follow.js +2 -2
- package/dist/cli/orientation.d.ts +4 -4
- package/dist/cli/orientation.js +58 -20
- package/dist/cli/program.js +3 -4
- package/dist/cli/telemetry.d.ts +1 -1
- package/dist/cli/telemetry.js +14 -20
- package/dist/comms/catch-host.d.ts +1 -1
- package/dist/comms/catch-host.js +3 -3
- package/dist/comms/connections.d.ts +1 -1
- package/dist/comms/connections.js +2 -2
- package/dist/comms/external-evidence.d.ts +5 -5
- package/dist/comms/external-evidence.js +4 -3
- package/dist/comms/fake-inbox.d.ts +2 -2
- package/dist/comms/fake-inbox.js +2 -2
- package/dist/comms/receiving-runtime.d.ts +4 -4
- package/dist/comms/sandbox-catch.d.ts +2 -2
- package/dist/comms/sandbox-catch.js +2 -2
- package/dist/comms/setup.d.ts +2 -2
- package/dist/comms/setup.js +29 -32
- package/dist/feedback/draft.js +2 -2
- package/dist/feedback/export.js +3 -2
- package/dist/feedback/proof.d.ts +1 -1
- package/dist/feedback/proof.js +2 -2
- package/dist/guest/bootstrap.js +6 -6
- package/dist/guest/browser-tools.js +8 -8
- package/dist/guest/chromium-text.js +17 -17
- package/dist/guest/desktop-executor.d.ts +2 -2
- package/dist/guest/desktop-executor.js +20 -20
- package/dist/guest/desktop-keys.js +4 -4
- package/dist/guest/desktop-media.d.ts +2 -2
- package/dist/guest/desktop-media.js +16 -16
- package/dist/guest/desktop-native.js +10 -10
- package/dist/guest/runtime-desktop.js +2 -2
- package/dist/guest/runtime-processes.d.ts +2 -2
- package/dist/guest/runtime-processes.js +2 -2
- package/dist/guest/runtime.js +5 -5
- package/dist/index.d.ts +13 -15
- package/dist/index.js +6 -8
- package/dist/observer/artifact.d.ts +1 -1
- package/dist/observer/artifact.js +1 -1
- package/dist/observer/data.d.ts +1 -1
- package/dist/observer/library.d.ts +2 -0
- package/dist/observer/library.js +4 -1
- package/dist/observer/render.js +7 -1
- package/dist/observer/run-routes.d.ts +1 -0
- package/dist/observer/run-routes.js +37 -12
- package/dist/observer/serve.d.ts +1 -1
- package/dist/observer-app.html +1 -1
- package/dist/routes/computer-use/bundle-parts.js +3 -2
- package/dist/routes/computer-use/bundle.d.ts +2 -2
- package/dist/routes/computer-use/bundle.js +7 -6
- package/dist/routes/computer-use/e2b-desktop/comms.d.ts +5 -5
- package/dist/routes/computer-use/e2b-desktop/comms.js +5 -4
- package/dist/routes/computer-use/e2b-desktop/fidelity.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.js +3 -3
- package/dist/routes/computer-use/e2b-desktop/start.js +3 -4
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +2 -2
- package/dist/routes/computer-use/e2b-desktop/teardown.js +2 -3
- package/dist/routes/computer-use/external-comms.d.ts +3 -3
- package/dist/routes/computer-use/fanout-bundle.js +11 -10
- package/dist/routes/computer-use/fanout-records.js +2 -2
- package/dist/routes/computer-use/live-flush.d.ts +3 -3
- package/dist/routes/computer-use/live-phase.d.ts +2 -3
- package/dist/routes/computer-use/live-phase.js +3 -3
- package/dist/routes/computer-use/local-tree-pack.d.ts +5 -5
- package/dist/routes/computer-use/local-tree-pack.js +1 -1
- package/dist/routes/computer-use/local-vm.d.ts +8 -8
- package/dist/routes/computer-use/local-vm.js +2 -2
- package/dist/routes/computer-use/participant-desktop.d.ts +3 -3
- package/dist/routes/computer-use/participant-desktop.js +1 -1
- package/dist/routes/computer-use/participant-execution.js +1 -1
- package/dist/routes/computer-use/participant-model.js +3 -3
- package/dist/routes/computer-use/participant-prompt.d.ts +5 -5
- package/dist/routes/computer-use/participant-prompt.js +3 -3
- package/dist/routes/computer-use/participant-runs.d.ts +10 -10
- package/dist/routes/computer-use/participant-runs.js +14 -13
- package/dist/routes/computer-use/plan.d.ts +12 -12
- package/dist/routes/computer-use/plan.js +24 -24
- package/dist/routes/computer-use/preflight.d.ts +5 -5
- package/dist/routes/computer-use/preflight.js +5 -5
- package/dist/routes/computer-use/rerun-selection.d.ts +1 -1
- package/dist/routes/computer-use/rerun-selection.js +3 -2
- package/dist/routes/computer-use/result.d.ts +4 -4
- package/dist/routes/computer-use/result.js +13 -12
- package/dist/routes/computer-use/route.d.ts +12 -12
- package/dist/routes/computer-use/route.js +19 -19
- package/dist/routes/computer-use/self-report.js +1 -1
- package/dist/routes/computer-use/setup.d.ts +22 -22
- package/dist/routes/computer-use/setup.js +15 -15
- package/dist/routes/computer-use/single-bundle.d.ts +4 -4
- package/dist/routes/computer-use/single-bundle.js +14 -13
- package/dist/routes/computer-use/subject-projection.d.ts +3 -3
- package/dist/routes/computer-use/types.d.ts +34 -34
- package/dist/routes/computer-use/types.js +3 -3
- package/dist/routes/preview.d.ts +6 -6
- package/dist/routes/preview.js +5 -5
- package/dist/routes/scripted/bundle.d.ts +6 -6
- package/dist/routes/scripted/bundle.js +10 -9
- package/dist/routes/scripted/plan.d.ts +7 -7
- package/dist/routes/scripted/plan.js +6 -6
- package/dist/routes/scripted/result.d.ts +3 -3
- package/dist/routes/scripted/result.js +9 -9
- package/dist/routes/scripted/route.d.ts +9 -9
- package/dist/routes/scripted/route.js +11 -11
- package/dist/routes/scripted/scenario.d.ts +1 -1
- package/dist/routes/scripted/scenario.js +3 -3
- package/dist/routes/scripted/setup.d.ts +6 -6
- package/dist/routes/scripted/setup.js +4 -4
- package/dist/routes/scripted/subject.d.ts +5 -5
- package/dist/routes/scripted/subject.js +3 -3
- package/dist/routes/scripted/surface-records.d.ts +1 -1
- package/dist/routes/scripted/surface-records.js +2 -2
- package/dist/routes/scripted/surface-sessions.d.ts +3 -3
- package/dist/routes/scripted/types.d.ts +17 -17
- package/dist/routes/scripted/types.js +1 -1
- package/dist/routes/shared-world/bundle-records.js +3 -3
- package/dist/routes/shared-world/bundle.js +15 -14
- package/dist/routes/shared-world/checkpoints.d.ts +3 -3
- package/dist/routes/shared-world/checkpoints.js +1 -1
- package/dist/routes/shared-world/comms.d.ts +9 -9
- package/dist/routes/shared-world/comms.js +6 -5
- package/dist/routes/shared-world/participant-specs.d.ts +3 -3
- package/dist/routes/shared-world/participant-specs.js +7 -7
- package/dist/routes/shared-world/plan.d.ts +9 -9
- package/dist/routes/shared-world/plan.js +9 -9
- package/dist/routes/shared-world/provisioned.d.ts +8 -8
- package/dist/routes/shared-world/provisioned.js +4 -4
- package/dist/routes/shared-world/result.d.ts +6 -6
- package/dist/routes/shared-world/result.js +11 -10
- package/dist/routes/shared-world/route.d.ts +11 -11
- package/dist/routes/shared-world/route.js +19 -19
- package/dist/routes/shared-world/setup.d.ts +13 -13
- package/dist/routes/shared-world/setup.js +19 -19
- package/dist/routes/shared-world/types.d.ts +27 -27
- package/dist/routes/terminal/adapter.d.ts +2 -2
- package/dist/routes/terminal/adapter.js +4 -4
- package/dist/routes/terminal/bundle.d.ts +11 -11
- package/dist/routes/terminal/bundle.js +12 -12
- package/dist/routes/terminal/credentials.d.ts +4 -4
- package/dist/routes/terminal/credentials.js +5 -5
- package/dist/routes/terminal/dry-run.d.ts +4 -4
- package/dist/routes/terminal/dry-run.js +9 -9
- package/dist/routes/terminal/ledger.d.ts +2 -2
- package/dist/routes/terminal/lifetime.d.ts +1 -1
- package/dist/routes/terminal/lifetime.js +1 -1
- package/dist/routes/terminal/live-finish.d.ts +2 -2
- package/dist/routes/terminal/live-finish.js +7 -7
- package/dist/routes/terminal/live-sandbox.d.ts +2 -2
- package/dist/routes/terminal/live-sandbox.js +3 -2
- package/dist/routes/terminal/participant-text.js +2 -1
- package/dist/routes/terminal/persona.d.ts +2 -2
- package/dist/routes/terminal/persona.js +2 -2
- package/dist/routes/terminal/plan.d.ts +7 -7
- package/dist/routes/terminal/plan.js +5 -5
- package/dist/routes/terminal/result.d.ts +5 -5
- package/dist/routes/terminal/result.js +7 -7
- package/dist/routes/terminal/route.d.ts +9 -9
- package/dist/routes/terminal/route.js +14 -14
- package/dist/routes/terminal/session.d.ts +5 -5
- package/dist/routes/terminal/session.js +7 -7
- package/dist/routes/terminal/trace.d.ts +2 -2
- package/dist/routes/terminal/types.d.ts +24 -24
- package/dist/routes/terminal/types.js +2 -2
- package/dist/run/bundle.d.ts +7 -7
- package/dist/run/bundle.js +2 -2
- package/dist/run/costs.d.ts +23 -0
- package/dist/run/costs.js +72 -43
- package/dist/run/dry-run-inputs.d.ts +1 -1
- package/dist/run/dry-run-inputs.js +1 -1
- package/dist/run/dry-run.js +2 -2
- package/dist/run/outcomes.d.ts +1 -1
- package/dist/run/participant.d.ts +1 -1
- package/dist/run/preflight-receipts.js +1 -1
- package/dist/run/pricing.js +2 -2
- package/dist/run/projection.d.ts +32 -32
- package/dist/run/projection.js +44 -44
- package/dist/run/reclaim.d.ts +1 -1
- package/dist/run/reclaim.js +1 -1
- package/dist/run/results.d.ts +2 -2
- package/dist/run/run-cost.d.ts +53 -0
- package/dist/run/run-cost.js +105 -0
- package/dist/run/run-index-analysis.d.ts +15 -0
- package/dist/run/run-index-analysis.js +77 -0
- package/dist/run/run-index.d.ts +11 -1
- package/dist/run/run-index.js +63 -49
- package/dist/run/run.d.ts +3 -3
- package/dist/run/run.js +2 -2
- package/dist/run/shared-world-evidence.d.ts +1 -1
- package/dist/run/shared-world-shape.js +1 -1
- package/dist/run/stats.d.ts +3 -3
- package/dist/run/stats.js +14 -14
- package/dist/run/status.d.ts +10 -2
- package/dist/run/status.js +6 -3
- package/dist/run/streams.d.ts +2 -2
- package/dist/run/study-provenance.d.ts +4 -2
- package/dist/run/study-provenance.js +7 -4
- package/dist/run/study-result.d.ts +0 -5
- package/dist/run/study-result.js +1 -1
- package/dist/run-study.d.ts +112 -0
- package/dist/{run-lab.js → run-study.js} +52 -45
- package/dist/{lab → study}/adapter-extension.d.ts +8 -20
- package/dist/{lab → study}/adapter-extension.js +2 -14
- package/dist/{lab → study}/adapter-scorer-loader.d.ts +10 -10
- package/dist/{lab → study}/adapter-scorer-loader.js +3 -3
- package/dist/study/composition-rules.d.ts +7 -0
- package/dist/{lab → study}/composition-rules.js +12 -12
- package/dist/study/config.d.ts +15 -0
- package/dist/{lab → study}/config.js +37 -20
- package/dist/study/convert.d.ts +1 -1
- package/dist/study/convert.js +15 -15
- package/dist/{lab → study}/device-presets.d.ts +2 -2
- package/dist/{lab → study}/discover.d.ts +29 -23
- package/dist/{lab → study}/discover.js +97 -118
- package/dist/{lab → study}/doctor.d.ts +11 -11
- package/dist/{lab → study}/doctor.js +18 -18
- package/dist/study/files.d.ts +2 -2
- package/dist/study/files.js +6 -5
- package/dist/{lab → study}/init-templates.d.ts +2 -2
- package/dist/{lab → study}/init-templates.js +16 -11
- package/dist/{lab → study}/init.d.ts +1 -1
- package/dist/{lab → study}/init.js +7 -4
- package/dist/{lab → study}/keys.d.ts +2 -2
- package/dist/{lab → study}/keys.js +10 -10
- package/dist/study/migrate.d.ts +1 -1
- package/dist/{lab → study}/parse/actors.d.ts +5 -5
- package/dist/{lab → study}/parse/actors.js +5 -5
- package/dist/study/parse/comms.d.ts +12 -0
- package/dist/study/parse/execution.d.ts +19 -0
- package/dist/{lab → study}/parse/study-v3.d.ts +3 -5
- package/dist/{lab → study}/parse/study-v3.js +3 -3
- package/dist/{lab → study}/parse/subject-state.d.ts +8 -8
- package/dist/{lab → study}/parse/subject-state.js +4 -4
- package/dist/{lab → study}/parse/subject.d.ts +3 -3
- package/dist/{lab → study}/parse/subject.js +8 -8
- package/dist/{lab → study}/parse/values.d.ts +2 -2
- package/dist/{lab → study}/persona-resolve.d.ts +4 -4
- package/dist/{lab → study}/persona-resolve.js +2 -2
- package/dist/{lab → study}/plan-base.d.ts +7 -7
- package/dist/{lab → study}/plan-base.js +1 -1
- package/dist/{lab → study}/plan-participants.d.ts +10 -10
- package/dist/{lab → study}/plan-participants.js +4 -4
- package/dist/{lab → study}/plan-types.d.ts +28 -28
- package/dist/{lab → study}/plan-types.js +2 -2
- package/dist/study/plan.d.ts +13 -0
- package/dist/{lab → study}/plan.js +20 -20
- package/dist/study/preflight-probes.d.ts +3 -0
- package/dist/{lab → study}/preflight-probes.js +9 -8
- package/dist/study/preflight-result.d.ts +8 -0
- package/dist/{lab → study}/preflight-result.js +3 -3
- package/dist/{lab → study}/preflight.d.ts +31 -31
- package/dist/{lab → study}/preflight.js +8 -8
- package/dist/{lab → study}/requirements.d.ts +3 -3
- package/dist/{lab → study}/requirements.js +1 -1
- package/dist/study/route-inputs.d.ts +28 -0
- package/dist/{lab → study}/route-inputs.js +1 -1
- package/dist/{lab → study}/routing.d.ts +12 -12
- package/dist/{lab/run-lab-events.d.ts → study/run-study-events.d.ts} +10 -10
- package/dist/{lab/run-lab-events.js → study/run-study-events.js} +3 -3
- package/dist/{lab/run-lab-homes.d.ts → study/run-study-homes.d.ts} +8 -17
- package/dist/study/run-study-homes.js +3 -0
- package/dist/study/run-study-options.d.ts +28 -0
- package/dist/{lab/run-lab-options.js → study/run-study-options.js} +13 -12
- package/dist/{lab/lab-deps.d.ts → study/study-deps.d.ts} +2 -2
- package/dist/{lab/lab-deps.js → study/study-deps.js} +2 -2
- package/dist/{lab → study}/summary.d.ts +10 -10
- package/dist/{lab → study}/summary.js +15 -15
- package/dist/{lab → study}/tasks.d.ts +3 -3
- package/dist/{lab → study}/tasks.js +1 -1
- package/dist/{lab → study}/types.d.ts +104 -104
- package/dist/{lab → study}/types.js +1 -1
- package/dist/{lab → study}/validation.d.ts +19 -19
- package/dist/{lab → study}/validation.js +13 -13
- package/dist/{lab → study}/warnings.d.ts +4 -4
- package/dist/{lab → study}/warnings.js +1 -1
- package/dist/subject/clone.d.ts +6 -6
- package/dist/subject/clone.js +2 -2
- package/dist/subject/local-tree-archive.d.ts +2 -2
- package/dist/subject/local-tree-archive.js +3 -3
- package/dist/subject/local-tree.d.ts +3 -3
- package/dist/subject/runtime.js +2 -2
- package/dist/subject/serve.d.ts +4 -4
- package/dist/subject/state.d.ts +3 -3
- package/dist/subject/state.js +1 -1
- package/dist/subject/steps.d.ts +1 -1
- package/dist/subject/steps.js +1 -1
- package/dist/substrates/desktop-session.js +2 -2
- package/dist/substrates/detached.d.ts +1 -1
- package/dist/substrates/e2b/desktop-browser.d.ts +4 -4
- package/dist/substrates/e2b/desktop-browser.js +1 -1
- package/dist/substrates/e2b/desktop-cdp.d.ts +1 -1
- package/dist/substrates/e2b/desktop-cdp.js +1 -1
- package/dist/substrates/e2b/desktop-executor.d.ts +1 -1
- package/dist/substrates/e2b/desktop-executor.js +12 -12
- package/dist/substrates/e2b/desktop-geometry.d.ts +1 -1
- package/dist/substrates/e2b/desktop-geometry.js +1 -1
- package/dist/substrates/e2b/desktop-media.d.ts +3 -3
- package/dist/substrates/e2b/lifetime.js +1 -1
- package/dist/substrates/e2b/sandbox.d.ts +3 -3
- package/dist/substrates/e2b/sandbox.js +1 -1
- package/dist/substrates/e2b/sdk.d.ts +1 -1
- package/dist/substrates/e2b/sdk.js +1 -1
- package/dist/substrates/local/runtime-config.d.ts +4 -4
- package/dist/substrates/local/runtime-config.js +4 -4
- package/dist/tui/contract.d.ts +8 -8
- package/dist/tui/contract.js +2 -2
- package/dist/tui/launch.d.ts +3 -3
- package/dist/tui/launch.js +22 -16
- package/dist/tui-app.js +139 -139
- package/dist/verify/subject.d.ts +1 -1
- package/dist/verify/subject.js +1 -1
- package/dist/verify/verify.js +2 -1
- package/docs/architecture/actor-contract.md +5 -5
- package/docs/architecture/browser-control.md +1 -1
- package/docs/architecture/external-public-shared-world.md +8 -7
- package/docs/architecture/github-feedback-loop.md +1 -1
- package/docs/architecture/task-protocol-support.md +1 -1
- package/docs/architecture/terminal-product-route.md +18 -18
- package/docs/contracts/adapter-fixtures.md +2 -2
- package/docs/contracts/errors.md +8 -8
- package/docs/contracts/run-bundle.md +1 -4
- package/docs/contracts/schemas.md +61 -42
- package/docs/contracts/study-analysis.md +4 -1
- package/docs/contracts/study-costs.md +17 -2
- package/examples/README.md +1 -1
- package/examples/scorer/scorer.mjs +3 -3
- package/package.json +1 -1
- package/skills/humanish/SKILL.md +8 -6
- package/skills/humanish/agents/openai.yaml +1 -1
- package/dist/cli/commands/lab-format.d.ts +0 -9
- package/dist/cli/commands/lab-route-run.d.ts +0 -19
- package/dist/cli/commands/lab-route-scripted.d.ts +0 -17
- package/dist/cli/commands/lab-route-terminal.d.ts +0 -17
- package/dist/cli/commands/lab-run.d.ts +0 -9
- package/dist/lab/composition-rules.d.ts +0 -7
- package/dist/lab/config.d.ts +0 -7
- package/dist/lab/deprecated-fields.d.ts +0 -10
- package/dist/lab/deprecated-fields.js +0 -24
- package/dist/lab/parse/comms.d.ts +0 -12
- package/dist/lab/parse/execution.d.ts +0 -19
- package/dist/lab/plan.d.ts +0 -13
- package/dist/lab/preflight-probes.d.ts +0 -3
- package/dist/lab/preflight-result.d.ts +0 -8
- package/dist/lab/route-inputs.d.ts +0 -28
- package/dist/lab/run-lab-homes.js +0 -7
- package/dist/lab/run-lab-options.d.ts +0 -28
- package/dist/library-aliases.d.ts +0 -56
- package/dist/library-aliases.js +0 -22
- package/dist/run-lab.d.ts +0 -112
- /package/dist/{lab → study}/device-presets.js +0 -0
- /package/dist/{lab → study}/init-paths.d.ts +0 -0
- /package/dist/{lab → study}/init-paths.js +0 -0
- /package/dist/{lab → study}/init-plan.d.ts +0 -0
- /package/dist/{lab → study}/init-plan.js +0 -0
- /package/dist/{lab → study}/parse/comms.js +0 -0
- /package/dist/{lab → study}/parse/execution.js +0 -0
- /package/dist/{lab → study}/parse/values.js +0 -0
- /package/dist/{lab → study}/participant-assignment.d.ts +0 -0
- /package/dist/{lab → study}/participant-assignment.js +0 -0
- /package/dist/{lab → study}/persona.d.ts +0 -0
- /package/dist/{lab → study}/persona.js +0 -0
- /package/dist/{lab → study}/routing.js +0 -0
|
@@ -8,11 +8,12 @@ import {} from "../../run/bundle.js";
|
|
|
8
8
|
import { participantResourceIds } from "../../run/participant-records.js";
|
|
9
9
|
import { resolveSelfReportedFriction } from "./self-report.js";
|
|
10
10
|
import {} from "./types.js";
|
|
11
|
+
import { plural } from "../../run/text.js";
|
|
11
12
|
/** The human-readable state story appended to the provenance event (and review.md via it). */
|
|
12
13
|
export function describeSubjectState(state, dryRun) {
|
|
13
14
|
switch (state.provenance) {
|
|
14
15
|
case "seeded":
|
|
15
|
-
return `seeded (${state.seed?.length ?? 0
|
|
16
|
+
return `seeded (${plural(state.seed?.length ?? 0, "step")}: ${(state.seed ?? []).map((record) => record.name).join(", ")})`;
|
|
16
17
|
case "unpinned":
|
|
17
18
|
return `unpinned (external: ${(state.externalEnvNames ?? []).join(", ")})`;
|
|
18
19
|
case "declared-not-run":
|
|
@@ -103,7 +104,7 @@ export function participantFeedbackCandidates(args) {
|
|
|
103
104
|
proposed_next_state: "study-quality-review",
|
|
104
105
|
acceptance_proof: [
|
|
105
106
|
feedbackProofCommands(args.runId).verify,
|
|
106
|
-
feedbackProofCommands(args.runId).
|
|
107
|
+
feedbackProofCommands(args.runId).observe,
|
|
107
108
|
],
|
|
108
109
|
});
|
|
109
110
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CuaActorDescriptor } from "../../actors/registry.js";
|
|
2
|
-
import type { ComputerUsePlan } from "../../
|
|
2
|
+
import type { ComputerUsePlan } from "../../study/plan-types.js";
|
|
3
3
|
import type { BundleRun, RunBundle, RunRerunLineage } from "../../run/bundle.js";
|
|
4
4
|
import { type Judgment } from "../../run/judge.js";
|
|
5
5
|
import type { CuaParticipantPlan, DesktopParticipantRun, CuaSubjectProjection, CuaSubjectProvenanceArg, ParticipantRunOutcome } from "./types.js";
|
|
@@ -20,7 +20,7 @@ export interface CuaRunBundleBase {
|
|
|
20
20
|
/**
|
|
21
21
|
* The judgment for the current state of a computer-use run, by the rule for its shape: one
|
|
22
22
|
* participant (its tallied status is the verdict) or several (every participant must pass). The bundle's
|
|
23
|
-
* verdict and the
|
|
23
|
+
* verdict and the study result's ok both come from it.
|
|
24
24
|
*/
|
|
25
25
|
export declare function judgeComputerUseRun(base: Pick<CuaRunBundleBase, "participantRuns" | "rerun">, state: {
|
|
26
26
|
dryRun: boolean;
|
|
@@ -9,6 +9,7 @@ import { providerResourcesForOutcome } from "./bundle-parts.js";
|
|
|
9
9
|
import { participantFactsOf } from "./participant-facts.js";
|
|
10
10
|
import { buildCuaFanoutBundle } from "./fanout-bundle.js";
|
|
11
11
|
import { buildSingleParticipantBundle } from "./single-bundle.js";
|
|
12
|
+
import { plural } from "../../run/text.js";
|
|
12
13
|
/** One participant without a rerun keeps the single-participant bundle shape and its rule. */
|
|
13
14
|
function isOneParticipantRun(base) {
|
|
14
15
|
return base.participantRuns.length === 1 && base.rerun === undefined;
|
|
@@ -16,7 +17,7 @@ function isOneParticipantRun(base) {
|
|
|
16
17
|
/**
|
|
17
18
|
* The judgment for the current state of a computer-use run, by the rule for its shape: one
|
|
18
19
|
* participant (its tallied status is the verdict) or several (every participant must pass). The bundle's
|
|
19
|
-
* verdict and the
|
|
20
|
+
* verdict and the study result's ok both come from it.
|
|
20
21
|
*/
|
|
21
22
|
export function judgeComputerUseRun(base, state) {
|
|
22
23
|
const inProgress = state.inProgress === true;
|
|
@@ -84,8 +85,8 @@ function singleParticipantArgs(base, state) {
|
|
|
84
85
|
? {}
|
|
85
86
|
: { caseGroup: spec.planned.labels.caseGroup }),
|
|
86
87
|
dryRun: state.dryRun,
|
|
87
|
-
|
|
88
|
-
...(plan.title ? {
|
|
88
|
+
studyId: plan.studyId,
|
|
89
|
+
...(plan.title ? { studyTitle: plan.title } : {}),
|
|
89
90
|
mission: spec.evidenceInstructions ?? spec.instructions,
|
|
90
91
|
...(spec.evidenceAssignment === undefined ? {} : { assignment: spec.evidenceAssignment }),
|
|
91
92
|
persona: spec.persona,
|
|
@@ -165,9 +166,9 @@ export function renderCuaReviewMarkdown(bundle) {
|
|
|
165
166
|
`- actor: ${trace.provider} (${trace.lane}/${trace.protocol})`,
|
|
166
167
|
// Name the trace's actual screenshot mode ("raw" | "blurred"); say
|
|
167
168
|
// nothing when no frames exist ("n/a") rather than claim a redaction that never ran.
|
|
168
|
-
`- evidence: ${trace.items.length
|
|
169
|
-
? `${trace.redaction.screenshots} screenshot
|
|
170
|
-
: "screenshot
|
|
169
|
+
`- evidence: ${plural(trace.items.length, "trace item")}, ${plural(trace.counts.screenshots ?? 0, trace.redaction.screenshots === "raw" || trace.redaction.screenshots === "blurred"
|
|
170
|
+
? `${trace.redaction.screenshots} screenshot`
|
|
171
|
+
: "screenshot")}`,
|
|
171
172
|
]
|
|
172
173
|
: []),
|
|
173
174
|
...(bundle.review.gaps.length > 0
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { type OriginMap } from "../../../comms/capture-surface.js";
|
|
2
2
|
import { type DeployedCommsCatch } from "../../../comms/sandbox-catch.js";
|
|
3
|
-
import type {
|
|
3
|
+
import type { StudyCommsEmail, StudyConfig } from "../../../study/types.js";
|
|
4
4
|
import type { Shell } from "../../../substrates/shell.js";
|
|
5
5
|
import type { ReadyParticipantDesktop } from "../participant-desktop.js";
|
|
6
6
|
import { type E2BDesktopDeps, type DesktopParticipantRun } from "../types.js";
|
|
7
7
|
/** A participant's captured-email wiring, decided from config before the sandbox exists. */
|
|
8
8
|
export interface ParticipantComms {
|
|
9
|
-
readonly email:
|
|
9
|
+
readonly email: StudyCommsEmail;
|
|
10
10
|
readonly port: number;
|
|
11
11
|
readonly smtpPort: number | undefined;
|
|
12
12
|
/** The loopback URL the persona opens to read captured mail. */
|
|
@@ -20,9 +20,9 @@ export interface ParticipantComms {
|
|
|
20
20
|
/**
|
|
21
21
|
* On an in-sandbox subject route, redirect the app's email-API sends into an in-sandbox catch
|
|
22
22
|
* (loopback) so its verification mail is captured instead of sent to the internet. Undefined when the
|
|
23
|
-
*
|
|
23
|
+
* study declares no fake email, which leaves the participant unchanged.
|
|
24
24
|
*/
|
|
25
|
-
export declare function planParticipantComms(comms:
|
|
25
|
+
export declare function planParticipantComms(comms: StudyConfig["comms"], serveUrl: string | undefined, targetUrl: string, inSandboxSubject: boolean): ParticipantComms | undefined;
|
|
26
26
|
/** The env the subject app reads to reach the catch, injected at sandbox create so it boots with it. */
|
|
27
27
|
export declare function participantCommsEnv(comms: ParticipantComms | undefined): Record<string, string>;
|
|
28
28
|
/** A started catch and the stop for its inbox-surface loop. */
|
|
@@ -33,7 +33,7 @@ export interface RunningCommsCatch {
|
|
|
33
33
|
}
|
|
34
34
|
/**
|
|
35
35
|
* Start the in-sandbox email catch before the subject serve, so the app's send-API base URL
|
|
36
|
-
* resolves the moment it boots. A comms-declared
|
|
36
|
+
* resolves the moment it boots. A comms-declared study that cannot stand the catch up fails closed
|
|
37
37
|
* rather than silently sending real mail.
|
|
38
38
|
*/
|
|
39
39
|
export declare function startCommsCatch(shell: Shell, comms: ParticipantComms, requestTimeoutMs: number): Promise<RunningCommsCatch>;
|
|
@@ -9,14 +9,15 @@ import { redactText, toErrorMessage } from "../../../evidence/redaction.js";
|
|
|
9
9
|
import { writeContainedOutputFile } from "../../../run/contained-output.js";
|
|
10
10
|
import { inboxRecipientFor, participantHasInboxRecipient } from "../participant-desktop.js";
|
|
11
11
|
import { participantServeUrl } from "../types.js";
|
|
12
|
-
import { addressedRecipients } from "../../../
|
|
12
|
+
import { addressedRecipients } from "../../../study/parse/comms.js";
|
|
13
|
+
import { plural } from "../../../run/text.js";
|
|
13
14
|
/** Mid-run inbox-surface render cadence (ms). Coarse enough that the per-tick `cat` + file writes stay
|
|
14
15
|
* cheap; fine enough that a verification email is visible seconds after the app sends it. */
|
|
15
16
|
const INBOX_SURFACE_CADENCE_MS = 2500;
|
|
16
17
|
/**
|
|
17
18
|
* On an in-sandbox subject route, redirect the app's email-API sends into an in-sandbox catch
|
|
18
19
|
* (loopback) so its verification mail is captured instead of sent to the internet. Undefined when the
|
|
19
|
-
*
|
|
20
|
+
* study declares no fake email, which leaves the participant unchanged.
|
|
20
21
|
*/
|
|
21
22
|
export function planParticipantComms(comms, serveUrl, targetUrl, inSandboxSubject) {
|
|
22
23
|
const email = inSandboxSubject && comms?.email?.kind === "fake" ? comms.email : undefined;
|
|
@@ -63,7 +64,7 @@ export function participantCommsEnv(comms) {
|
|
|
63
64
|
}
|
|
64
65
|
/**
|
|
65
66
|
* Start the in-sandbox email catch before the subject serve, so the app's send-API base URL
|
|
66
|
-
* resolves the moment it boots. A comms-declared
|
|
67
|
+
* resolves the moment it boots. A comms-declared study that cannot stand the catch up fails closed
|
|
67
68
|
* rather than silently sending real mail.
|
|
68
69
|
*/
|
|
69
70
|
export async function startCommsCatch(shell, comms, requestTimeoutMs) {
|
|
@@ -203,7 +204,7 @@ export async function drainCommsEvidence(args) {
|
|
|
203
204
|
// Captured mail that matched no declared recipient must not vanish silently: tell the
|
|
204
205
|
// operator to declare comms.email.recipients[].address to match
|
|
205
206
|
// the address the app actually sends to (e.g. the one the persona surface will sign up with).
|
|
206
|
-
warnings.push(`The email catch captured ${collected.captured
|
|
207
|
+
warnings.push(`The email catch captured ${plural(collected.captured, "email send")}, but none matched a declared recipient inbox, so no email evidence was written. Set comms.email.recipients[].address to the address the app sends to.`);
|
|
207
208
|
}
|
|
208
209
|
else {
|
|
209
210
|
// Zero captures is the silent-broken shape: the app never posted to the catch at
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ResidualConfig } from "../../../
|
|
1
|
+
import type { ResidualConfig } from "../../../study/plan-types.js";
|
|
2
2
|
import type { RunDesktopGeometry } from "../../../run/streams.js";
|
|
3
3
|
import type { DesktopBrowserFamily, DesktopBrowserLaunchIdentity } from "../../../substrates/e2b/desktop-browser.js";
|
|
4
4
|
import { makeChromeBrowserStateObserver } from "../../../substrates/e2b/desktop-cdp.js";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type E2BDesktopSandbox } from "../../../substrates/e2b/sdk.js";
|
|
2
2
|
import type { E2BParticipantContext, E2BParticipantState } from "./state.js";
|
|
3
|
-
export declare const
|
|
3
|
+
export declare const CUA_ACTOR_STUDY_PROVIDER_METADATA: {
|
|
4
4
|
readonly mode: "cua-actor-lab";
|
|
5
5
|
readonly tool: "humanish";
|
|
6
6
|
};
|
|
@@ -11,7 +11,7 @@ import { loadE2BDesktopModule } from "../../../substrates/e2b/sdk.js";
|
|
|
11
11
|
import { e2bShell } from "../../../substrates/e2b/shell.js";
|
|
12
12
|
import { attachReceivingInbox, participantCommsEnv, startCommsCatch } from "./comms.js";
|
|
13
13
|
import { participantSubjectEnv } from "../types.js";
|
|
14
|
-
export const
|
|
14
|
+
export const CUA_ACTOR_STUDY_PROVIDER_METADATA = {
|
|
15
15
|
mode: "cua-actor-lab",
|
|
16
16
|
tool: "humanish",
|
|
17
17
|
};
|
|
@@ -37,8 +37,8 @@ export async function acquireParticipantDesktop(ctx, state) {
|
|
|
37
37
|
requestTimeoutMs: deps.requestTimeoutMs,
|
|
38
38
|
timeoutMs: deps.sandboxMs,
|
|
39
39
|
metadata: {
|
|
40
|
-
...
|
|
41
|
-
labId: deps.
|
|
40
|
+
...CUA_ACTOR_STUDY_PROVIDER_METADATA,
|
|
41
|
+
labId: deps.studyId,
|
|
42
42
|
recordId: spec.recordId,
|
|
43
43
|
participantId: spec.planned.id,
|
|
44
44
|
participantIndex: String(spec.planned.index),
|
|
@@ -7,7 +7,6 @@ import { captureDesktopBrowserGeometry } from "../../../substrates/e2b/desktop-g
|
|
|
7
7
|
import { prepareDesktopMedia, startE2BDesktopMedia, } from "../../../substrates/e2b/desktop-media.js";
|
|
8
8
|
import { startE2BDesktopRecording } from "../../../substrates/e2b/desktop-recording.js";
|
|
9
9
|
import { applyParticipantMobileFidelity, mobileLaunchFlags } from "./fidelity.js";
|
|
10
|
-
import { streamEvent } from "../../../lab/run-lab-homes.js";
|
|
11
10
|
/** Start the declared speech worker and screen recording. A recording that cannot start is a warning. */
|
|
12
11
|
export async function startParticipantMedia(ctx, state, desktop) {
|
|
13
12
|
const { spec, deps, warnings } = ctx;
|
|
@@ -58,7 +57,7 @@ export async function openParticipantSurface(ctx, state, desktop) {
|
|
|
58
57
|
// uploaded first, and a feed that cannot be produced fails the participant closed here.
|
|
59
58
|
const mediaEvidence = requestedMedia === undefined
|
|
60
59
|
? undefined
|
|
61
|
-
: await prepareDesktopMedia(desktop, requestedMedia, residual.policies?.mediaPermission ?? "prompt", deps.
|
|
60
|
+
: await prepareDesktopMedia(desktop, requestedMedia, residual.policies?.mediaPermission ?? "prompt", deps.studyCwd, deps.requestTimeoutMs);
|
|
62
61
|
const emulationFlags = mobileLaunchFlags(residual, spec);
|
|
63
62
|
const browserLaunch = await openDesktopBrowserTarget(desktop, targetUrl, deps.requestTimeoutMs, residual.execution?.desktop?.browser, [...emulationFlags, ...(mediaEvidence?.flags ?? [])], state.speech?.env ?? state.recording?.env);
|
|
64
63
|
state.desktopBrowser =
|
|
@@ -169,14 +168,14 @@ export async function startParticipantStream(ctx, state, desktop) {
|
|
|
169
168
|
});
|
|
170
169
|
if (typeof candidateStreamUrl === "string" && candidateStreamUrl.trim().length > 0) {
|
|
171
170
|
state.streamUrl = candidateStreamUrl;
|
|
172
|
-
await deps.onStream(
|
|
171
|
+
await deps.onStream({
|
|
173
172
|
type: "ready",
|
|
174
173
|
participantId: spec.planned.id,
|
|
175
174
|
sandboxId: desktop.sandboxId,
|
|
176
175
|
recordId: spec.recordId,
|
|
177
176
|
streamId: spec.streamId,
|
|
178
177
|
url: candidateStreamUrl,
|
|
179
|
-
})
|
|
178
|
+
});
|
|
180
179
|
}
|
|
181
180
|
else {
|
|
182
181
|
warnings.push("Live desktop stream started but did not return a usable watch URL; Observer will fall back to screenshots.");
|
|
@@ -12,7 +12,7 @@ import type { E2BDesktopSandbox } from "../../../substrates/e2b/sdk.js";
|
|
|
12
12
|
import type { ParticipantDesktopEvidence } from "../participant-desktop.js";
|
|
13
13
|
import type { ParticipantComms, RunningCommsCatch } from "./comms.js";
|
|
14
14
|
import type { ParticipantFidelity } from "./fidelity.js";
|
|
15
|
-
import type {
|
|
15
|
+
import type { CuaActorStudyErrorCode, E2BDesktopDeps, DesktopParticipantRun, SandboxReleaseFact } from "../types.js";
|
|
16
16
|
/** What every step of one participant reads: its spec, the run's dependencies and where it points. */
|
|
17
17
|
export interface E2BParticipantContext {
|
|
18
18
|
readonly spec: DesktopParticipantRun;
|
|
@@ -38,7 +38,7 @@ export interface E2BParticipantState {
|
|
|
38
38
|
desktopResources: DesktopResourceObservation | undefined;
|
|
39
39
|
released: boolean;
|
|
40
40
|
sandboxRelease: SandboxReleaseFact | undefined;
|
|
41
|
-
failureCode:
|
|
41
|
+
failureCode: CuaActorStudyErrorCode | undefined;
|
|
42
42
|
commsCatch: RunningCommsCatch | undefined;
|
|
43
43
|
commsArtifactPath: string | undefined;
|
|
44
44
|
receivingInboxUrl: string | undefined;
|
|
@@ -7,7 +7,6 @@ import { readE2BRelease } from "../../../substrates/e2b/sandbox.js";
|
|
|
7
7
|
import { e2bShell } from "../../../substrates/e2b/shell.js";
|
|
8
8
|
import { drainCommsEvidence } from "./comms.js";
|
|
9
9
|
import { finalParticipantGeometry } from "./fidelity.js";
|
|
10
|
-
import { streamEvent } from "../../../lab/run-lab-homes.js";
|
|
11
10
|
/**
|
|
12
11
|
* Each route's own keep flag gates its own participants only: a clone.keep can never leak into a
|
|
13
12
|
* local-tree participant's teardown decision, and vice versa.
|
|
@@ -150,12 +149,12 @@ export async function finishE2BDesktop(ctx, state, failed) {
|
|
|
150
149
|
// break teardown.
|
|
151
150
|
if (state.streamUrl !== undefined) {
|
|
152
151
|
try {
|
|
153
|
-
await deps.onStream(
|
|
152
|
+
await deps.onStream({
|
|
154
153
|
type: "ended",
|
|
155
154
|
participantId: spec.planned.id,
|
|
156
155
|
recordId: spec.recordId,
|
|
157
156
|
streamId: spec.streamId,
|
|
158
|
-
})
|
|
157
|
+
});
|
|
159
158
|
}
|
|
160
159
|
catch {
|
|
161
160
|
// viewer-side only; nothing to record
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { StudyCommsEmail, StudyCommsExternal } from "../../study/types.js";
|
|
2
2
|
import type { PreparedRunArtifactPaths } from "../../run/paths.js";
|
|
3
3
|
import type { DesktopParticipantRun, ParticipantRunOutcome } from "./types.js";
|
|
4
4
|
/**
|
|
@@ -10,8 +10,8 @@ import type { DesktopParticipantRun, ParticipantRunOutcome } from "./types.js";
|
|
|
10
10
|
* Returns the warnings the drain produced; participants that received mail get `commsArtifactPath`.
|
|
11
11
|
*/
|
|
12
12
|
export declare function drainExternalComms(args: {
|
|
13
|
-
externalCommsConfig:
|
|
14
|
-
externalCommsEmail:
|
|
13
|
+
externalCommsConfig: StudyCommsExternal;
|
|
14
|
+
externalCommsEmail: StudyCommsEmail;
|
|
15
15
|
env: Record<string, string | undefined>;
|
|
16
16
|
runPaths: PreparedRunArtifactPaths;
|
|
17
17
|
participantRuns: readonly DesktopParticipantRun[];
|
|
@@ -10,6 +10,7 @@ import { participantPassed, participantStatus } from "../../run/judge.js";
|
|
|
10
10
|
import { buildRunCostSummary, desktopSpanToMinutes } from "../../run/cost-summary.js";
|
|
11
11
|
import { formatParticipantPlanEntry } from "./participant-runs.js";
|
|
12
12
|
import { fanoutParticipantRecords } from "./fanout-records.js";
|
|
13
|
+
import { plural } from "../../run/text.js";
|
|
13
14
|
/** The run's first two events: its creation and the fan-out plan. */
|
|
14
15
|
function fanoutPlanEvents(args) {
|
|
15
16
|
const { specs, plan } = args;
|
|
@@ -19,14 +20,14 @@ function fanoutPlanEvents(args) {
|
|
|
19
20
|
at: args.run.createdAt,
|
|
20
21
|
level: "info",
|
|
21
22
|
type: "cua-lab.run.created",
|
|
22
|
-
message: `Created computer-use fan-out run for ${plan.
|
|
23
|
+
message: `Created computer-use fan-out run for ${plan.studyId} (actor ${args.descriptor.id}, ${specs.length} participants, one world each).`,
|
|
23
24
|
});
|
|
24
25
|
events.push({
|
|
25
26
|
id: "event-001-fanout-plan",
|
|
26
27
|
at: args.run.createdAt,
|
|
27
28
|
level: "info",
|
|
28
29
|
type: "cua-lab.fanout.plan",
|
|
29
|
-
message: `Fan-out plan: ${args.participantPlan.laneCount
|
|
30
|
+
message: `Fan-out plan: ${plural(args.participantPlan.laneCount, "participant")} (${args.participantPlan.strategy}), concurrency ${args.participantPlan.concurrency}, ${plural(args.participantPlan.waves, "wave")}; session budget ${Math.round(args.participantPlan.perLaneSessionBudgetMs / 1000)}s per participant; worst-case ~${args.participantPlan.worstCaseSandboxMinutes} sandbox-minutes${args.dryRun ? " (dry-run: $0)" : ""}. Participants: ${args.participantPlan.lanes.map(formatParticipantPlanEntry).join(", ")}.`,
|
|
30
31
|
});
|
|
31
32
|
return events;
|
|
32
33
|
}
|
|
@@ -75,7 +76,7 @@ function fanoutReview(args, streams) {
|
|
|
75
76
|
? `Live computer-use fan-out is running (${specs.length} participants, one world each); terminal participant evidence has not been written yet.`
|
|
76
77
|
: args.dryRun
|
|
77
78
|
? `${args.rerun ? `Rerun from ${args.rerun.sourceRunId}: ` : ""}Dry-run fan-out: ${specs.length} participants composed for ${args.descriptor.id} against ${args.appUrl}, one world each; no desktops launched, $0 spend.`
|
|
78
|
-
: `${args.rerun ? `Rerun from ${args.rerun.sourceRunId}: ` : ""}Computer-use fan-out (${specs.length} participants, one world each): ${passedParticipants}/${specs.length
|
|
79
|
+
: `${args.rerun ? `Rerun from ${args.rerun.sourceRunId}: ` : ""}Computer-use fan-out (${specs.length} participants, one world each): ${passedParticipants}/${plural(specs.length, "participant")} reached a terminal, engaged verdict${participants ? `: ${formatParticipantOutcomes(participants, participantEndings)}` : ""}${runTasks ? `; tasks: ${formatRunTaskFunnel(runTasks)}` : ""}.`,
|
|
79
80
|
gaps: args.inProgress === true
|
|
80
81
|
? ["Live fan-out session is still running."]
|
|
81
82
|
: args.dryRun
|
|
@@ -136,8 +137,8 @@ function fanoutFeedbackCandidates(args) {
|
|
|
136
137
|
? []
|
|
137
138
|
: participantFeedbackCandidates({
|
|
138
139
|
runId: args.run.runId,
|
|
139
|
-
scenarioId: `cua-${plan.
|
|
140
|
-
adapterId: plan.
|
|
140
|
+
scenarioId: `cua-${plan.studyId}`,
|
|
141
|
+
adapterId: plan.studyId,
|
|
141
142
|
goal: redactText(specs[0].evidenceInstructions ?? specs[0].instructions),
|
|
142
143
|
substrate: plan.residual.execution?.target === "local" ? "local-desktop" : "e2b-desktop",
|
|
143
144
|
participants: specs.map((spec, index) => {
|
|
@@ -179,7 +180,7 @@ export function buildCuaFanoutBundle(args) {
|
|
|
179
180
|
at: args.run.createdAt,
|
|
180
181
|
level: "info",
|
|
181
182
|
type: "cua-lab.fanout.rerun",
|
|
182
|
-
message: `Rerun selected ${args.rerun.selectedLaneIds.length
|
|
183
|
+
message: `Rerun selected ${plural(args.rerun.selectedLaneIds.length, "participant")} from ${args.rerun.sourceRunId}: ${args.rerun.previous.map((prior) => `${prior.laneId} was ${prior.status}${prior.completionReason ? `/${prior.completionReason}` : ""}`).join(", ")}. This is a new linked run; the source run verdict is unchanged.`,
|
|
183
184
|
});
|
|
184
185
|
}
|
|
185
186
|
specs.forEach((spec, index) => {
|
|
@@ -218,12 +219,12 @@ export function buildCuaFanoutBundle(args) {
|
|
|
218
219
|
persona: {
|
|
219
220
|
id: specs[0].persona.id,
|
|
220
221
|
name: `Computer-use fan-out (${specs.length} participants)`,
|
|
221
|
-
source: `study:${plan.
|
|
222
|
+
source: `study:${plan.studyId}`,
|
|
222
223
|
sourceDigest: specs[0].persona.promptDigest,
|
|
223
224
|
},
|
|
224
225
|
scenario: {
|
|
225
|
-
id: `cua-${plan.
|
|
226
|
-
title: plan.title ?? `Computer-use fan-out: ${plan.
|
|
226
|
+
id: `cua-${plan.studyId}`,
|
|
227
|
+
title: plan.title ?? `Computer-use fan-out: ${plan.studyId}`,
|
|
227
228
|
// Redacted at write time, like every other raw-text surface in the bundle. Participant records are
|
|
228
229
|
// digest-only by design, but scenario.goal keeps one participant's composed instructions verbatim,
|
|
229
230
|
// and an adopter whose authored participant text must name a runtime world URL (an inbox on a route
|
|
@@ -233,7 +234,7 @@ export function buildCuaFanoutBundle(args) {
|
|
|
233
234
|
//
|
|
234
235
|
// The instructions the model actually receives are untouched; only the persisted copy changes.
|
|
235
236
|
goal: redactText(specs[0].evidenceInstructions ?? specs[0].instructions),
|
|
236
|
-
source: `study:${plan.
|
|
237
|
+
source: `study:${plan.studyId}`,
|
|
237
238
|
sourceDigest: specs[0].persona.promptDigest,
|
|
238
239
|
},
|
|
239
240
|
lifecycle: [
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Each participant's records in a computer-use fan-out bundle: its simulation, its stream, its
|
|
2
2
|
// subject provenance event and the events of its outcome (session, geometry warnings, phase trail).
|
|
3
|
-
import { participantAssignment } from "../../
|
|
3
|
+
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
4
4
|
import { participantEvent, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
5
5
|
import { declaredScreenForRender } from "../../substrates/e2b/desktop-geometry.js";
|
|
6
6
|
import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel } from "./bundle-parts.js";
|
|
@@ -66,7 +66,7 @@ function fanoutParticipantRecord(args, spec, index, view) {
|
|
|
66
66
|
const { outcome, publicTargetUrl, session, status, reason } = view;
|
|
67
67
|
return participantRecord(spec, index + 1, {
|
|
68
68
|
personaId: spec.persona.id,
|
|
69
|
-
scenarioId: `cua-${plan.
|
|
69
|
+
scenarioId: `cua-${plan.studyId}`,
|
|
70
70
|
status,
|
|
71
71
|
streamKind: "browser",
|
|
72
72
|
mode: "browser-sim",
|
|
@@ -2,7 +2,7 @@ import type { ActorTokenUsage, ActorTraceItem } from "../../actors/contract.js";
|
|
|
2
2
|
import type { CuaLiveMetadata } from "../../actors/computer-use/loop.js";
|
|
3
3
|
import { type ObserverResult } from "../../observer/render.js";
|
|
4
4
|
import type { RunBundle } from "../../run/bundle.js";
|
|
5
|
-
import type {
|
|
5
|
+
import type { RunStudyHomes } from "../../study/run-study-homes.js";
|
|
6
6
|
import type { DesktopParticipantRun } from "./types.js";
|
|
7
7
|
export interface LiveTraceFlush {
|
|
8
8
|
/** Record a participant's recorded-so-far items; the bundle rewrite follows on the flush
|
|
@@ -32,9 +32,9 @@ export declare function startLiveTraceFlush(args: {
|
|
|
32
32
|
* the run. The URLs carry auth, so they live in memory and on the Observer result, never in run
|
|
33
33
|
* artifacts.
|
|
34
34
|
*/
|
|
35
|
-
export declare function trackRuntimeStreams(onStream:
|
|
35
|
+
export declare function trackRuntimeStreams(onStream: RunStudyHomes["onStream"] | undefined): {
|
|
36
36
|
/** Records each stream for the Observer, after the caller's onStream. */
|
|
37
|
-
onStream: NonNullable<
|
|
37
|
+
onStream: NonNullable<RunStudyHomes["onStream"]>;
|
|
38
38
|
/** From now on, keep this live Observer's stream list current. */
|
|
39
39
|
showIn: (observer: ObserverResult & {
|
|
40
40
|
ok: true;
|
|
@@ -5,7 +5,7 @@ import type { CuaParticipantsSetup, CuaRunSetup } from "./setup.js";
|
|
|
5
5
|
* Runs the participants (a dry run runs none). Returns the refusal when real email setup fails, or
|
|
6
6
|
* the participant outcomes and the warnings the finish records.
|
|
7
7
|
*/
|
|
8
|
-
export declare function
|
|
8
|
+
export declare function runStudyParticipants(setup: CuaRunSetup, participants: CuaParticipantsSetup): Promise<{
|
|
9
9
|
ok: false;
|
|
10
10
|
result: {
|
|
11
11
|
automaticAnalysis?: import("../../analysis/job.js").AutomaticAnalysisOutcome;
|
|
@@ -13,7 +13,6 @@ export declare function runLabParticipants(setup: CuaRunSetup, participants: Cua
|
|
|
13
13
|
schema: typeof import("../../run/study-result.js").STUDY_RESULT_SCHEMA;
|
|
14
14
|
route: "computer-use";
|
|
15
15
|
studyId: string;
|
|
16
|
-
labId: string;
|
|
17
16
|
ok: boolean;
|
|
18
17
|
cwd: string;
|
|
19
18
|
actor: string;
|
|
@@ -40,7 +39,7 @@ export declare function runLabParticipants(setup: CuaRunSetup, participants: Cua
|
|
|
40
39
|
diagnostics?: import("./diagnostics.js").CuaDiagnostics;
|
|
41
40
|
warnings: string[];
|
|
42
41
|
error?: {
|
|
43
|
-
code: import("./types.js").
|
|
42
|
+
code: import("./types.js").CuaActorStudyErrorCode;
|
|
44
43
|
message: string;
|
|
45
44
|
};
|
|
46
45
|
runId: string;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// receiving, the participants themselves, and the drain of an operator-hosted inbox.
|
|
3
3
|
import { prepareReceivingRun } from "../../comms/receiving-runtime.js";
|
|
4
4
|
import { liveObserverResult } from "../../observer/live.js";
|
|
5
|
-
import { pricedModel } from "../../
|
|
5
|
+
import { pricedModel } from "../../study/plan-base.js";
|
|
6
6
|
import { runAllCuaParticipants } from "./participant-execution.js";
|
|
7
7
|
import { startLiveTraceFlush } from "./live-flush.js";
|
|
8
8
|
import { drainExternalComms } from "./external-comms.js";
|
|
@@ -12,7 +12,7 @@ import { participantSubjectEnv } from "./types.js";
|
|
|
12
12
|
* Runs the participants (a dry run runs none). Returns the refusal when real email setup fails, or
|
|
13
13
|
* the participant outcomes and the warnings the finish records.
|
|
14
14
|
*/
|
|
15
|
-
export async function
|
|
15
|
+
export async function runStudyParticipants(setup, participants) {
|
|
16
16
|
const { plan, input, cwd, streams, descriptor, run } = setup;
|
|
17
17
|
const { participantRuns, participantPlan, bundleBase } = setup;
|
|
18
18
|
const { env, knownSecretValues, deps, liveTrace, externalComms, inProgress, fail } = participants;
|
|
@@ -22,7 +22,7 @@ export async function runLabParticipants(setup, participants) {
|
|
|
22
22
|
const { runId, paths: runPaths } = run;
|
|
23
23
|
// A live run writes what it is doing as it does it, whether or not anyone is currently watching.
|
|
24
24
|
// This is not gated on `options.onObserverReady` (the interactive Observer callback): gated, a
|
|
25
|
-
// run launched by an agent (`
|
|
25
|
+
// run launched by an agent (`run --json`), detached, or from the terminal surface records
|
|
26
26
|
// nothing at all until it completes, and anything asking "what is this participant doing right
|
|
27
27
|
// now" got silence for the whole run. Who reads the evidence is not the run's business; the
|
|
28
28
|
// callback below stays conditional, the writing does not.
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { StudyConfig } from "../../study/types.js";
|
|
2
2
|
import { type LocalTreeArchive } from "../../subject/local-tree-archive.js";
|
|
3
|
-
import type {
|
|
3
|
+
import type { StudyDeps } from "../../study/study-deps.js";
|
|
4
4
|
/**
|
|
5
5
|
* Pack the working tree for a local-tree run and report what left the host on stderr, by counts
|
|
6
6
|
* and digest only, never paths or file names.
|
|
7
7
|
*/
|
|
8
|
-
export declare function packRunLocalTree(deps:
|
|
9
|
-
readonly subject: Pick<
|
|
8
|
+
export declare function packRunLocalTree(deps: StudyDeps, config: {
|
|
9
|
+
readonly subject: Pick<StudyConfig["subject"], "localTree">;
|
|
10
10
|
}, cwd: string): Promise<{
|
|
11
11
|
archive: LocalTreeArchive;
|
|
12
12
|
buffer: ArrayBuffer;
|
|
@@ -14,7 +14,7 @@ export declare function packRunLocalTree(deps: LabDeps, config: {
|
|
|
14
14
|
/**
|
|
15
15
|
* Default local-tree packing implementation: createLocalTreeArchive(root, opts) on the host,
|
|
16
16
|
* then a single read of the produced archive file into an ArrayBuffer for upload. The DI seam
|
|
17
|
-
* (
|
|
17
|
+
* (StudyDeps.packLocalTree) overrides this in deterministic tests so they never require
|
|
18
18
|
* tar/git.
|
|
19
19
|
*/
|
|
20
20
|
export declare function defaultPackLocalTree(args: {
|
|
@@ -24,7 +24,7 @@ export async function packRunLocalTree(deps, config, cwd) {
|
|
|
24
24
|
/**
|
|
25
25
|
* Default local-tree packing implementation: createLocalTreeArchive(root, opts) on the host,
|
|
26
26
|
* then a single read of the produced archive file into an ArrayBuffer for upload. The DI seam
|
|
27
|
-
* (
|
|
27
|
+
* (StudyDeps.packLocalTree) overrides this in deterministic tests so they never require
|
|
28
28
|
* tar/git.
|
|
29
29
|
*/
|
|
30
30
|
export async function defaultPackLocalTree(args) {
|
|
@@ -1,23 +1,23 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
1
|
+
import type { InternalRunStudyOptions } from "../../run-study.js";
|
|
2
|
+
import type { StudyConfig } from "../../study/types.js";
|
|
3
3
|
import type { LocalVmInput } from "./types.js";
|
|
4
4
|
import { type LocalFirecrackerAssets } from "../../substrates/local/firecracker-desktop.js";
|
|
5
|
-
type LocalVmRunOptions =
|
|
6
|
-
config:
|
|
5
|
+
type LocalVmRunOptions = InternalRunStudyOptions & {
|
|
6
|
+
config: StudyConfig;
|
|
7
7
|
assets?: LocalFirecrackerAssets;
|
|
8
8
|
signal?: AbortSignal;
|
|
9
9
|
};
|
|
10
10
|
/** A local browser study's bindings, and the cleanup of what its participants started. */
|
|
11
11
|
export interface LocalVmRun {
|
|
12
|
-
/** The caller's
|
|
13
|
-
readonly options:
|
|
12
|
+
/** The caller's runStudyWith options, with the account provider when the study runs one. */
|
|
13
|
+
readonly options: InternalRunStudyOptions;
|
|
14
14
|
/** The desktop, the analysis gate and the abort signal the computer-use run takes. */
|
|
15
15
|
readonly localVm: LocalVmInput;
|
|
16
16
|
close(): Promise<void>;
|
|
17
17
|
}
|
|
18
18
|
/**
|
|
19
|
-
* Local desktop/provider composition: the options the
|
|
20
|
-
* anything starts on a
|
|
19
|
+
* Local desktop/provider composition: the options the study is planned and run with. It throws before
|
|
20
|
+
* anything starts on a study the local VM cannot run.
|
|
21
21
|
*/
|
|
22
22
|
export declare function prepareLocalVmRun(options: LocalVmRunOptions): LocalVmRun;
|
|
23
23
|
export {};
|
|
@@ -152,8 +152,8 @@ function accountProvider(state) {
|
|
|
152
152
|
};
|
|
153
153
|
}
|
|
154
154
|
/**
|
|
155
|
-
* Local desktop/provider composition: the options the
|
|
156
|
-
* anything starts on a
|
|
155
|
+
* Local desktop/provider composition: the options the study is planned and run with. It throws before
|
|
156
|
+
* anything starts on a study the local VM cannot run.
|
|
157
157
|
*/
|
|
158
158
|
export function prepareLocalVmRun(options) {
|
|
159
159
|
const config = localBrowserDefaults(options.config);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { CuaExecutor } from "../../actors/computer-use/loop.js";
|
|
2
2
|
import type { DesktopParticipantRun, ParticipantRunOutcome } from "./types.js";
|
|
3
|
-
import type {
|
|
3
|
+
import type { StudyCommsEmail, StudyCommsRecipient } from "../../study/types.js";
|
|
4
4
|
import type { PreparedOutputRoot } from "../../run/contained-output.js";
|
|
5
5
|
/** A prepared desktop supplies only participant input/observation and its inbox location. */
|
|
6
6
|
export interface ReadyParticipantDesktop {
|
|
@@ -41,8 +41,8 @@ export type ParticipantDesktopFactory = (run: DesktopParticipantRun, warnings: s
|
|
|
41
41
|
/** The participant's addressed comms recipient, when one exists: the gate and the address source for the
|
|
42
42
|
* inbox instruction. A participant told to check an inbox it can never receive into would stall,
|
|
43
43
|
* so no addressed recipient means no instruction. */
|
|
44
|
-
export declare function inboxRecipientFor(commsEmail:
|
|
44
|
+
export declare function inboxRecipientFor(commsEmail: StudyCommsEmail, participantId: string): StudyCommsRecipient | undefined;
|
|
45
45
|
/** True when a participant has a declared comms recipient with an address, so the drain can actually match the
|
|
46
46
|
* mail the persona will be told to read. Gates the inbox instruction to participants that can receive
|
|
47
47
|
* mail; a participant told to check an inbox it can never receive into would just stall. */
|
|
48
|
-
export declare function participantHasInboxRecipient(commsEmail:
|
|
48
|
+
export declare function participantHasInboxRecipient(commsEmail: StudyCommsEmail, participantId: string): boolean;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { recipientParticipantId } from "../../
|
|
1
|
+
import { recipientParticipantId } from "../../study/parse/comms.js";
|
|
2
2
|
/** The participant's addressed comms recipient, when one exists: the gate and the address source for the
|
|
3
3
|
* inbox instruction. A participant told to check an inbox it can never receive into would stall,
|
|
4
4
|
* so no addressed recipient means no instruction. */
|
|
@@ -309,7 +309,7 @@ export function toParticipantResult(spec, outcome, subject, dryRun) {
|
|
|
309
309
|
? `Computer-use session ended with a harness error: ${session.reason}`
|
|
310
310
|
: session?.status !== "passed"
|
|
311
311
|
? `Computer-use session ended with ${session?.status ?? "unknown"}: ${session?.reason ?? "no terminal reason"}`
|
|
312
|
-
: "
|
|
312
|
+
: "The computer-use run did not produce a terminal session."),
|
|
313
313
|
},
|
|
314
314
|
}),
|
|
315
315
|
};
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// finished session. The desktop side stays with the participant runner.
|
|
4
4
|
import { startClaudeSession } from "../../actors/local-agent/claude-session.js";
|
|
5
5
|
import { createLocalAgentProvider } from "../../actors/local-agent/cli.js";
|
|
6
|
-
import { pricedModel } from "../../
|
|
6
|
+
import { pricedModel } from "../../study/plan-base.js";
|
|
7
7
|
import { estimateActorCostForExecution, round6 } from "../../run/pricing.js";
|
|
8
8
|
import { redactText } from "../../evidence/redaction.js";
|
|
9
9
|
import { createRestrictedCodexParticipant, } from "../../actors/codex/restricted-participant.js";
|
|
@@ -54,7 +54,7 @@ export async function startParticipantModel(spec, deps, executor) {
|
|
|
54
54
|
// started with no memory of the last. HUMANISH_LOCAL_AGENT_ONE_SHOT=1 keeps that path
|
|
55
55
|
// reachable as a measurement switch: MemTrapBench (2026-08) reports memory frameworks
|
|
56
56
|
// degrading agent performance by 10-40% on some tasks, so "remembers" has to be measured
|
|
57
|
-
// against "does not" on the same
|
|
57
|
+
// against "does not" on the same study, not assumed. The trace records which one ran.
|
|
58
58
|
const oneShot = env.HUMANISH_LOCAL_AGENT_ONE_SHOT !== undefined &&
|
|
59
59
|
env.HUMANISH_LOCAL_AGENT_ONE_SHOT !== "" &&
|
|
60
60
|
env.HUMANISH_LOCAL_AGENT_ONE_SHOT !== "0";
|
|
@@ -208,7 +208,7 @@ export async function closeParticipantModel(model, warnings) {
|
|
|
208
208
|
/** Prices the finished session's tokens onto its trace, writes the trace, and warns on raw screenshots. */
|
|
209
209
|
export async function recordParticipantTrace(spec, deps, session, warnings) {
|
|
210
210
|
// Per-participant model-token cost estimate, attached to the trace before it is persisted (the model
|
|
211
|
-
// id is authoritative here: provider.version). Kept at the
|
|
211
|
+
// id is authoritative here: provider.version). Kept at the study boundary so the pure loop
|
|
212
212
|
// never depends on the operator rate table. estimateActorCost declares absent (null) for an
|
|
213
213
|
// unknown rate / missing usage rather than guessing.
|
|
214
214
|
session.trace.estimatedCost = estimateActorCostForExecution(session.trace.tokenUsage, session.trace.ids.model, session.trace.executionProfile);
|