humanish 0.106.1 → 0.107.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -4
- package/ARCHITECTURE.md +17 -16
- package/CHANGELOG.md +14 -0
- package/CONTEXT.md +42 -39
- package/CONTRIBUTING.md +8 -8
- package/README.md +8 -3
- package/dist/actors/codex/app-server-client.js +2 -1
- package/dist/actors/codex/app-server-trace.d.ts +0 -1
- package/dist/actors/codex/app-server-trace.js +1 -3
- package/dist/actors/codex/app-server.js +2 -1
- package/dist/actors/codex/protocol-compat.d.ts +27 -0
- package/dist/actors/codex/protocol-compat.js +357 -0
- package/dist/actors/codex/protocol-contract.d.ts +61 -0
- package/dist/actors/codex/protocol-contract.js +471 -0
- package/dist/actors/codex/qualified-versions.d.ts +30 -3
- package/dist/actors/codex/qualified-versions.js +50 -5
- package/dist/actors/codex/restricted-executable.d.ts +24 -0
- package/dist/actors/codex/restricted-executable.js +97 -22
- package/dist/actors/codex/restricted-launch.d.ts +7 -1
- package/dist/actors/codex/restricted-launch.js +42 -3
- package/dist/actors/codex/restricted-notifications.d.ts +101 -0
- package/dist/actors/codex/restricted-notifications.js +277 -0
- package/dist/actors/codex/restricted-participant.d.ts +30 -0
- package/dist/actors/codex/restricted-participant.js +60 -2
- package/dist/actors/codex/restricted-policy.d.ts +9 -1
- package/dist/actors/codex/restricted-session.d.ts +14 -1
- package/dist/actors/codex/restricted-session.js +50 -17
- package/dist/actors/codex/restricted-transport.d.ts +29 -0
- package/dist/actors/codex/restricted-transport.js +139 -27
- package/dist/actors/codex/restricted-turn.d.ts +19 -4
- package/dist/actors/codex/restricted-turn.js +43 -30
- package/dist/actors/computer-use/actor.d.ts +1 -1
- package/dist/actors/computer-use/actor.js +4 -4
- package/dist/actors/computer-use/loop/backstop.js +1 -1
- package/dist/actors/computer-use/loop/ending.d.ts +2 -2
- package/dist/actors/computer-use/loop/ending.js +2 -2
- package/dist/actors/computer-use/loop/spend.d.ts +1 -1
- package/dist/actors/computer-use/loop/spend.js +3 -3
- package/dist/actors/computer-use/loop/trace.js +2 -2
- package/dist/actors/computer-use/loop/types.d.ts +14 -5
- package/dist/actors/computer-use/loop.js +4 -4
- package/dist/actors/computer-use/openai-provider.d.ts +1 -1
- package/dist/actors/computer-use/openai-provider.js +2 -2
- package/dist/actors/contract.d.ts +9 -7
- package/dist/actors/local-agent/cli.js +1 -1
- package/dist/actors/scripted-browser/actor.d.ts +2 -2
- package/dist/actors/scripted-browser/journey.js +15 -4
- package/dist/actors/scripted-browser/steps.js +18 -0
- package/dist/actors/scripted-browser/types.d.ts +11 -1
- package/dist/actors/stop-conditions.js +1 -3
- package/dist/analysis/automatic-completion.d.ts +13 -8
- package/dist/analysis/automatic-completion.js +14 -8
- package/dist/analysis/automatic-config.d.ts +1 -1
- package/dist/analysis/automatic-config.js +2 -2
- package/dist/analysis/automatic.d.ts +10 -4
- package/dist/analysis/automatic.js +17 -13
- package/dist/analysis/diagnostics.d.ts +23 -0
- package/dist/analysis/diagnostics.js +112 -0
- package/dist/analysis/evidence-sources.js +2 -2
- package/dist/analysis/evidence.js +6 -6
- package/dist/analysis/execute.d.ts +7 -1
- package/dist/analysis/execute.js +39 -12
- package/dist/analysis/job.d.ts +8 -5
- package/dist/analysis/job.js +15 -14
- package/dist/analysis/load.d.ts +1 -1
- package/dist/analysis/load.js +5 -5
- package/dist/analysis/provider.d.ts +9 -1
- package/dist/analysis/restricted-codex.d.ts +2 -0
- package/dist/analysis/restricted-codex.js +10 -1
- package/dist/analysis/service.d.ts +2 -0
- package/dist/analysis/service.js +31 -12
- package/dist/analysis/sharing.d.ts +2 -2
- package/dist/analysis/sharing.js +5 -5
- package/dist/analysis/store-executions.d.ts +5 -5
- package/dist/analysis/store-executions.js +12 -12
- package/dist/analysis/store.d.ts +4 -4
- package/dist/analysis/store.js +13 -13
- package/dist/cli/commands/analysis-signals.d.ts +2 -1
- package/dist/cli/commands/analysis-signals.js +4 -1
- package/dist/cli/commands/analyze.js +4 -2
- package/dist/cli/commands/comms.js +8 -1
- package/dist/cli/commands/lab-route-computer-use.d.ts +0 -2
- package/dist/cli/commands/lab-route-computer-use.js +26 -14
- package/dist/cli/commands/lab-route-preview.d.ts +1 -3
- package/dist/cli/commands/lab-route-preview.js +3 -4
- package/dist/cli/commands/lab-route-run.d.ts +6 -4
- package/dist/cli/commands/lab-route-run.js +6 -2
- package/dist/cli/commands/lab-route-scripted.d.ts +0 -2
- package/dist/cli/commands/lab-route-scripted.js +0 -1
- package/dist/cli/commands/lab-route-shared-world.d.ts +0 -2
- package/dist/cli/commands/lab-route-shared-world.js +0 -1
- package/dist/cli/commands/lab-route-terminal.d.ts +0 -2
- package/dist/cli/commands/lab-route-terminal.js +0 -1
- package/dist/cli/commands/lab-run.js +76 -45
- package/dist/cli/commands/lab-scorer.d.ts +0 -2
- package/dist/cli/commands/lab.js +12 -6
- package/dist/cli/commands/observe.js +22 -0
- package/dist/cli/commands/run-signals.d.ts +30 -0
- package/dist/cli/commands/run-signals.js +146 -0
- package/dist/cli/commands/runs.js +10 -8
- package/dist/cli/commands/watch.js +12 -10
- package/dist/cli/doctor.d.ts +3 -0
- package/dist/cli/doctor.js +10 -2
- package/dist/cli/io.d.ts +5 -4
- package/dist/cli/io.js +12 -8
- package/dist/cli/program.js +3 -3
- package/dist/cli/renamed-options.d.ts +11 -0
- package/dist/cli/renamed-options.js +18 -0
- package/dist/cli/telemetry.d.ts +2 -2
- package/dist/cli/telemetry.js +1 -1
- package/dist/comms/capture-surface.d.ts +1 -1
- package/dist/comms/catch-host.d.ts +1 -1
- package/dist/comms/connections.js +4 -4
- package/dist/comms/external-evidence.d.ts +27 -0
- package/dist/comms/external-evidence.js +93 -0
- package/dist/comms/lease-store.js +7 -7
- package/dist/comms/types.js +1 -1
- package/dist/evidence/browser-hygiene.d.ts +1 -1
- package/dist/evidence/browser-hygiene.js +1 -1
- package/dist/evidence/redaction.js +1 -1
- package/dist/evidence/secret-scrub.d.ts +9 -0
- package/dist/evidence/secret-scrub.js +96 -0
- package/dist/feedback/draft.js +7 -9
- package/dist/feedback/export-bundle.js +4 -4
- package/dist/feedback/export.js +9 -9
- package/dist/feedback/feedback.d.ts +1 -1
- package/dist/index.d.ts +3 -6
- package/dist/index.js +3 -6
- package/dist/keys/key-resolution.d.ts +3 -1
- package/dist/keys/key-resolution.js +18 -16
- package/dist/lab/adapter-extension.d.ts +28 -6
- package/dist/lab/adapter-extension.js +65 -126
- package/dist/lab/adapter-scorer-loader.d.ts +25 -5
- package/dist/lab/adapter-scorer-loader.js +24 -0
- package/dist/lab/composition-rules.js +20 -20
- package/dist/lab/config.js +28 -26
- package/dist/lab/device-presets.d.ts +2 -2
- package/dist/lab/discover.js +1 -3
- package/dist/lab/doctor.d.ts +44 -9
- package/dist/lab/doctor.js +149 -41
- package/dist/lab/init-plan.js +1 -3
- package/dist/lab/keys.js +1 -1
- package/dist/lab/lab-deps.d.ts +85 -0
- package/dist/lab/lab-deps.js +4 -0
- package/dist/lab/parse/actors.js +8 -7
- package/dist/lab/parse/comms.js +3 -2
- package/dist/lab/parse/execution.js +3 -2
- package/dist/lab/parse/subject-state.js +2 -1
- package/dist/lab/parse/subject.js +2 -1
- package/dist/lab/parse/values.d.ts +0 -1
- package/dist/lab/parse/values.js +0 -3
- package/dist/lab/persona-resolve.d.ts +3 -3
- package/dist/lab/persona-resolve.js +4 -4
- package/dist/lab/persona.js +1 -3
- package/dist/lab/plan-base.d.ts +0 -2
- package/dist/lab/plan-base.js +6 -4
- package/dist/lab/plan-participants.d.ts +5 -5
- package/dist/lab/plan-participants.js +2 -2
- package/dist/lab/plan-types.d.ts +22 -4
- package/dist/lab/plan.d.ts +3 -11
- package/dist/lab/plan.js +18 -42
- package/dist/lab/preflight-result.js +0 -2
- package/dist/lab/preflight.d.ts +0 -3
- package/dist/lab/preflight.js +1 -1
- package/dist/lab/requirements.d.ts +20 -0
- package/dist/lab/requirements.js +54 -0
- package/dist/lab/route-inputs.d.ts +23 -12
- package/dist/lab/route-inputs.js +48 -51
- package/dist/lab/routing.d.ts +19 -24
- package/dist/lab/routing.js +14 -47
- package/dist/lab/run-lab-events.d.ts +6 -11
- package/dist/lab/run-lab-events.js +3 -11
- package/dist/lab/run-lab-homes.d.ts +64 -0
- package/dist/lab/run-lab-homes.js +3 -0
- package/dist/lab/run-lab-options.d.ts +12 -66
- package/dist/lab/run-lab-options.js +45 -275
- package/dist/lab/summary.d.ts +9 -4
- package/dist/lab/summary.js +15 -8
- package/dist/lab/types.d.ts +132 -132
- package/dist/lab/validation.d.ts +15 -15
- package/dist/lab/validation.js +28 -28
- package/dist/lab/warnings.js +20 -19
- package/dist/observer/artifact.d.ts +1 -1
- package/dist/observer/artifact.js +3 -3
- package/dist/observer/data.js +2 -2
- package/dist/observer/render.js +2 -2
- package/dist/observer/run-routes.d.ts +1 -1
- package/dist/observer/run-routes.js +5 -5
- package/dist/observer/serve.d.ts +17 -0
- package/dist/observer/serve.js +34 -3
- package/dist/routes/computer-use/bundle-parts.d.ts +1 -1
- package/dist/routes/computer-use/bundle-parts.js +10 -3
- package/dist/routes/computer-use/bundle.d.ts +4 -11
- package/dist/routes/computer-use/bundle.js +9 -15
- package/dist/routes/computer-use/diagnostics.d.ts +1 -1
- package/dist/routes/computer-use/diagnostics.js +1 -1
- package/dist/routes/computer-use/e2b-desktop/comms.d.ts +10 -10
- package/dist/routes/computer-use/e2b-desktop/comms.js +11 -14
- package/dist/routes/computer-use/e2b-desktop/desktop.d.ts +2 -2
- package/dist/routes/computer-use/e2b-desktop/desktop.js +4 -7
- package/dist/routes/computer-use/e2b-desktop/fidelity.d.ts +13 -13
- package/dist/routes/computer-use/e2b-desktop/fidelity.js +11 -11
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +3 -3
- package/dist/routes/computer-use/e2b-desktop/prepare.js +33 -36
- package/dist/routes/computer-use/e2b-desktop/start.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/start.js +24 -23
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +5 -4
- package/dist/routes/computer-use/e2b-desktop/state.js +5 -3
- package/dist/routes/computer-use/e2b-desktop/teardown.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/teardown.js +29 -17
- package/dist/routes/computer-use/external-comms.d.ts +4 -4
- package/dist/routes/computer-use/external-comms.js +19 -43
- package/dist/routes/computer-use/fanout-bundle.d.ts +3 -3
- package/dist/routes/computer-use/fanout-bundle.js +25 -29
- package/dist/routes/computer-use/{fanout-lanes.js → fanout-records.js} +25 -19
- package/dist/routes/computer-use/in-process-desktop.d.ts +3 -10
- package/dist/routes/computer-use/in-process-desktop.js +3 -9
- package/dist/routes/computer-use/live-flush.d.ts +7 -5
- package/dist/routes/computer-use/live-flush.js +19 -27
- package/dist/routes/computer-use/{run-lanes.js → live-phase.js} +12 -11
- package/dist/routes/computer-use/local-tree-pack.d.ts +3 -3
- package/dist/routes/computer-use/local-tree-pack.js +4 -5
- package/dist/routes/computer-use/local-vm.d.ts +11 -8
- package/dist/routes/computer-use/local-vm.js +39 -66
- package/dist/routes/computer-use/participant-desktop.d.ts +8 -23
- package/dist/routes/computer-use/participant-desktop.js +5 -25
- package/dist/routes/computer-use/{lanes.d.ts → participant-execution.d.ts} +10 -10
- package/dist/routes/computer-use/{lanes.js → participant-execution.js} +35 -22
- package/dist/routes/computer-use/participant-facts.d.ts +7 -1
- package/dist/routes/computer-use/participant-facts.js +12 -1
- package/dist/routes/computer-use/participant-model.d.ts +19 -11
- package/dist/routes/computer-use/participant-model.js +49 -25
- package/dist/routes/computer-use/participant-prompt.d.ts +3 -3
- package/dist/routes/computer-use/participant-prompt.js +4 -4
- package/dist/routes/computer-use/participant-runs.d.ts +6 -6
- package/dist/routes/computer-use/participant-runs.js +8 -8
- package/dist/routes/computer-use/plan.d.ts +21 -29
- package/dist/routes/computer-use/plan.js +52 -60
- package/dist/routes/computer-use/preflight.d.ts +3 -5
- package/dist/routes/computer-use/preflight.js +20 -16
- package/dist/routes/computer-use/rerun-selection.js +10 -6
- package/dist/routes/computer-use/result.d.ts +1 -1
- package/dist/routes/computer-use/result.js +40 -30
- package/dist/routes/computer-use/route.d.ts +3 -4
- package/dist/routes/computer-use/route.js +21 -19
- package/dist/routes/computer-use/self-report.d.ts +2 -2
- package/dist/routes/computer-use/self-report.js +4 -4
- package/dist/routes/computer-use/setup.d.ts +8 -14
- package/dist/routes/computer-use/setup.js +96 -72
- package/dist/routes/computer-use/single-bundle.d.ts +8 -11
- package/dist/routes/computer-use/single-bundle.js +20 -25
- package/dist/routes/computer-use/subject-projection.d.ts +5 -7
- package/dist/routes/computer-use/subject-projection.js +20 -19
- package/dist/routes/computer-use/types.d.ts +169 -210
- package/dist/routes/computer-use/types.js +13 -5
- package/dist/routes/preview.js +0 -1
- package/dist/routes/scripted/bundle.d.ts +3 -6
- package/dist/routes/scripted/bundle.js +11 -16
- package/dist/routes/scripted/plan.d.ts +5 -7
- package/dist/routes/scripted/plan.js +6 -4
- package/dist/routes/scripted/result.d.ts +4 -1
- package/dist/routes/scripted/result.js +10 -6
- package/dist/routes/scripted/route.js +10 -11
- package/dist/routes/scripted/scenario.d.ts +1 -1
- package/dist/routes/scripted/scenario.js +1 -1
- package/dist/routes/scripted/setup.d.ts +3 -2
- package/dist/routes/scripted/setup.js +22 -21
- package/dist/routes/scripted/subject.d.ts +8 -3
- package/dist/routes/scripted/subject.js +9 -5
- package/dist/routes/scripted/surface-records.js +1 -1
- package/dist/routes/scripted/surface-sessions.d.ts +2 -2
- package/dist/routes/scripted/surface-sessions.js +4 -4
- package/dist/routes/scripted/types.d.ts +19 -36
- package/dist/routes/scripted/types.js +1 -2
- package/dist/routes/shared-world/{seat-records.js → bundle-records.js} +8 -8
- package/dist/routes/shared-world/bundle.d.ts +2 -2
- package/dist/routes/shared-world/bundle.js +22 -26
- package/dist/routes/shared-world/checkpoints.d.ts +4 -4
- package/dist/routes/shared-world/checkpoints.js +4 -4
- package/dist/routes/shared-world/comms.d.ts +5 -4
- package/dist/routes/shared-world/comms.js +28 -41
- package/dist/routes/shared-world/external-public.d.ts +5 -5
- package/dist/routes/shared-world/external-public.js +19 -20
- package/dist/routes/shared-world/handoff.d.ts +4 -4
- package/dist/routes/shared-world/handoff.js +50 -50
- package/dist/routes/shared-world/lobby-code.d.ts +5 -5
- package/dist/routes/shared-world/lobby-code.js +13 -13
- package/dist/routes/shared-world/{seats.d.ts → participant-specs.d.ts} +8 -6
- package/dist/routes/shared-world/{seats.js → participant-specs.js} +43 -48
- package/dist/routes/shared-world/plan.d.ts +2 -6
- package/dist/routes/shared-world/plan.js +1 -2
- package/dist/routes/shared-world/provenance.d.ts +4 -4
- package/dist/routes/shared-world/provenance.js +4 -4
- package/dist/routes/shared-world/provisioned.d.ts +3 -2
- package/dist/routes/shared-world/provisioned.js +46 -40
- package/dist/routes/shared-world/result.js +31 -19
- package/dist/routes/shared-world/route.js +37 -41
- package/dist/routes/shared-world/setup.d.ts +5 -4
- package/dist/routes/shared-world/setup.js +12 -14
- package/dist/routes/shared-world/types.d.ts +53 -126
- package/dist/routes/shared-world/types.js +2 -2
- package/dist/routes/terminal/adapter.d.ts +8 -8
- package/dist/routes/terminal/adapter.js +28 -125
- package/dist/routes/terminal/bundle.d.ts +7 -12
- package/dist/routes/terminal/bundle.js +24 -34
- package/dist/routes/terminal/credentials.d.ts +2 -2
- package/dist/routes/terminal/credentials.js +16 -16
- package/dist/routes/terminal/dry-run.d.ts +1 -1
- package/dist/routes/terminal/dry-run.js +9 -11
- package/dist/routes/terminal/encoding.d.ts +1 -1
- package/dist/routes/terminal/encoding.js +2 -2
- package/dist/routes/terminal/ledger.d.ts +6 -6
- package/dist/routes/terminal/ledger.js +9 -9
- package/dist/routes/terminal/live-finish.d.ts +1 -2
- package/dist/routes/terminal/live-finish.js +21 -20
- package/dist/routes/terminal/live-sandbox.d.ts +3 -2
- package/dist/routes/terminal/live-sandbox.js +20 -20
- package/dist/routes/terminal/participant-text.d.ts +23 -0
- package/dist/routes/terminal/participant-text.js +165 -0
- package/dist/routes/terminal/plan.d.ts +2 -6
- package/dist/routes/terminal/plan.js +13 -9
- package/dist/routes/terminal/recorder.d.ts +6 -1
- package/dist/routes/terminal/recorder.js +12 -4
- package/dist/routes/terminal/result.js +4 -4
- package/dist/routes/terminal/route.js +26 -26
- package/dist/routes/terminal/runtime.d.ts +3 -2
- package/dist/routes/terminal/runtime.js +3 -3
- package/dist/routes/terminal/sandbox.d.ts +3 -3
- package/dist/routes/terminal/sandbox.js +9 -9
- package/dist/routes/terminal/session.js +16 -14
- package/dist/routes/terminal/token-usage.js +5 -5
- package/dist/routes/terminal/trace.d.ts +5 -8
- package/dist/routes/terminal/trace.js +9 -64
- package/dist/routes/terminal/types.d.ts +85 -93
- package/dist/routes/terminal/types.js +9 -4
- package/dist/run/active-runs.d.ts +13 -0
- package/dist/run/active-runs.js +12 -0
- package/dist/run/bundle-shape.d.ts +2 -1
- package/dist/run/bundle-shape.js +4 -4
- package/dist/run/bundle.d.ts +63 -56
- package/dist/run/bundle.js +9 -8
- package/dist/run/cost-summary.js +3 -2
- package/dist/run/costs.d.ts +6 -6
- package/dist/run/costs.js +10 -10
- package/dist/run/detail.d.ts +5 -5
- package/dist/run/detail.js +6 -6
- package/dist/run/dry-run.js +5 -12
- package/dist/run/{study-files.d.ts → evidence-files.d.ts} +4 -4
- package/dist/run/{study-files.js → evidence-files.js} +5 -5
- package/dist/run/feedback-shape.js +1 -1
- package/dist/run/judge.d.ts +16 -4
- package/dist/run/judge.js +24 -7
- package/dist/run/locate.d.ts +2 -0
- package/dist/run/locate.js +3 -1
- package/dist/run/outcomes.d.ts +3 -3
- package/dist/run/outcomes.js +3 -3
- package/dist/run/participant-records.d.ts +2 -2
- package/dist/run/participant-records.js +2 -2
- package/dist/run/paths.js +1 -1
- package/dist/run/pricing.d.ts +19 -9
- package/dist/run/pricing.js +30 -18
- package/dist/run/projection.d.ts +13 -13
- package/dist/run/projection.js +8 -8
- package/dist/run/reclaim.d.ts +6 -0
- package/dist/run/reclaim.js +13 -3
- package/dist/run/results.d.ts +1 -1
- package/dist/run/run-index.d.ts +1 -1
- package/dist/run/run-index.js +11 -11
- package/dist/run/run.d.ts +4 -2
- package/dist/run/run.js +9 -5
- package/dist/run/sandbox-receipts.d.ts +4 -4
- package/dist/run/sandbox-receipts.js +4 -4
- package/dist/run/shared-world-evidence.d.ts +47 -47
- package/dist/run/shared-world-shape.d.ts +3 -3
- package/dist/run/shared-world-shape.js +11 -11
- package/dist/run/stats.d.ts +5 -5
- package/dist/run/stats.js +8 -8
- package/dist/run/status.d.ts +16 -6
- package/dist/run/status.js +31 -8
- package/dist/run/streams.d.ts +26 -19
- package/dist/run/terminal-contract.d.ts +5 -5
- package/dist/run/terminal-contract.js +4 -4
- package/dist/run/transient-comms-secrets.d.ts +5 -0
- package/dist/run/transient-comms-secrets.js +12 -0
- package/dist/run-lab.d.ts +36 -34
- package/dist/run-lab.js +41 -37
- package/dist/subject/desktop-cli.d.ts +1 -1
- package/dist/subject/desktop-cli.js +1 -1
- package/dist/subject/local-tree-archive.d.ts +1 -1
- package/dist/subject/local-tree-archive.js +4 -4
- package/dist/subject/local-tree.d.ts +2 -2
- package/dist/subject/local-tree.js +1 -1
- package/dist/subject/node-bootstrap.d.ts +1 -1
- package/dist/subject/node-bootstrap.js +4 -4
- package/dist/subject/runtime.js +5 -5
- package/dist/subject/serve.js +1 -1
- package/dist/subject/steps.d.ts +4 -4
- package/dist/subject/steps.js +5 -5
- package/dist/substrates/e2b/cdp-probe.d.ts +2 -2
- package/dist/substrates/e2b/cdp-probe.js +1 -1
- package/dist/substrates/e2b/desktop-browser.d.ts +3 -3
- package/dist/substrates/e2b/desktop-browser.js +4 -4
- package/dist/substrates/e2b/desktop-cdp.d.ts +7 -7
- package/dist/substrates/e2b/desktop-cdp.js +4 -4
- package/dist/substrates/e2b/desktop-geometry.d.ts +4 -4
- package/dist/substrates/e2b/desktop-geometry.js +3 -3
- package/dist/substrates/e2b/desktop-media.js +2 -2
- package/dist/substrates/e2b/sandbox.js +1 -1
- package/dist/substrates/e2b/sdk.js +2 -2
- package/dist/substrates/local/runtime-host.d.ts +2 -0
- package/dist/substrates/local/runtime-host.js +6 -0
- package/dist/tui/actions.d.ts +7 -5
- package/dist/tui/actions.js +18 -9
- package/dist/tui-app.js +42 -42
- package/dist/verify/actor.d.ts +10 -10
- package/dist/verify/actor.js +15 -15
- package/dist/verify/artifacts.d.ts +1 -1
- package/dist/verify/artifacts.js +3 -3
- package/dist/verify/costs.d.ts +2 -2
- package/dist/verify/costs.js +2 -2
- package/dist/verify/liveness.d.ts +1 -1
- package/dist/verify/liveness.js +4 -1
- package/dist/verify/rerun.d.ts +1 -1
- package/dist/verify/rerun.js +1 -1
- package/dist/verify/shared-world.js +2 -2
- package/dist/verify/verify.js +6 -6
- package/docs/README.md +1 -1
- package/docs/architecture/actor-contract.md +60 -22
- package/docs/architecture/desktop-sessions.md +9 -9
- package/docs/architecture/external-public-shared-world.md +5 -0
- package/docs/architecture/observer.md +4 -4
- package/docs/architecture/restricted-codex-analysis.md +36 -2
- package/docs/architecture/state-driven-executor.md +4 -5
- package/docs/architecture/task-protocol-support.md +3 -5
- package/docs/architecture/{terminal-product-lane.md → terminal-product-route.md} +24 -16
- package/docs/contracts/run-bundle.md +10 -8
- package/docs/contracts/schemas.md +109 -53
- package/docs/contracts/study-analysis.md +24 -0
- package/docs/decisions/0004-participant-desktops-stay-in-the-route.md +2 -1
- package/docs/product/automatic-analysis.md +1 -0
- package/docs/ramp/README.md +2 -1
- package/package.json +4 -4
- package/skills/humanish/SKILL.md +3 -2
- package/skills/humanish/agents/openai.yaml +1 -1
- package/dist/deprecated.d.ts +0 -90
- package/dist/deprecated.js +0 -106
- package/dist/lab/bag-overrides.d.ts +0 -18
- package/dist/lab/bag-overrides.js +0 -121
- package/dist/routes/computer-use/legacy-lane-spec.d.ts +0 -62
- package/dist/routes/computer-use/legacy-lane-spec.js +0 -39
- /package/dist/routes/computer-use/{fanout-lanes.d.ts → fanout-records.d.ts} +0 -0
- /package/dist/routes/computer-use/{run-lanes.d.ts → live-phase.d.ts} +0 -0
- /package/dist/routes/shared-world/{seat-records.d.ts → bundle-records.d.ts} +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CuaExecutor } from "../../actors/computer-use/loop.js";
|
|
2
|
-
import type {
|
|
2
|
+
import type { DesktopParticipantRun, ParticipantRunOutcome } from "./types.js";
|
|
3
3
|
import type { LabCommsEmail, LabCommsRecipient } from "../../lab/types.js";
|
|
4
4
|
import type { PreparedOutputRoot } from "../../run/contained-output.js";
|
|
5
5
|
/** A prepared desktop supplies only participant input/observation and its inbox location. */
|
|
@@ -13,10 +13,10 @@ export interface ReadyParticipantDesktop {
|
|
|
13
13
|
}
|
|
14
14
|
/** Existing bundle fields. Provider-specific facts remain optional and must be measured. */
|
|
15
15
|
export type ParticipantDesktopEvidence = {
|
|
16
|
-
/** The substrate confirmed the desktop was released (an E2B kill or a local VM shutdown). The
|
|
16
|
+
/** The substrate confirmed the desktop was released (an E2B kill or a local VM shutdown). The participant
|
|
17
17
|
* outcome records it as `killed`. */
|
|
18
18
|
released: boolean;
|
|
19
|
-
} & Pick<ParticipantRunOutcome, "sandboxId" | "desktopDurationMs" | "desktopResources" | "streamUrlPresent" | "subjectCommit" | "desktopBrowser" | "desktopGeometry" | "stateStepRecords" | "phaseRecords" | "failureCode" | "commsArtifactPath" | "recording">;
|
|
19
|
+
} & Pick<ParticipantRunOutcome, "sandboxId" | "sandboxRelease" | "desktopDurationMs" | "desktopResources" | "streamUrlPresent" | "subjectCommit" | "desktopBrowser" | "desktopGeometry" | "stateStepRecords" | "phaseRecords" | "failureCode" | "commsArtifactPath" | "recording">;
|
|
20
20
|
/**
|
|
21
21
|
* The desktop one participant runs on, owned from preparation through final evidence collection.
|
|
22
22
|
* Call methods sequentially: prepare, openSession, then finalize in a finally block.
|
|
@@ -38,26 +38,11 @@ export interface ParticipantDesktop {
|
|
|
38
38
|
}
|
|
39
39
|
/** Builds one participant's desktop. The factory must not acquire anything; prepare does. */
|
|
40
40
|
export type ParticipantDesktopFactory = (run: DesktopParticipantRun, warnings: string[], artifactRoot: PreparedOutputRoot) => ParticipantDesktop;
|
|
41
|
-
/**
|
|
42
|
-
*
|
|
43
|
-
* symbol keeps it out of the public CuaActorLabHooks, whose deprecated createDesktopLane takes the
|
|
44
|
-
* flat lane record; withHookOverrides forwards it like any other member of the bag.
|
|
45
|
-
*/
|
|
46
|
-
export declare const PARTICIPANT_DESKTOP: unique symbol;
|
|
47
|
-
/** A hook bag that may carry a study's own desktop factory. */
|
|
48
|
-
export type HooksWithParticipantDesktop = CuaActorLabHooks & {
|
|
49
|
-
readonly [PARTICIPANT_DESKTOP]?: ParticipantDesktopFactory;
|
|
50
|
-
};
|
|
51
|
-
/**
|
|
52
|
-
* The desktop factory a run uses: the study's own, else the caller's deprecated createDesktopLane,
|
|
53
|
-
* given the flat view of each run. Undefined means the route provisions its own desktop.
|
|
54
|
-
*/
|
|
55
|
-
export declare function participantDesktopOf(hooks: CuaActorLabHooks): ParticipantDesktopFactory | undefined;
|
|
56
|
-
/** The lane's addressed comms recipient, when one exists — the gate AND the address source for the
|
|
57
|
-
* inbox instruction (#351). A lane told to check an inbox it can never receive into would stall,
|
|
41
|
+
/** The participant's addressed comms recipient, when one exists — the gate AND the address source for the
|
|
42
|
+
* inbox instruction (#351). A participant told to check an inbox it can never receive into would stall,
|
|
58
43
|
* so no addressed recipient means no instruction. */
|
|
59
44
|
export declare function inboxRecipientFor(commsEmail: LabCommsEmail, participantId: string): LabCommsRecipient | undefined;
|
|
60
|
-
/** True when a
|
|
61
|
-
* mail the persona will be told to read. Gates the inbox instruction to
|
|
62
|
-
* a
|
|
45
|
+
/** True when a participant has a declared comms recipient WITH an address, so the drain can actually match the
|
|
46
|
+
* mail the persona will be told to read. Gates the inbox instruction to participants that can receive
|
|
47
|
+
* mail — a participant told to check an inbox it can never receive into would just stall. */
|
|
63
48
|
export declare function participantHasInboxRecipient(commsEmail: LabCommsEmail, participantId: string): boolean;
|
|
@@ -1,33 +1,13 @@
|
|
|
1
|
-
import { legacyHookSpecOf } from "./legacy-lane-spec.js";
|
|
2
1
|
import { recipientParticipantId } from "../../lab/parse/comms.js";
|
|
3
|
-
/**
|
|
4
|
-
*
|
|
5
|
-
* symbol keeps it out of the public CuaActorLabHooks, whose deprecated createDesktopLane takes the
|
|
6
|
-
* flat lane record; withHookOverrides forwards it like any other member of the bag.
|
|
7
|
-
*/
|
|
8
|
-
export const PARTICIPANT_DESKTOP = Symbol("humanish.participantDesktop");
|
|
9
|
-
/**
|
|
10
|
-
* The desktop factory a run uses: the study's own, else the caller's deprecated createDesktopLane,
|
|
11
|
-
* given the flat view of each run. Undefined means the route provisions its own desktop.
|
|
12
|
-
*/
|
|
13
|
-
export function participantDesktopOf(hooks) {
|
|
14
|
-
const own = hooks[PARTICIPANT_DESKTOP];
|
|
15
|
-
if (own !== undefined)
|
|
16
|
-
return own;
|
|
17
|
-
const caller = hooks.createDesktopLane;
|
|
18
|
-
return caller === undefined
|
|
19
|
-
? undefined
|
|
20
|
-
: (run, warnings, artifactRoot) => caller(legacyHookSpecOf(run), warnings, artifactRoot);
|
|
21
|
-
}
|
|
22
|
-
/** The lane's addressed comms recipient, when one exists — the gate AND the address source for the
|
|
23
|
-
* inbox instruction (#351). A lane told to check an inbox it can never receive into would stall,
|
|
2
|
+
/** The participant's addressed comms recipient, when one exists — the gate AND the address source for the
|
|
3
|
+
* inbox instruction (#351). A participant told to check an inbox it can never receive into would stall,
|
|
24
4
|
* so no addressed recipient means no instruction. */
|
|
25
5
|
export function inboxRecipientFor(commsEmail, participantId) {
|
|
26
6
|
return (commsEmail.recipients ?? []).find((recipient) => recipientParticipantId(recipient) === participantId && recipient.address !== undefined);
|
|
27
7
|
}
|
|
28
|
-
/** True when a
|
|
29
|
-
* mail the persona will be told to read. Gates the inbox instruction to
|
|
30
|
-
* a
|
|
8
|
+
/** True when a participant has a declared comms recipient WITH an address, so the drain can actually match the
|
|
9
|
+
* mail the persona will be told to read. Gates the inbox instruction to participants that can receive
|
|
10
|
+
* mail — a participant told to check an inbox it can never receive into would just stall. */
|
|
31
11
|
export function participantHasInboxRecipient(commsEmail, participantId) {
|
|
32
12
|
return inboxRecipientFor(commsEmail, participantId) !== undefined;
|
|
33
13
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type PreparedOutputRoot } from "../../run/contained-output.js";
|
|
2
2
|
import type { CuaParticipantDeps, CuaParticipantPlan, CuaParticipantResult, DesktopParticipantRun, CuaSubjectProjection, ParticipantRunOutcome } from "./types.js";
|
|
3
|
-
/** Build a
|
|
3
|
+
/** Build a participant's writeScreenshot closure: writes under screenshots/<screenshotDir>/ and records
|
|
4
4
|
* the relative path the trace references (screenshots/<name> at N=1; screenshots/<laneId>/<name>
|
|
5
5
|
* at N>1). */
|
|
6
6
|
export declare function makeParticipantWriteScreenshot(artifactRoot: PreparedOutputRoot, spec: {
|
|
@@ -10,12 +10,12 @@ export declare function makeParticipantWriteScreenshot(artifactRoot: PreparedOut
|
|
|
10
10
|
* evidence and cleanup; this runner owns the model, trace and participant outcome. */
|
|
11
11
|
export declare function runCuaParticipant(spec: DesktopParticipantRun, deps: CuaParticipantDeps): Promise<ParticipantRunOutcome>;
|
|
12
12
|
/**
|
|
13
|
-
* Run N>1 E2B
|
|
14
|
-
* start), and session fail-fast on HARNESS errors only (queued
|
|
15
|
-
* pinned reason + a fail-fast event; mission verdicts never trip it).
|
|
16
|
-
* OWN sandbox by id; nothing here ever enumerates.
|
|
13
|
+
* Run N>1 E2B participants with bounded concurrency, a pipeline gate (the first provisions
|
|
14
|
+
* before the rest start), and session fail-fast on HARNESS errors only (queued participants
|
|
15
|
+
* become `blocked` with a pinned reason + a fail-fast event; mission verdicts never trip it).
|
|
16
|
+
* Each participant tears down ITS OWN sandbox by id; nothing here ever enumerates.
|
|
17
17
|
*
|
|
18
|
-
* Exported for the #342 total-runner tests: the injectable runner lets a test make one
|
|
18
|
+
* Exported for the #342 total-runner tests: the injectable runner lets a test make one participant
|
|
19
19
|
* THROW (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
20
20
|
* the default.
|
|
21
21
|
*/
|
|
@@ -23,13 +23,13 @@ export declare function runCuaParticipants(runs: DesktopParticipantRun[], deps:
|
|
|
23
23
|
outcomes: ParticipantRunOutcome[];
|
|
24
24
|
failFastReason?: string;
|
|
25
25
|
}>;
|
|
26
|
-
/** Project one
|
|
26
|
+
/** Project one participant outcome (or a dry-run contract spec) into the public CuaParticipantResult. */
|
|
27
27
|
export declare function toParticipantResult(spec: DesktopParticipantRun, outcome: ParticipantRunOutcome | undefined, subject: CuaSubjectProjection, dryRun: boolean): CuaParticipantResult;
|
|
28
28
|
/**
|
|
29
|
-
* Run every
|
|
30
|
-
* hosted
|
|
29
|
+
* Run every participant of a live run. The in-process route drives its single participant in this
|
|
30
|
+
* process; one hosted participant runs alone; a fan-out runs at the plan's concurrency and may stop early.
|
|
31
31
|
*/
|
|
32
|
-
export declare function runAllCuaParticipants(runs: readonly DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalProvisioned">, participantPlan: CuaParticipantPlan,
|
|
32
|
+
export declare function runAllCuaParticipants(runs: readonly DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalProvisioned">, participantPlan: CuaParticipantPlan, inProcess: boolean): Promise<{
|
|
33
33
|
outcomes: ParticipantRunOutcome[];
|
|
34
34
|
failFastReason: string | undefined;
|
|
35
35
|
}>;
|
|
@@ -8,7 +8,7 @@ import { redactText, toErrorMessage } from "../../evidence/redaction.js";
|
|
|
8
8
|
import { assertSafeOutputPathSegment, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
9
9
|
import { participantOutcomeOk } from "./participant-facts.js";
|
|
10
10
|
import { closeParticipantModel, judgeParticipantSession, participantSessionOptions, recordParticipantTrace, startParticipantModel, } from "./participant-model.js";
|
|
11
|
-
/** Build a
|
|
11
|
+
/** Build a participant's writeScreenshot closure: writes under screenshots/<screenshotDir>/ and records
|
|
12
12
|
* the relative path the trace references (screenshots/<name> at N=1; screenshots/<laneId>/<name>
|
|
13
13
|
* at N>1). */
|
|
14
14
|
export function makeParticipantWriteScreenshot(artifactRoot, spec, screenshots) {
|
|
@@ -30,7 +30,7 @@ export function makeParticipantWriteScreenshot(artifactRoot, spec, screenshots)
|
|
|
30
30
|
return rel;
|
|
31
31
|
};
|
|
32
32
|
}
|
|
33
|
-
/** A blocked
|
|
33
|
+
/** A blocked participant outcome (pipeline gate / fail-fast skipped it before it ran). */
|
|
34
34
|
function skippedOutcome(spec, reason) {
|
|
35
35
|
return {
|
|
36
36
|
spec,
|
|
@@ -57,6 +57,7 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
57
57
|
let session;
|
|
58
58
|
let sessionError;
|
|
59
59
|
let providerCleanupError;
|
|
60
|
+
let providerPolicyError;
|
|
60
61
|
let provisioned = false;
|
|
61
62
|
let signaled = false;
|
|
62
63
|
const signal = (ok) => {
|
|
@@ -71,7 +72,7 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
71
72
|
await desktop.prepare();
|
|
72
73
|
const ready = await desktop.openSession();
|
|
73
74
|
model = await startParticipantModel(spec, deps, ready.executor);
|
|
74
|
-
// World is ready: release the pipeline gate so the remaining
|
|
75
|
+
// World is ready: release the pipeline gate so the remaining participants may start.
|
|
75
76
|
provisioned = true;
|
|
76
77
|
signal(true);
|
|
77
78
|
session = await deps.runSession(participantSessionOptions(spec, deps, ready, model.provider, writeScreenshot));
|
|
@@ -82,25 +83,35 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
82
83
|
finally {
|
|
83
84
|
// A provider whose cleanup is unconfirmed fails the run as an execution, apart from how the
|
|
84
85
|
// session ended: a session that passed stays a passed participant.
|
|
85
|
-
|
|
86
|
+
const closed = await closeParticipantModel(model, warnings);
|
|
87
|
+
if (closed.unconfirmed)
|
|
86
88
|
providerCleanupError = "Model provider cleanup is unconfirmed.";
|
|
87
|
-
|
|
89
|
+
// The session's evidence would read clean, so a late refusal fails the run as an execution.
|
|
90
|
+
if (closed.refusal !== undefined)
|
|
91
|
+
providerPolicyError = redactText(deps.scrubKnownValues(closed.refusal === "codex_tool_call"
|
|
92
|
+
? `Codex reported a disallowed item after the participant's last request (${closed.refusal}).`
|
|
93
|
+
: `Codex output after the participant's last request could not be checked against the item policy (${closed.refusal}).`));
|
|
88
94
|
try {
|
|
89
95
|
if (!provisioned)
|
|
90
96
|
signal(false);
|
|
91
97
|
}
|
|
92
98
|
finally {
|
|
93
99
|
await desktop.finalize({
|
|
94
|
-
failed: sessionError !== undefined ||
|
|
100
|
+
failed: sessionError !== undefined ||
|
|
101
|
+
providerCleanupError !== undefined ||
|
|
102
|
+
providerPolicyError !== undefined ||
|
|
103
|
+
session === undefined,
|
|
95
104
|
});
|
|
96
105
|
}
|
|
97
106
|
}
|
|
98
107
|
if (session)
|
|
99
108
|
await recordParticipantTrace(spec, deps, session, warnings);
|
|
100
109
|
const { noEngagement, selfReportedBlocker, reportedFriction } = judgeParticipantSession(session, deps, warnings);
|
|
101
|
-
// A provider-cleanup failure still trips fail-fast and counts in the
|
|
110
|
+
// A provider-cleanup or provider-policy failure still trips fail-fast and counts in the
|
|
111
|
+
// participant summary's harness errors.
|
|
102
112
|
const harnessError = sessionError !== undefined ||
|
|
103
113
|
providerCleanupError !== undefined ||
|
|
114
|
+
providerPolicyError !== undefined ||
|
|
104
115
|
session?.completionReason === "harness_error";
|
|
105
116
|
const { released, ...desktopEvidence } = desktop.snapshot();
|
|
106
117
|
return {
|
|
@@ -108,6 +119,7 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
108
119
|
...(session ? { session } : {}),
|
|
109
120
|
...(sessionError === undefined ? {} : { sessionError }),
|
|
110
121
|
...(providerCleanupError === undefined ? {} : { providerCleanupError }),
|
|
122
|
+
...(providerPolicyError === undefined ? {} : { providerPolicyError }),
|
|
111
123
|
...desktopEvidence,
|
|
112
124
|
killed: released,
|
|
113
125
|
screenshots,
|
|
@@ -119,12 +131,12 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
119
131
|
};
|
|
120
132
|
}
|
|
121
133
|
/**
|
|
122
|
-
* Run N>1 E2B
|
|
123
|
-
* start), and session fail-fast on HARNESS errors only (queued
|
|
124
|
-
* pinned reason + a fail-fast event; mission verdicts never trip it).
|
|
125
|
-
* OWN sandbox by id; nothing here ever enumerates.
|
|
134
|
+
* Run N>1 E2B participants with bounded concurrency, a pipeline gate (the first provisions
|
|
135
|
+
* before the rest start), and session fail-fast on HARNESS errors only (queued participants
|
|
136
|
+
* become `blocked` with a pinned reason + a fail-fast event; mission verdicts never trip it).
|
|
137
|
+
* Each participant tears down ITS OWN sandbox by id; nothing here ever enumerates.
|
|
126
138
|
*
|
|
127
|
-
* Exported for the #342 total-runner tests: the injectable runner lets a test make one
|
|
139
|
+
* Exported for the #342 total-runner tests: the injectable runner lets a test make one participant
|
|
128
140
|
* THROW (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
129
141
|
* the default.
|
|
130
142
|
*/
|
|
@@ -136,8 +148,9 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
136
148
|
resolveGate = resolve;
|
|
137
149
|
rejectGate = () => reject(new Error("gate"));
|
|
138
150
|
});
|
|
139
|
-
// The gate is rejected on
|
|
140
|
-
// later
|
|
151
|
+
// The gate is rejected on the first participant's provisioning failure; swallow the unhandled
|
|
152
|
+
// rejection if no later participant ever awaits it (concurrency could let the first finish
|
|
153
|
+
// alone).
|
|
141
154
|
gate.catch(() => undefined);
|
|
142
155
|
const outcomes = await mapWithConcurrency(runs, concurrency, async (spec, index) => {
|
|
143
156
|
if (index > 0) {
|
|
@@ -151,8 +164,8 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
151
164
|
if (failFast.tripped) {
|
|
152
165
|
return skippedOutcome(spec, `skipped: ${failFast.reason}`);
|
|
153
166
|
}
|
|
154
|
-
// The
|
|
155
|
-
// guard, one
|
|
167
|
+
// The participant runner is TOTAL (#342): every exit path returns a recorded outcome. Without this
|
|
168
|
+
// guard, one participant's late throw (e.g. its trace write hitting ENOSPC after its own sandbox was
|
|
156
169
|
// already torn down) rejected the whole map while sibling workers kept launching sandboxes
|
|
157
170
|
// nobody would ever record — the run spent money and then reported nothing.
|
|
158
171
|
let outcome;
|
|
@@ -174,7 +187,7 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
174
187
|
});
|
|
175
188
|
}
|
|
176
189
|
catch (error) {
|
|
177
|
-
//
|
|
190
|
+
// The first participant may have thrown before signaling the provisioning gate — release the followers as
|
|
178
191
|
// blocked rather than leaving them awaiting a gate that will never settle.
|
|
179
192
|
if (index === 0)
|
|
180
193
|
rejectGate?.();
|
|
@@ -202,7 +215,7 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
202
215
|
});
|
|
203
216
|
return { outcomes, ...(failFast.tripped ? { failFastReason: failFast.reason } : {}) };
|
|
204
217
|
}
|
|
205
|
-
/** Project one
|
|
218
|
+
/** Project one participant outcome (or a dry-run contract spec) into the public CuaParticipantResult. */
|
|
206
219
|
export function toParticipantResult(spec, outcome, subject, dryRun) {
|
|
207
220
|
const base = {
|
|
208
221
|
id: spec.planned.id,
|
|
@@ -302,11 +315,11 @@ export function toParticipantResult(spec, outcome, subject, dryRun) {
|
|
|
302
315
|
};
|
|
303
316
|
}
|
|
304
317
|
/**
|
|
305
|
-
* Run every
|
|
306
|
-
* hosted
|
|
318
|
+
* Run every participant of a live run. The in-process route drives its single participant in this
|
|
319
|
+
* process; one hosted participant runs alone; a fan-out runs at the plan's concurrency and may stop early.
|
|
307
320
|
*/
|
|
308
|
-
export async function runAllCuaParticipants(runs, deps, participantPlan,
|
|
309
|
-
if (
|
|
321
|
+
export async function runAllCuaParticipants(runs, deps, participantPlan, inProcess) {
|
|
322
|
+
if (inProcess) {
|
|
310
323
|
// The caller's executor stands in for a desktop, and the shared runner drives the participant.
|
|
311
324
|
const outcome = await runCuaParticipant(runs[0], {
|
|
312
325
|
...deps,
|
|
@@ -1,6 +1,12 @@
|
|
|
1
|
-
import { type ParticipantFacts } from "../../run/judge.js";
|
|
1
|
+
import { type ExecutionFailure, type ParticipantFacts } from "../../run/judge.js";
|
|
2
2
|
import type { ParticipantRunOutcome } from "./types.js";
|
|
3
3
|
/** A participant outcome's facts for the judge. */
|
|
4
4
|
export declare function participantFactsOf(outcome: ParticipantRunOutcome): ParticipantFacts;
|
|
5
5
|
/** Whether a participant passed. A dry-run participant passes as a contract. */
|
|
6
6
|
export declare function participantOutcomeOk(outcome: ParticipantRunOutcome | undefined, dryRun: boolean): boolean;
|
|
7
|
+
/**
|
|
8
|
+
* One sandbox-cleanup failure per participant desktop not confirmed released: an E2B sandbox that
|
|
9
|
+
* was not killed, or a desktop whose release says why (a local VM names its container). A
|
|
10
|
+
* participant that never acquired a desktop (skipped, in-process) adds none.
|
|
11
|
+
*/
|
|
12
|
+
export declare function unreleasedSandboxFailures(outcomes: readonly ParticipantRunOutcome[] | undefined, runId: string): ExecutionFailure[];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// A participant run outcome's facts for the judge (src/run/judge.ts), and the pass check built on
|
|
2
2
|
// them. The bundle builders, the result, the participant runner and shared-world all read these,
|
|
3
3
|
// so they sit in a file that imports only the judge and the types.
|
|
4
|
-
import { participantPassed } from "../../run/judge.js";
|
|
4
|
+
import { participantPassed, sandboxCleanupFailure, } from "../../run/judge.js";
|
|
5
5
|
/** A participant outcome's facts for the judge. */
|
|
6
6
|
export function participantFactsOf(outcome) {
|
|
7
7
|
return {
|
|
@@ -20,3 +20,14 @@ export function participantOutcomeOk(outcome, dryRun) {
|
|
|
20
20
|
return true;
|
|
21
21
|
return outcome !== undefined && participantPassed(participantFactsOf(outcome));
|
|
22
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* One sandbox-cleanup failure per participant desktop not confirmed released: an E2B sandbox that
|
|
25
|
+
* was not killed, or a desktop whose release says why (a local VM names its container). A
|
|
26
|
+
* participant that never acquired a desktop (skipped, in-process) adds none.
|
|
27
|
+
*/
|
|
28
|
+
export function unreleasedSandboxFailures(outcomes, runId) {
|
|
29
|
+
return (outcomes ?? [])
|
|
30
|
+
.filter((outcome) => outcome.sandboxRelease !== undefined ||
|
|
31
|
+
(outcome.sandboxId !== undefined && !outcome.killed))
|
|
32
|
+
.map((outcome) => sandboxCleanupFailure(outcome.spec.planned.id, outcome.sandboxRelease?.warning, runId, outcome.sandboxRelease?.recovery));
|
|
33
|
+
}
|
|
@@ -1,28 +1,36 @@
|
|
|
1
1
|
import type { CuaExecutor, CuaLoopResult, CuaProvider } from "../../actors/computer-use/loop.js";
|
|
2
2
|
import type { CuaActorSessionOptions } from "../../actors/computer-use/actor.js";
|
|
3
3
|
import { startClaudeSession } from "../../actors/local-agent/claude-session.js";
|
|
4
|
-
import { createRestrictedCodexParticipant } from "../../actors/codex/restricted-participant.js";
|
|
5
|
-
import type {
|
|
4
|
+
import { createRestrictedCodexParticipant, type ParticipantProviderCloseResult } from "../../actors/codex/restricted-participant.js";
|
|
5
|
+
import type { ParticipantModelDeps, CuaRunBudget, DesktopParticipantRun } from "./types.js";
|
|
6
6
|
import type { ReadyParticipantDesktop } from "./participant-desktop.js";
|
|
7
|
-
/**
|
|
7
|
+
/**
|
|
8
|
+
* Registers a caller's provider as backed by a native Codex session: the participant runner reads the session's
|
|
9
|
+
* run warnings and a refusal no request reported from `report` when it closes the provider.
|
|
10
|
+
*/
|
|
11
|
+
export declare function withCloseReport<T extends CuaProvider>(provider: T, report: () => Promise<ParticipantProviderCloseResult>): T;
|
|
12
|
+
/** The model a participant brings besides the default API client, and the handles its cleanup needs. */
|
|
8
13
|
export interface ParticipantModel {
|
|
9
14
|
provider?: CuaProvider;
|
|
10
15
|
codexParticipant?: ReturnType<typeof createRestrictedCodexParticipant>;
|
|
11
16
|
claudeSession?: Awaited<ReturnType<typeof startClaudeSession>>;
|
|
12
17
|
}
|
|
13
|
-
/** Starts the
|
|
14
|
-
export declare function startParticipantModel(spec: DesktopParticipantRun, deps:
|
|
15
|
-
/** The options the
|
|
16
|
-
export declare function participantSessionOptions(spec: DesktopParticipantRun, deps:
|
|
18
|
+
/** Starts the participant's model: a caller's provider, the operator's Codex, a Claude session, or none. */
|
|
19
|
+
export declare function startParticipantModel(spec: DesktopParticipantRun, deps: ParticipantModelDeps, executor: CuaExecutor): Promise<ParticipantModel>;
|
|
20
|
+
/** The options the participant's session runs with: prompt, model settings, spend caps and callbacks. */
|
|
21
|
+
export declare function participantSessionOptions(spec: DesktopParticipantRun, deps: ParticipantModelDeps, ready: ReadyParticipantDesktop, provider: CuaProvider | undefined, writeScreenshot: NonNullable<CuaActorSessionOptions["writeScreenshot"]>): CuaActorSessionOptions;
|
|
17
22
|
/**
|
|
18
|
-
* Closes the
|
|
23
|
+
* Closes the participant's model, in the order it was opened. Returns true when a provider's cleanup is
|
|
19
24
|
* unconfirmed; each failure also adds a warning.
|
|
20
25
|
*/
|
|
21
|
-
export declare function closeParticipantModel(model: ParticipantModel, warnings: string[]): Promise<
|
|
26
|
+
export declare function closeParticipantModel(model: ParticipantModel, warnings: string[]): Promise<{
|
|
27
|
+
unconfirmed: boolean;
|
|
28
|
+
refusal: string | undefined;
|
|
29
|
+
}>;
|
|
22
30
|
/** Prices the finished session's tokens onto its trace, writes the trace, and warns on raw screenshots. */
|
|
23
|
-
export declare function recordParticipantTrace(spec: DesktopParticipantRun, deps:
|
|
31
|
+
export declare function recordParticipantTrace(spec: DesktopParticipantRun, deps: ParticipantModelDeps, session: CuaLoopResult, warnings: string[]): Promise<void>;
|
|
24
32
|
/** The checks that keep a goal_satisfied session from counting as a pass, with their warnings. */
|
|
25
|
-
export declare function judgeParticipantSession(session: CuaLoopResult | undefined, deps:
|
|
33
|
+
export declare function judgeParticipantSession(session: CuaLoopResult | undefined, deps: ParticipantModelDeps, warnings: string[]): {
|
|
26
34
|
noEngagement: boolean;
|
|
27
35
|
selfReportedBlocker: boolean;
|
|
28
36
|
reportedFriction: boolean;
|
|
@@ -1,30 +1,36 @@
|
|
|
1
|
-
// The model side of one computer-use
|
|
1
|
+
// The model side of one computer-use participant: the provider a local agent brings, the session options
|
|
2
2
|
// the loop runs with, the study's shared spend ledger, the provider cleanup, and the checks on the
|
|
3
|
-
// finished session. The desktop side stays with the
|
|
3
|
+
// finished session. The desktop side stays with the participant runner.
|
|
4
4
|
import { startClaudeSession } from "../../actors/local-agent/claude-session.js";
|
|
5
5
|
import { createLocalAgentProvider } from "../../actors/local-agent/cli.js";
|
|
6
6
|
import { pricedModel } from "../../lab/plan-base.js";
|
|
7
7
|
import { estimateActorCostForExecution, round6 } from "../../run/pricing.js";
|
|
8
8
|
import { redactText } from "../../evidence/redaction.js";
|
|
9
|
-
import { createRestrictedCodexParticipant } from "../../actors/codex/restricted-participant.js";
|
|
9
|
+
import { createRestrictedCodexParticipant, } from "../../actors/codex/restricted-participant.js";
|
|
10
10
|
import { writeContainedOutputFile } from "../../run/contained-output.js";
|
|
11
11
|
import { withInboxMission } from "./participant-prompt.js";
|
|
12
12
|
import { resolveSelfReportedBlocker, resolveSelfReportedFriction, sessionEnding, } from "./self-report.js";
|
|
13
13
|
import { hollowCompletion } from "../../run/judge.js";
|
|
14
|
-
|
|
15
|
-
|
|
14
|
+
// Caller providers backed by a native Codex session (the local study's), with their close report.
|
|
15
|
+
const closeReports = new WeakMap();
|
|
16
|
+
/**
|
|
17
|
+
* Registers a caller's provider as backed by a native Codex session: the participant runner reads the session's
|
|
18
|
+
* run warnings and a refusal no request reported from `report` when it closes the provider.
|
|
19
|
+
*/
|
|
20
|
+
export function withCloseReport(provider, report) {
|
|
21
|
+
closeReports.set(provider, report);
|
|
22
|
+
return provider;
|
|
23
|
+
}
|
|
24
|
+
/** Starts the participant's model: a caller's provider, the operator's Codex, a Claude session, or none. */
|
|
16
25
|
export async function startParticipantModel(spec, deps, executor) {
|
|
17
|
-
const {
|
|
18
|
-
if (deps.
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
lane: legacyHookSpecOf(spec),
|
|
24
|
-
laneCount: deps.participantCount,
|
|
25
|
-
executor,
|
|
26
|
-
}),
|
|
26
|
+
const { env, brain } = deps;
|
|
27
|
+
if (deps.createProvider) {
|
|
28
|
+
const participant = {
|
|
29
|
+
id: spec.planned.id,
|
|
30
|
+
index: spec.planned.index,
|
|
31
|
+
count: deps.participantCount,
|
|
27
32
|
};
|
|
33
|
+
return { provider: await deps.createProvider(participant, executor) };
|
|
28
34
|
}
|
|
29
35
|
const localAgent = brain.kind === "local-agent" ? brain.agent : undefined;
|
|
30
36
|
if (localAgent === "codex") {
|
|
@@ -73,16 +79,15 @@ export async function startParticipantModel(spec, deps, executor) {
|
|
|
73
79
|
}
|
|
74
80
|
return {};
|
|
75
81
|
}
|
|
76
|
-
/** The options the
|
|
82
|
+
/** The options the participant's session runs with: prompt, model settings, spend caps and callbacks. */
|
|
77
83
|
export function participantSessionOptions(spec, deps, ready, provider, writeScreenshot) {
|
|
78
|
-
const { config } = deps;
|
|
79
84
|
const { executor, inbox } = ready;
|
|
80
85
|
// The FAIL-CLOSED spend cap (execution.caps.maxUsd) is wired into the loop as maxUsd + an
|
|
81
86
|
// injected pure per-turn estimator keyed on the resolved model. Preflight already refused a
|
|
82
87
|
// cap on an unpriced model, so the estimate is measurable whenever a cap is in force. The
|
|
83
88
|
// model id here matches provider.version (openai-responses-cu resolves the default when unset).
|
|
84
89
|
const capModelId = pricedModel(deps.brain);
|
|
85
|
-
const maxUsd =
|
|
90
|
+
const maxUsd = deps.caps.maxUsd;
|
|
86
91
|
return {
|
|
87
92
|
instructions: inbox
|
|
88
93
|
? withInboxMission(spec, inbox.url, inbox.address, inbox.receiving).instructions
|
|
@@ -96,7 +101,8 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
|
|
|
96
101
|
openai: {
|
|
97
102
|
apiKey: deps.openaiApiKey,
|
|
98
103
|
...(deps.brain.declaredModel ? { model: deps.brain.declaredModel } : {}),
|
|
99
|
-
// Per-
|
|
104
|
+
// Per-participant, not per-actor: two participants at different efforts is the control this
|
|
105
|
+
// exists for.
|
|
100
106
|
...(spec.planned.limits.reasoningEffort === undefined
|
|
101
107
|
? {}
|
|
102
108
|
: { reasoningEffort: spec.planned.limits.reasoningEffort }),
|
|
@@ -123,8 +129,9 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
|
|
|
123
129
|
: { stopWhen: spec.planned.limits.stopWhen }),
|
|
124
130
|
...(spec.planned.limits.dwell === undefined ? {} : { dwell: spec.planned.limits.dwell }),
|
|
125
131
|
...(spec.planned.tasks === undefined ? {} : { tasks: spec.planned.tasks }),
|
|
126
|
-
// The STUDY budget (#299): this
|
|
127
|
-
// and stops when the RUN total crosses the cap — independent of the per-
|
|
132
|
+
// The STUDY budget (#299): this participant notes its own running estimate on the shared
|
|
133
|
+
// ledger and stops when the RUN total crosses the cap — independent of the per-participant
|
|
134
|
+
// maxUsd above.
|
|
128
135
|
...(deps.runBudget === undefined
|
|
129
136
|
? {}
|
|
130
137
|
: {
|
|
@@ -142,18 +149,19 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
|
|
|
142
149
|
...(deps.onTrace === undefined
|
|
143
150
|
? {}
|
|
144
151
|
: {
|
|
145
|
-
// Forwards the RUNNING usage as well: the
|
|
152
|
+
// Forwards the RUNNING usage as well: the participant runner is where both are known, and usage
|
|
146
153
|
// without it never reaches the flush — which is how the live cost stayed unknown.
|
|
147
154
|
onTrace: (items, usage, metadata) => deps.onTrace?.(spec.planned.id, items, usage, metadata),
|
|
148
155
|
}),
|
|
149
156
|
};
|
|
150
157
|
}
|
|
151
158
|
/**
|
|
152
|
-
* Closes the
|
|
159
|
+
* Closes the participant's model, in the order it was opened. Returns true when a provider's cleanup is
|
|
153
160
|
* unconfirmed; each failure also adds a warning.
|
|
154
161
|
*/
|
|
155
162
|
export async function closeParticipantModel(model, warnings) {
|
|
156
163
|
let unconfirmed = false;
|
|
164
|
+
let refusal;
|
|
157
165
|
try {
|
|
158
166
|
if (model.codexParticipant === undefined)
|
|
159
167
|
await model.provider?.close?.();
|
|
@@ -164,6 +172,8 @@ export async function closeParticipantModel(model, warnings) {
|
|
|
164
172
|
}
|
|
165
173
|
try {
|
|
166
174
|
const cleanup = await model.codexParticipant?.close();
|
|
175
|
+
warnings.push(...(cleanup?.warnings ?? []));
|
|
176
|
+
refusal = cleanup?.refusal;
|
|
167
177
|
if (cleanup?.status === "unconfirmed") {
|
|
168
178
|
warnings.push("Model provider cleanup is unconfirmed.");
|
|
169
179
|
unconfirmed = true;
|
|
@@ -173,17 +183,31 @@ export async function closeParticipantModel(model, warnings) {
|
|
|
173
183
|
warnings.push("Model provider cleanup is unconfirmed.");
|
|
174
184
|
unconfirmed = true;
|
|
175
185
|
}
|
|
186
|
+
const report = model.codexParticipant === undefined && model.provider !== undefined
|
|
187
|
+
? closeReports.get(model.provider)
|
|
188
|
+
: undefined;
|
|
189
|
+
if (report) {
|
|
190
|
+
// The provider's own close reported cleanup above; this reads what its session found.
|
|
191
|
+
try {
|
|
192
|
+
const found = await report();
|
|
193
|
+
warnings.push(...(found.warnings ?? []));
|
|
194
|
+
refusal ??= found.refusal;
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
/* Cleanup was already reported through the provider's close. */
|
|
198
|
+
}
|
|
199
|
+
}
|
|
176
200
|
try {
|
|
177
201
|
await model.claudeSession?.close();
|
|
178
202
|
}
|
|
179
203
|
catch {
|
|
180
204
|
warnings.push("Claude session cleanup failed; desktop cleanup will still run.");
|
|
181
205
|
}
|
|
182
|
-
return unconfirmed;
|
|
206
|
+
return { unconfirmed, refusal };
|
|
183
207
|
}
|
|
184
208
|
/** Prices the finished session's tokens onto its trace, writes the trace, and warns on raw screenshots. */
|
|
185
209
|
export async function recordParticipantTrace(spec, deps, session, warnings) {
|
|
186
|
-
// Per-
|
|
210
|
+
// Per-participant model-token cost ESTIMATE, attached to the trace before it is persisted (the model
|
|
187
211
|
// id is authoritative here — provider.version). Kept at the lab boundary so the pure loop
|
|
188
212
|
// never depends on the operator rate table. estimateActorCost declares absent (null) for an
|
|
189
213
|
// unknown rate / missing usage rather than guessing.
|
|
@@ -15,8 +15,8 @@ export declare const DEFAULT_MISSION = "You are testing a web application. The b
|
|
|
15
15
|
export declare const CLOSING_LINE_DIRECTIVE: string;
|
|
16
16
|
/** The persona id a computer-use participant gets when neither it nor its actor names one. */
|
|
17
17
|
export declare const FALLBACK_PERSONA_ID = "cua-operator";
|
|
18
|
-
/** Compose one
|
|
19
|
-
* At N=1 (homogeneous, no roster) this reproduces the prior composeInstructions byte-for-byte. */
|
|
18
|
+
/** Compose one participant's actor prompt: persona line + device line + mission + per-participant
|
|
19
|
+
* steer. At N=1 (homogeneous, no roster) this reproduces the prior composeInstructions byte-for-byte. */
|
|
20
20
|
export declare function composeParticipantInstructions(args: {
|
|
21
21
|
mission: string;
|
|
22
22
|
persona?: string;
|
|
@@ -46,7 +46,7 @@ export declare function composeParticipantInstructions(args: {
|
|
|
46
46
|
instructions: string;
|
|
47
47
|
persona: ActorPersonaRef;
|
|
48
48
|
};
|
|
49
|
-
/** Runtime-inject the persona inbox instruction into a
|
|
49
|
+
/** Runtime-inject the persona inbox instruction into a participant's prompt (#297 slice B). The inbox URL is a
|
|
50
50
|
* runtime loopback/getHost address (not secret), so — mirroring the lobby-code runtime injection — this
|
|
51
51
|
* augments only the instructions the model receives; the authored prompt + its digest are unchanged.
|
|
52
52
|
* Returns a new spec (never mutates). Shared by the CUA + concurrent shared-world routes. */
|
|
@@ -19,8 +19,8 @@ export const CLOSING_LINE_DIRECTIVE = "When you stop, make the FIRST line of you
|
|
|
19
19
|
"Then, from the next line, say what you did, what confused you, and where you hesitated.";
|
|
20
20
|
/** The persona id a computer-use participant gets when neither it nor its actor names one. */
|
|
21
21
|
export const FALLBACK_PERSONA_ID = "cua-operator";
|
|
22
|
-
/** Compose one
|
|
23
|
-
* At N=1 (homogeneous, no roster) this reproduces the prior composeInstructions byte-for-byte. */
|
|
22
|
+
/** Compose one participant's actor prompt: persona line + device line + mission + per-participant
|
|
23
|
+
* steer. At N=1 (homogeneous, no roster) this reproduces the prior composeInstructions byte-for-byte. */
|
|
24
24
|
export function composeParticipantInstructions(args) {
|
|
25
25
|
const { name, preset } = args.device;
|
|
26
26
|
const deviceLine = preset.isMobile
|
|
@@ -31,7 +31,7 @@ export function composeParticipantInstructions(args) {
|
|
|
31
31
|
// be measured optimizes for the measurement instead of using the product (src/lab/tasks.ts).
|
|
32
32
|
const taskLines = renderTaskPrompt(args.tasks ?? []);
|
|
33
33
|
// A resolved persona contributes its compiled directives (friction tolerance, skill bias,
|
|
34
|
-
// accessibility behavior, constraints) through the SAME persona.ts compiler the terminal
|
|
34
|
+
// accessibility behavior, constraints) through the SAME persona.ts compiler the terminal route
|
|
35
35
|
// uses, so one persona file means one behavior across every route.
|
|
36
36
|
const personaLine = args.resolvedPersona
|
|
37
37
|
? renderPersonaPromptSection(args.resolvedPersona)
|
|
@@ -64,7 +64,7 @@ export function composeParticipantInstructions(args) {
|
|
|
64
64
|
},
|
|
65
65
|
};
|
|
66
66
|
}
|
|
67
|
-
/** Runtime-inject the persona inbox instruction into a
|
|
67
|
+
/** Runtime-inject the persona inbox instruction into a participant's prompt (#297 slice B). The inbox URL is a
|
|
68
68
|
* runtime loopback/getHost address (not secret), so — mirroring the lobby-code runtime injection — this
|
|
69
69
|
* augments only the instructions the model receives; the authored prompt + its digest are unchanged.
|
|
70
70
|
* Returns a new spec (never mutates). Shared by the CUA + concurrent shared-world routes. */
|