humanish 0.107.0 → 0.109.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +73 -69
- package/CHANGELOG.md +27 -0
- package/CONTEXT.md +51 -44
- package/README.md +148 -175
- package/SECURITY.md +16 -26
- package/TELEMETRY.md +8 -6
- package/dist/actors/affordance.d.ts +4 -4
- package/dist/actors/affordance.js +12 -12
- package/dist/actors/agent-session.d.ts +2 -2
- package/dist/actors/agent-session.js +7 -7
- package/dist/actors/codex/app-server-actor-trace.d.ts +1 -1
- package/dist/actors/codex/app-server-actor-trace.js +1 -1
- package/dist/actors/codex/app-server-trace.js +2 -1
- package/dist/actors/codex/app-server-ui.js +1 -1
- package/dist/actors/codex/codex-admission.d.ts +91 -0
- package/dist/actors/codex/codex-admission.js +150 -0
- package/dist/actors/codex/restricted-executable.d.ts +4 -4
- package/dist/actors/codex/restricted-executable.js +8 -8
- package/dist/actors/codex/restricted-launch.d.ts +2 -1
- package/dist/actors/codex/restricted-launch.js +1 -1
- package/dist/actors/codex/restricted-participant.js +20 -18
- package/dist/actors/codex/restricted-policy.d.ts +2 -2
- package/dist/actors/codex/restricted-policy.js +2 -2
- package/dist/actors/codex/restricted-session.d.ts +3 -3
- package/dist/actors/codex/restricted-session.js +10 -5
- package/dist/actors/computer-use/actor.d.ts +18 -18
- package/dist/actors/computer-use/actor.js +4 -4
- package/dist/actors/computer-use/admission-limit.d.ts +3 -3
- package/dist/actors/computer-use/admission-limit.js +4 -4
- package/dist/actors/computer-use/executor-error.d.ts +2 -2
- package/dist/actors/computer-use/executor-error.js +3 -3
- package/dist/actors/computer-use/loop/actions.d.ts +3 -3
- package/dist/actors/computer-use/loop/actions.js +15 -15
- package/dist/actors/computer-use/loop/backstop.d.ts +3 -3
- package/dist/actors/computer-use/loop/backstop.js +14 -12
- package/dist/actors/computer-use/loop/debrief.js +2 -2
- package/dist/actors/computer-use/loop/ending.d.ts +6 -6
- package/dist/actors/computer-use/loop/ending.js +18 -17
- package/dist/actors/computer-use/loop/observation.d.ts +2 -2
- package/dist/actors/computer-use/loop/observation.js +15 -14
- package/dist/actors/computer-use/loop/provider-call.d.ts +1 -1
- package/dist/actors/computer-use/loop/provider-call.js +12 -12
- package/dist/actors/computer-use/loop/session.d.ts +4 -4
- package/dist/actors/computer-use/loop/session.js +4 -4
- package/dist/actors/computer-use/loop/spend.d.ts +1 -1
- package/dist/actors/computer-use/loop/spend.js +1 -1
- package/dist/actors/computer-use/loop/trace.js +11 -9
- package/dist/actors/computer-use/loop/types.d.ts +47 -45
- package/dist/actors/computer-use/loop/usage.d.ts +2 -2
- package/dist/actors/computer-use/loop/usage.js +2 -2
- package/dist/actors/computer-use/loop.d.ts +6 -1
- package/dist/actors/computer-use/loop.js +26 -4
- package/dist/actors/computer-use/openai-context.d.ts +37 -0
- package/dist/actors/computer-use/openai-context.js +214 -0
- package/dist/actors/computer-use/openai-provider.d.ts +13 -10
- package/dist/actors/computer-use/openai-provider.js +52 -41
- package/dist/actors/computer-use/openai-requests.d.ts +36 -10
- package/dist/actors/computer-use/openai-requests.js +82 -15
- package/dist/actors/computer-use/openai-wire.d.ts +19 -7
- package/dist/actors/computer-use/openai-wire.js +36 -34
- package/dist/actors/computer-use/provider-error.d.ts +4 -4
- package/dist/actors/computer-use/provider-error.js +6 -6
- package/dist/actors/contract.d.ts +82 -47
- package/dist/actors/contract.js +30 -24
- package/dist/actors/local-agent/claude-session.d.ts +1 -1
- package/dist/actors/local-agent/claude-session.js +13 -12
- package/dist/actors/local-agent/cli.d.ts +9 -7
- package/dist/actors/local-agent/cli.js +24 -28
- package/dist/actors/local-agent/readiness.d.ts +2 -2
- package/dist/actors/local-agent/readiness.js +4 -4
- package/dist/actors/reasoning-effort.js +8 -8
- package/dist/actors/registry.d.ts +12 -12
- package/dist/actors/registry.js +21 -20
- package/dist/actors/scripted-browser/actor.d.ts +10 -10
- package/dist/actors/scripted-browser/actor.js +17 -17
- package/dist/actors/scripted-browser/browser-command.js +1 -1
- package/dist/actors/scripted-browser/steps.js +1 -1
- package/dist/actors/scripted-browser/types.d.ts +1 -1
- package/dist/actors/stop-conditions.d.ts +1 -1
- package/dist/analysis/automatic-completion.d.ts +7 -7
- package/dist/analysis/automatic-completion.js +1 -1
- package/dist/analysis/automatic-config.d.ts +6 -6
- package/dist/analysis/automatic-config.js +1 -1
- package/dist/analysis/codex-config.d.ts +6 -2
- package/dist/analysis/codex-config.js +22 -14
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +1 -1
- package/dist/analysis/service.js +5 -5
- package/dist/analysis/types.d.ts +6 -0
- package/dist/browser-control/client.js +16 -16
- package/dist/browser-control/dispatcher.js +4 -4
- package/dist/browser-control/protocol.js +14 -14
- package/dist/browser-control/recording-transfer.js +4 -4
- package/dist/browser-control/transport.js +7 -7
- package/dist/cli/commands/analysis-signals.d.ts +2 -2
- package/dist/cli/commands/analyze.js +19 -16
- package/dist/cli/commands/codex.js +9 -9
- package/dist/cli/commands/comms.js +48 -35
- package/dist/cli/commands/feedback.js +20 -21
- package/dist/cli/commands/migrate.d.ts +4 -0
- package/dist/cli/commands/migrate.js +65 -0
- package/dist/cli/commands/observe.d.ts +0 -1
- package/dist/cli/commands/observe.js +136 -112
- package/dist/cli/commands/run-command.d.ts +10 -0
- package/dist/cli/commands/run-command.js +85 -0
- package/dist/cli/commands/runs.js +136 -139
- package/dist/cli/commands/setup.d.ts +2 -2
- package/dist/cli/commands/setup.js +27 -24
- package/dist/cli/commands/study-format.d.ts +9 -0
- package/dist/cli/commands/{lab-format.js → study-format.js} +39 -27
- package/dist/cli/commands/{lab-route-computer-use.d.ts → study-route-computer-use.d.ts} +6 -6
- package/dist/cli/commands/{lab-route-computer-use.js → study-route-computer-use.js} +17 -18
- package/dist/cli/commands/{lab-route-open.d.ts → study-route-open.d.ts} +4 -4
- package/dist/cli/commands/{lab-route-open.js → study-route-open.js} +2 -2
- package/dist/cli/commands/{lab-route-preview.d.ts → study-route-preview.d.ts} +7 -7
- package/dist/cli/commands/{lab-route-preview.js → study-route-preview.js} +7 -6
- package/dist/cli/commands/study-route-run.d.ts +19 -0
- package/dist/cli/commands/{lab-route-run.js → study-route-run.js} +6 -6
- package/dist/cli/commands/study-route-scripted.d.ts +17 -0
- package/dist/cli/commands/{lab-route-scripted.js → study-route-scripted.js} +4 -4
- package/dist/cli/commands/{lab-route-shared-world.d.ts → study-route-shared-world.d.ts} +5 -5
- package/dist/cli/commands/{lab-route-shared-world.js → study-route-shared-world.js} +10 -10
- package/dist/cli/commands/study-route-terminal.d.ts +17 -0
- package/dist/cli/commands/{lab-route-terminal.js → study-route-terminal.js} +4 -4
- package/dist/cli/commands/study-run.d.ts +9 -0
- package/dist/cli/commands/{lab-run.js → study-run.js} +44 -45
- package/dist/cli/commands/{lab-scorer.d.ts → study-scorer.d.ts} +7 -7
- package/dist/cli/commands/{lab-scorer.js → study-scorer.js} +2 -2
- package/dist/cli/commands/study.d.ts +3 -0
- package/dist/cli/commands/study.js +146 -0
- package/dist/cli/commands/tui.d.ts +4 -4
- package/dist/cli/commands/tui.js +31 -31
- package/dist/cli/commands/watch.js +71 -121
- package/dist/cli/deprecations.d.ts +6 -0
- package/dist/cli/deprecations.js +18 -0
- package/dist/cli/doctor.d.ts +18 -8
- package/dist/cli/doctor.js +114 -59
- package/dist/cli/first-run-path.d.ts +15 -13
- package/dist/cli/first-run-path.js +35 -33
- package/dist/cli/io.d.ts +44 -9
- package/dist/cli/io.js +98 -23
- package/dist/cli/observer-follow.d.ts +3 -3
- package/dist/cli/observer-follow.js +6 -7
- package/dist/cli/orientation.d.ts +10 -7
- package/dist/cli/orientation.js +89 -47
- package/dist/cli/product-sentence.d.ts +6 -0
- package/dist/cli/product-sentence.js +6 -0
- package/dist/cli/program.d.ts +6 -1
- package/dist/cli/program.js +111 -43
- package/dist/cli/telemetry.d.ts +16 -16
- package/dist/cli/telemetry.js +46 -44
- package/dist/cli.js +1 -1
- package/dist/comms/capture-surface.d.ts +13 -12
- package/dist/comms/capture-surface.js +23 -22
- package/dist/comms/catch-host.d.ts +7 -7
- package/dist/comms/catch-host.js +17 -17
- package/dist/comms/connections.d.ts +1 -1
- package/dist/comms/connections.js +2 -2
- package/dist/comms/email-catch.d.ts +3 -3
- package/dist/comms/email-catch.js +8 -8
- package/dist/comms/external-evidence.d.ts +5 -5
- package/dist/comms/external-evidence.js +5 -4
- package/dist/comms/extract.js +2 -2
- package/dist/comms/fake-inbox.d.ts +3 -3
- package/dist/comms/fake-inbox.js +3 -3
- package/dist/comms/receiving-render.d.ts +1 -1
- package/dist/comms/receiving-render.js +2 -2
- package/dist/comms/receiving-runtime.d.ts +4 -4
- package/dist/comms/sandbox-catch-script.d.ts +6 -6
- package/dist/comms/sandbox-catch-script.js +6 -6
- package/dist/comms/sandbox-catch.d.ts +38 -39
- package/dist/comms/sandbox-catch.js +36 -37
- package/dist/comms/setup.d.ts +2 -2
- package/dist/comms/setup.js +56 -25
- package/dist/comms/thread-evidence.d.ts +1 -1
- package/dist/comms/thread-evidence.js +6 -6
- package/dist/comms/types.d.ts +6 -6
- package/dist/comms/types.js +3 -3
- package/dist/evidence/frame-signature.d.ts +1 -1
- package/dist/evidence/frame-signature.js +4 -4
- package/dist/evidence/redaction.d.ts +3 -3
- package/dist/evidence/redaction.js +4 -4
- package/dist/feedback/draft.js +38 -80
- package/dist/feedback/export-bundle.js +1 -1
- package/dist/feedback/export.d.ts +5 -1
- package/dist/feedback/export.js +10 -10
- package/dist/feedback/feedback.d.ts +2 -2
- package/dist/feedback/feedback.js +21 -3
- package/dist/feedback/proof.d.ts +1 -1
- package/dist/feedback/proof.js +2 -2
- package/dist/guest/bootstrap.js +6 -6
- package/dist/guest/browser-tools.js +8 -8
- package/dist/guest/chromium-text.js +19 -19
- package/dist/guest/desktop-executor.d.ts +2 -2
- package/dist/guest/desktop-executor.js +20 -20
- package/dist/guest/desktop-keys.js +4 -4
- package/dist/guest/desktop-media.d.ts +2 -2
- package/dist/guest/desktop-media.js +16 -16
- package/dist/guest/desktop-native.d.ts +1 -1
- package/dist/guest/desktop-native.js +10 -10
- package/dist/guest/runtime-desktop.js +2 -2
- package/dist/guest/runtime-processes.d.ts +2 -2
- package/dist/guest/runtime-processes.js +2 -2
- package/dist/guest/runtime.js +6 -6
- package/dist/index.d.ts +14 -14
- package/dist/index.js +8 -8
- package/dist/keys/key-resolution.d.ts +5 -5
- package/dist/keys/key-resolution.js +23 -23
- package/dist/observer/artifact.d.ts +2 -2
- package/dist/observer/artifact.js +7 -7
- package/dist/observer/data.d.ts +6 -6
- package/dist/observer/data.js +5 -4
- package/dist/observer/exposure.d.ts +1 -1
- package/dist/observer/exposure.js +11 -11
- package/dist/observer/http.js +1 -1
- package/dist/observer/library.d.ts +3 -1
- package/dist/observer/library.js +8 -3
- package/dist/observer/listen.js +5 -4
- package/dist/observer/render.d.ts +1 -1
- package/dist/observer/render.js +17 -6
- package/dist/observer/run-routes.d.ts +4 -3
- package/dist/observer/run-routes.js +38 -13
- package/dist/observer/serve.d.ts +1 -1
- package/dist/observer/serve.js +7 -3
- package/dist/observer/tunnel.js +1 -1
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle-parts.d.ts +9 -9
- package/dist/routes/computer-use/bundle-parts.js +18 -17
- package/dist/routes/computer-use/bundle.d.ts +2 -2
- package/dist/routes/computer-use/bundle.js +10 -9
- package/dist/routes/computer-use/e2b-desktop/comms.d.ts +6 -6
- package/dist/routes/computer-use/e2b-desktop/comms.js +18 -17
- package/dist/routes/computer-use/e2b-desktop/desktop.js +4 -4
- package/dist/routes/computer-use/e2b-desktop/fidelity.d.ts +5 -4
- package/dist/routes/computer-use/e2b-desktop/fidelity.js +11 -10
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.js +10 -10
- package/dist/routes/computer-use/e2b-desktop/start.js +7 -7
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +2 -2
- package/dist/routes/computer-use/e2b-desktop/teardown.js +2 -2
- package/dist/routes/computer-use/external-comms.d.ts +4 -4
- package/dist/routes/computer-use/external-comms.js +1 -1
- package/dist/routes/computer-use/fanout-bundle.js +27 -26
- package/dist/routes/computer-use/fanout-records.js +12 -13
- package/dist/routes/computer-use/live-flush.d.ts +4 -4
- package/dist/routes/computer-use/live-flush.js +6 -6
- package/dist/routes/computer-use/live-phase.d.ts +5 -4
- package/dist/routes/computer-use/live-phase.js +8 -8
- package/dist/routes/computer-use/local-tree-pack.d.ts +5 -5
- package/dist/routes/computer-use/local-tree-pack.js +1 -1
- package/dist/routes/computer-use/local-vm.d.ts +8 -8
- package/dist/routes/computer-use/local-vm.js +3 -3
- package/dist/routes/computer-use/participant-desktop.d.ts +7 -7
- package/dist/routes/computer-use/participant-desktop.js +5 -5
- package/dist/routes/computer-use/participant-execution.d.ts +4 -4
- package/dist/routes/computer-use/participant-execution.js +10 -10
- package/dist/routes/computer-use/participant-model.d.ts +6 -0
- package/dist/routes/computer-use/participant-model.js +33 -15
- package/dist/routes/computer-use/participant-prompt.d.ts +15 -15
- package/dist/routes/computer-use/participant-prompt.js +9 -9
- package/dist/routes/computer-use/participant-runs.d.ts +16 -16
- package/dist/routes/computer-use/participant-runs.js +26 -25
- package/dist/routes/computer-use/plan.d.ts +12 -12
- package/dist/routes/computer-use/plan.js +42 -42
- package/dist/routes/computer-use/preflight.d.ts +5 -5
- package/dist/routes/computer-use/preflight.js +25 -25
- package/dist/routes/computer-use/rerun-selection.d.ts +1 -1
- package/dist/routes/computer-use/rerun-selection.js +4 -3
- package/dist/routes/computer-use/result.d.ts +4 -4
- package/dist/routes/computer-use/result.js +22 -21
- package/dist/routes/computer-use/route.d.ts +12 -11
- package/dist/routes/computer-use/route.js +31 -30
- package/dist/routes/computer-use/self-report.d.ts +7 -7
- package/dist/routes/computer-use/self-report.js +32 -32
- package/dist/routes/computer-use/setup.d.ts +23 -23
- package/dist/routes/computer-use/setup.js +28 -29
- package/dist/routes/computer-use/single-bundle.d.ts +11 -11
- package/dist/routes/computer-use/single-bundle.js +40 -41
- package/dist/routes/computer-use/subject-projection.d.ts +4 -4
- package/dist/routes/computer-use/subject-projection.js +6 -6
- package/dist/routes/computer-use/types.d.ts +71 -73
- package/dist/routes/computer-use/types.js +12 -12
- package/dist/routes/preview.d.ts +13 -7
- package/dist/routes/preview.js +11 -7
- package/dist/routes/scripted/bundle.d.ts +7 -7
- package/dist/routes/scripted/bundle.js +20 -18
- package/dist/routes/scripted/plan.d.ts +8 -8
- package/dist/routes/scripted/plan.js +13 -13
- package/dist/routes/scripted/result.d.ts +3 -3
- package/dist/routes/scripted/result.js +13 -13
- package/dist/routes/scripted/route.d.ts +9 -9
- package/dist/routes/scripted/route.js +15 -16
- package/dist/routes/scripted/scenario.d.ts +1 -1
- package/dist/routes/scripted/scenario.js +5 -5
- package/dist/routes/scripted/setup.d.ts +6 -6
- package/dist/routes/scripted/setup.js +8 -8
- package/dist/routes/scripted/subject.d.ts +5 -5
- package/dist/routes/scripted/subject.js +4 -4
- package/dist/routes/scripted/surface-records.d.ts +1 -1
- package/dist/routes/scripted/surface-records.js +4 -4
- package/dist/routes/scripted/surface-sessions.d.ts +3 -3
- package/dist/routes/scripted/types.d.ts +18 -20
- package/dist/routes/scripted/types.js +2 -2
- package/dist/routes/shared-world/bundle-records.js +11 -10
- package/dist/routes/shared-world/bundle.d.ts +5 -5
- package/dist/routes/shared-world/bundle.js +46 -43
- package/dist/routes/shared-world/checkpoints.d.ts +4 -4
- package/dist/routes/shared-world/checkpoints.js +2 -2
- package/dist/routes/shared-world/comms.d.ts +14 -14
- package/dist/routes/shared-world/comms.js +19 -18
- package/dist/routes/shared-world/external-public.d.ts +4 -3
- package/dist/routes/shared-world/external-public.js +16 -12
- package/dist/routes/shared-world/handoff.d.ts +7 -6
- package/dist/routes/shared-world/handoff.js +65 -57
- package/dist/routes/shared-world/lobby-code.d.ts +2 -2
- package/dist/routes/shared-world/lobby-code.js +9 -9
- package/dist/routes/shared-world/participant-specs.d.ts +10 -9
- package/dist/routes/shared-world/participant-specs.js +38 -31
- package/dist/routes/shared-world/plan.d.ts +9 -9
- package/dist/routes/shared-world/plan.js +16 -16
- package/dist/routes/shared-world/provenance.d.ts +2 -2
- package/dist/routes/shared-world/provenance.js +2 -2
- package/dist/routes/shared-world/provisioned.d.ts +8 -8
- package/dist/routes/shared-world/provisioned.js +18 -17
- package/dist/routes/shared-world/result.d.ts +6 -6
- package/dist/routes/shared-world/result.js +29 -29
- package/dist/routes/shared-world/route.d.ts +11 -11
- package/dist/routes/shared-world/route.js +40 -39
- package/dist/routes/shared-world/setup.d.ts +16 -16
- package/dist/routes/shared-world/setup.js +23 -23
- package/dist/routes/shared-world/types.d.ts +55 -53
- package/dist/routes/shared-world/types.js +4 -4
- package/dist/routes/terminal/adapter.d.ts +5 -5
- package/dist/routes/terminal/adapter.js +9 -9
- package/dist/routes/terminal/bundle.d.ts +13 -13
- package/dist/routes/terminal/bundle.js +28 -27
- package/dist/routes/terminal/credentials.d.ts +8 -8
- package/dist/routes/terminal/credentials.js +12 -12
- package/dist/routes/terminal/dry-run.d.ts +4 -4
- package/dist/routes/terminal/dry-run.js +12 -12
- package/dist/routes/terminal/encoding.js +2 -2
- package/dist/routes/terminal/ledger.d.ts +5 -5
- package/dist/routes/terminal/ledger.js +14 -14
- package/dist/routes/terminal/lifetime.d.ts +1 -1
- package/dist/routes/terminal/lifetime.js +1 -1
- package/dist/routes/terminal/live-finish.d.ts +2 -2
- package/dist/routes/terminal/live-finish.js +10 -10
- package/dist/routes/terminal/live-sandbox.d.ts +2 -2
- package/dist/routes/terminal/live-sandbox.js +15 -15
- package/dist/routes/terminal/participant-text.js +2 -1
- package/dist/routes/terminal/persona.d.ts +2 -2
- package/dist/routes/terminal/persona.js +3 -3
- package/dist/routes/terminal/plan.d.ts +7 -7
- package/dist/routes/terminal/plan.js +15 -15
- package/dist/routes/terminal/recorder.js +2 -2
- package/dist/routes/terminal/result.d.ts +5 -5
- package/dist/routes/terminal/result.js +14 -14
- package/dist/routes/terminal/route.d.ts +9 -9
- package/dist/routes/terminal/route.js +19 -20
- package/dist/routes/terminal/sandbox.d.ts +1 -1
- package/dist/routes/terminal/sandbox.js +1 -1
- package/dist/routes/terminal/session.d.ts +5 -5
- package/dist/routes/terminal/session.js +14 -13
- package/dist/routes/terminal/token-usage.d.ts +1 -1
- package/dist/routes/terminal/token-usage.js +7 -8
- package/dist/routes/terminal/trace.d.ts +4 -4
- package/dist/routes/terminal/trace.js +2 -2
- package/dist/routes/terminal/types.d.ts +48 -50
- package/dist/routes/terminal/types.js +5 -5
- package/dist/run/bundle-shape.js +3 -3
- package/dist/run/bundle.d.ts +49 -49
- package/dist/run/bundle.js +3 -2
- package/dist/run/cost-summary.js +3 -3
- package/dist/run/costs.d.ts +23 -0
- package/dist/run/costs.js +79 -44
- package/dist/run/detail.d.ts +6 -6
- package/dist/run/detail.js +4 -4
- package/dist/run/dry-run-inputs.d.ts +1 -1
- package/dist/run/dry-run-inputs.js +2 -2
- package/dist/run/dry-run.js +20 -20
- package/dist/run/judge.d.ts +19 -13
- package/dist/run/judge.js +18 -8
- package/dist/run/outcomes.d.ts +1 -1
- package/dist/run/participant.d.ts +1 -1
- package/dist/run/paths.d.ts +1 -1
- package/dist/run/paths.js +1 -1
- package/dist/run/preflight-receipts.js +1 -1
- package/dist/run/pricing.d.ts +11 -6
- package/dist/run/pricing.js +22 -15
- package/dist/run/projection.d.ts +46 -46
- package/dist/run/projection.js +57 -57
- package/dist/run/reclaim.d.ts +1 -1
- package/dist/run/reclaim.js +3 -3
- package/dist/run/results.d.ts +5 -5
- package/dist/run/run-cost.d.ts +53 -0
- package/dist/run/run-cost.js +105 -0
- package/dist/run/run-index-analysis.d.ts +15 -0
- package/dist/run/run-index-analysis.js +77 -0
- package/dist/run/run-index.d.ts +19 -7
- package/dist/run/run-index.js +77 -64
- package/dist/run/run-not-found.d.ts +5 -0
- package/dist/run/run-not-found.js +15 -0
- package/dist/run/run.d.ts +6 -5
- package/dist/run/run.js +3 -3
- package/dist/run/sandbox-receipts.d.ts +2 -2
- package/dist/run/sandbox-receipts.js +3 -3
- package/dist/run/shared-world-evidence.d.ts +37 -36
- package/dist/run/shared-world-evidence.js +4 -4
- package/dist/run/shared-world-shape.d.ts +1 -1
- package/dist/run/shared-world-shape.js +7 -7
- package/dist/run/stats.d.ts +14 -8
- package/dist/run/stats.js +121 -63
- package/dist/run/status.d.ts +16 -29
- package/dist/run/status.js +10 -29
- package/dist/run/stored-runs.js +2 -1
- package/dist/run/stream-shape.js +1 -1
- package/dist/run/streams.d.ts +7 -7
- package/dist/run/study-provenance.d.ts +49 -0
- package/dist/run/study-provenance.js +66 -0
- package/dist/run/study-result.d.ts +11 -0
- package/dist/run/study-result.js +9 -0
- package/dist/run/text.d.ts +2 -0
- package/dist/run/text.js +4 -0
- package/dist/run-study.d.ts +112 -0
- package/dist/{run-lab.js → run-study.js} +52 -47
- package/dist/{lab → study}/adapter-extension.d.ts +15 -27
- package/dist/{lab → study}/adapter-extension.js +10 -45
- package/dist/{lab → study}/adapter-scorer-loader.d.ts +17 -17
- package/dist/{lab → study}/adapter-scorer-loader.js +34 -34
- package/dist/study/composition-rules.d.ts +7 -0
- package/dist/{lab → study}/composition-rules.js +53 -53
- package/dist/study/config.d.ts +15 -0
- package/dist/{lab → study}/config.js +98 -41
- package/dist/study/convert.d.ts +33 -0
- package/dist/study/convert.js +432 -0
- package/dist/{lab → study}/device-presets.d.ts +10 -10
- package/dist/{lab → study}/device-presets.js +11 -11
- package/dist/study/discover.d.ts +75 -0
- package/dist/{lab → study}/discover.js +134 -95
- package/dist/{lab → study}/doctor.d.ts +19 -12
- package/dist/{lab → study}/doctor.js +58 -20
- package/dist/study/field-paths.d.ts +12 -0
- package/dist/study/field-paths.js +126 -0
- package/dist/study/files.d.ts +26 -0
- package/dist/study/files.js +44 -0
- package/dist/{lab → study}/init-templates.d.ts +2 -2
- package/dist/study/init-templates.js +421 -0
- package/dist/{lab → study}/init.d.ts +2 -2
- package/dist/{lab → study}/init.js +46 -11
- package/dist/study/keys.d.ts +4 -0
- package/dist/{lab → study}/keys.js +46 -12
- package/dist/study/migrate.d.ts +48 -0
- package/dist/study/migrate.js +408 -0
- package/dist/study/owned-files.d.ts +38 -0
- package/dist/study/owned-files.js +182 -0
- package/dist/{lab → study}/parse/actors.d.ts +5 -5
- package/dist/{lab → study}/parse/actors.js +14 -14
- package/dist/study/parse/comms.d.ts +12 -0
- package/dist/{lab → study}/parse/comms.js +5 -5
- package/dist/study/parse/execution.d.ts +19 -0
- package/dist/{lab → study}/parse/execution.js +13 -13
- package/dist/study/parse/study-v3.d.ts +21 -0
- package/dist/study/parse/study-v3.js +259 -0
- package/dist/study/parse/subject-state.d.ts +30 -0
- package/dist/{lab → study}/parse/subject-state.js +19 -19
- package/dist/{lab → study}/parse/subject.d.ts +3 -3
- package/dist/{lab → study}/parse/subject.js +34 -34
- package/dist/{lab → study}/parse/values.d.ts +3 -3
- package/dist/{lab → study}/parse/values.js +3 -3
- package/dist/{lab → study}/persona-resolve.d.ts +8 -8
- package/dist/{lab → study}/persona-resolve.js +6 -6
- package/dist/{lab → study}/plan-base.d.ts +7 -7
- package/dist/{lab → study}/plan-base.js +1 -1
- package/dist/{lab → study}/plan-participants.d.ts +12 -12
- package/dist/{lab → study}/plan-participants.js +4 -4
- package/dist/{lab → study}/plan-types.d.ts +31 -31
- package/dist/{lab → study}/plan-types.js +2 -2
- package/dist/study/plan.d.ts +13 -0
- package/dist/study/plan.js +101 -0
- package/dist/study/preflight-probes.d.ts +3 -0
- package/dist/{lab → study}/preflight-probes.js +20 -19
- package/dist/study/preflight-result.d.ts +8 -0
- package/dist/{lab → study}/preflight-result.js +5 -4
- package/dist/study/preflight.d.ts +94 -0
- package/dist/{lab → study}/preflight.js +12 -12
- package/dist/{lab → study}/requirements.d.ts +3 -3
- package/dist/{lab → study}/requirements.js +1 -1
- package/dist/study/route-inputs.d.ts +28 -0
- package/dist/{lab → study}/route-inputs.js +1 -1
- package/dist/{lab → study}/routing.d.ts +39 -33
- package/dist/{lab → study}/routing.js +60 -29
- package/dist/{lab/run-lab-events.d.ts → study/run-study-events.d.ts} +10 -13
- package/dist/{lab/run-lab-events.js → study/run-study-events.js} +3 -9
- package/dist/{lab/run-lab-homes.d.ts → study/run-study-homes.d.ts} +21 -18
- package/dist/{lab/run-lab-homes.js → study/run-study-homes.js} +1 -1
- package/dist/study/run-study-options.d.ts +28 -0
- package/dist/{lab/run-lab-options.js → study/run-study-options.js} +29 -25
- package/dist/{lab/lab-deps.d.ts → study/study-deps.d.ts} +8 -13
- package/dist/{lab/lab-deps.js → study/study-deps.js} +2 -2
- package/dist/{lab → study}/summary.d.ts +16 -15
- package/dist/{lab → study}/summary.js +21 -21
- package/dist/{lab → study}/tasks.d.ts +18 -18
- package/dist/{lab → study}/tasks.js +14 -14
- package/dist/{lab → study}/types.d.ts +202 -196
- package/dist/{lab → study}/types.js +3 -1
- package/dist/{lab → study}/validation.d.ts +27 -27
- package/dist/{lab → study}/validation.js +53 -51
- package/dist/study/warnings.d.ts +12 -0
- package/dist/{lab → study}/warnings.js +62 -34
- package/dist/subject/clone.d.ts +7 -7
- package/dist/subject/clone.js +2 -2
- package/dist/subject/desktop-cli.d.ts +1 -1
- package/dist/subject/desktop-cli.js +2 -2
- package/dist/subject/local-tree-archive.d.ts +2 -2
- package/dist/subject/local-tree-archive.js +4 -4
- package/dist/subject/local-tree.d.ts +4 -4
- package/dist/subject/node-bootstrap.js +1 -1
- package/dist/subject/runtime.js +9 -10
- package/dist/subject/serve.d.ts +5 -5
- package/dist/subject/serve.js +3 -3
- package/dist/subject/state.d.ts +4 -4
- package/dist/subject/state.js +1 -1
- package/dist/subject/steps.d.ts +6 -6
- package/dist/subject/steps.js +5 -5
- package/dist/substrates/command-failure.d.ts +3 -3
- package/dist/substrates/command-failure.js +3 -3
- package/dist/substrates/desktop-session.js +2 -2
- package/dist/substrates/detached.d.ts +6 -6
- package/dist/substrates/detached.js +7 -7
- package/dist/substrates/e2b/cdp-probe.d.ts +9 -9
- package/dist/substrates/e2b/cdp-probe.js +14 -17
- package/dist/substrates/e2b/desktop-browser.d.ts +6 -6
- package/dist/substrates/e2b/desktop-browser.js +9 -9
- package/dist/substrates/e2b/desktop-cdp.d.ts +10 -10
- package/dist/substrates/e2b/desktop-cdp.js +9 -9
- package/dist/substrates/e2b/desktop-executor.d.ts +2 -2
- package/dist/substrates/e2b/desktop-executor.js +13 -13
- package/dist/substrates/e2b/desktop-geometry.d.ts +4 -4
- package/dist/substrates/e2b/desktop-geometry.js +8 -8
- package/dist/substrates/e2b/desktop-media.d.ts +4 -4
- package/dist/substrates/e2b/desktop-media.js +1 -1
- package/dist/substrates/e2b/desktop-screenshot-cleanup.d.ts +3 -3
- package/dist/substrates/e2b/desktop-screenshot-cleanup.js +3 -3
- package/dist/substrates/e2b/lifetime.js +1 -1
- package/dist/substrates/e2b/sandbox.d.ts +4 -4
- package/dist/substrates/e2b/sandbox.js +2 -2
- package/dist/substrates/e2b/sdk.d.ts +12 -12
- package/dist/substrates/e2b/sdk.js +7 -7
- package/dist/substrates/local/runtime-config.d.ts +4 -4
- package/dist/substrates/local/runtime-config.js +4 -4
- package/dist/tui/actions.d.ts +1 -1
- package/dist/tui/actions.js +13 -13
- package/dist/tui/contract.d.ts +23 -23
- package/dist/tui/contract.js +11 -11
- package/dist/tui/launch.d.ts +7 -7
- package/dist/tui/launch.js +36 -31
- package/dist/tui/project.js +5 -5
- package/dist/tui-app.js +139 -139
- package/dist/verify/actor.d.ts +8 -8
- package/dist/verify/actor.js +36 -36
- package/dist/verify/artifacts.d.ts +2 -2
- package/dist/verify/artifacts.js +3 -3
- package/dist/verify/context-growth.d.ts +9 -0
- package/dist/verify/context-growth.js +62 -0
- package/dist/verify/costs.d.ts +6 -6
- package/dist/verify/costs.js +8 -8
- package/dist/verify/shared-world-concurrent.d.ts +2 -2
- package/dist/verify/shared-world-concurrent.js +79 -76
- package/dist/verify/shared-world.d.ts +2 -2
- package/dist/verify/shared-world.js +16 -16
- package/dist/verify/subject.d.ts +5 -5
- package/dist/verify/subject.js +19 -19
- package/dist/verify/verify.js +94 -44
- package/docs/README.md +25 -32
- package/docs/architecture/actor-contract.md +60 -91
- package/docs/architecture/browser-control.md +1 -1
- package/docs/architecture/desktop-sessions.md +7 -7
- package/docs/architecture/external-public-shared-world.md +68 -65
- package/docs/architecture/github-feedback-loop.md +2 -5
- package/docs/architecture/guest-desktop.md +2 -2
- package/docs/architecture/observer.md +34 -34
- package/docs/architecture/restricted-codex-analysis.md +34 -28
- package/docs/architecture/serve.md +34 -37
- package/docs/architecture/state-driven-executor.md +55 -58
- package/docs/architecture/task-protocol-support.md +6 -6
- package/docs/architecture/terminal-product-route.md +70 -110
- package/docs/contracts/README.md +24 -0
- package/docs/contracts/adapter-admission.md +3 -3
- package/docs/contracts/adapter-fixtures.md +4 -7
- package/docs/contracts/core.md +1 -3
- package/docs/contracts/errors.md +62 -0
- package/docs/contracts/feedback.md +8 -9
- package/docs/contracts/policy.md +7 -12
- package/docs/contracts/run-bundle.md +42 -44
- package/docs/contracts/schemas.md +446 -368
- package/docs/contracts/study-analysis.md +7 -4
- package/docs/contracts/study-costs.md +28 -9
- package/docs/decisions/0001-run-bundle-is-the-source-of-truth.md +1 -1
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +2 -2
- package/docs/decisions/0003-contained-paths-and-cleanup-authority.md +3 -3
- package/docs/principles/actor-fidelity.md +4 -4
- package/docs/principles/engineering.md +3 -3
- package/docs/principles/invariants-and-defaults.md +38 -38
- package/docs/principles/self-driving-harness.md +1 -5
- package/docs/principles/three-roles.md +1 -1
- package/docs/product/automatic-analysis.md +7 -7
- package/examples/README.md +1 -1
- package/examples/participant/README.md +5 -5
- package/examples/participant/run.mjs +11 -10
- package/examples/scorer/README.md +3 -3
- package/examples/scorer/run.mjs +11 -12
- package/examples/scorer/scorer.mjs +3 -3
- package/package.json +16 -17
- package/skills/humanish/SKILL.md +64 -60
- package/skills/humanish/agents/openai.yaml +1 -1
- package/AGENTS.md +0 -99
- package/CONTRIBUTING.md +0 -148
- package/dist/actors/codex/qualified-versions.d.ts +0 -72
- package/dist/actors/codex/qualified-versions.js +0 -132
- package/dist/cli/commands/lab-format.d.ts +0 -8
- package/dist/cli/commands/lab-route-run.d.ts +0 -19
- package/dist/cli/commands/lab-route-scripted.d.ts +0 -17
- package/dist/cli/commands/lab-route-terminal.d.ts +0 -17
- package/dist/cli/commands/lab-run.d.ts +0 -9
- package/dist/cli/commands/lab.d.ts +0 -3
- package/dist/cli/commands/lab.js +0 -199
- package/dist/cli/renamed-options.d.ts +0 -11
- package/dist/cli/renamed-options.js +0 -18
- package/dist/lab/composition-rules.d.ts +0 -7
- package/dist/lab/config.d.ts +0 -7
- package/dist/lab/discover.d.ts +0 -65
- package/dist/lab/init-templates.js +0 -416
- package/dist/lab/keys.d.ts +0 -2
- package/dist/lab/parse/comms.d.ts +0 -12
- package/dist/lab/parse/execution.d.ts +0 -19
- package/dist/lab/parse/subject-state.d.ts +0 -30
- package/dist/lab/plan.d.ts +0 -19
- package/dist/lab/plan.js +0 -132
- package/dist/lab/preflight-probes.d.ts +0 -3
- package/dist/lab/preflight-result.d.ts +0 -7
- package/dist/lab/preflight.d.ts +0 -92
- package/dist/lab/route-inputs.d.ts +0 -28
- package/dist/lab/run-lab-options.d.ts +0 -28
- package/dist/lab/warnings.d.ts +0 -2
- package/dist/observer/static.d.ts +0 -39
- package/dist/observer/static.js +0 -369
- package/dist/run-lab.d.ts +0 -112
- package/docs/architecture/comms-inbox.md +0 -122
- package/docs/architecture/desktop-recording.md +0 -80
- package/docs/architecture/local-browser-runtime.md +0 -178
- package/docs/architecture/observer-review.md +0 -144
- package/docs/architecture/participant-media.md +0 -90
- package/docs/architecture/project-layout.md +0 -62
- package/docs/architecture/real-email-receiving.md +0 -136
- package/docs/assets/humanish-drawdb-hero.png +0 -0
- package/docs/assets/humanish-observer-hero.png +0 -0
- package/docs/goals/current.md +0 -207
- package/docs/product/lobby-trivia-3player-external-public.md +0 -77
- package/docs/product/open-source-install-experience.md +0 -240
- package/docs/ramp/README.md +0 -295
- package/docs/release/open-source-readiness.md +0 -222
- package/docs/release/public-readiness-standard.md +0 -210
- package/docs/roadmap/world-class-open-source-v0.md +0 -311
- /package/dist/{lab → study}/init-paths.d.ts +0 -0
- /package/dist/{lab → study}/init-paths.js +0 -0
- /package/dist/{lab → study}/init-plan.d.ts +0 -0
- /package/dist/{lab → study}/init-plan.js +0 -0
- /package/dist/{lab → study}/participant-assignment.d.ts +0 -0
- /package/dist/{lab → study}/participant-assignment.js +0 -0
- /package/dist/{lab → study}/persona.d.ts +0 -0
- /package/dist/{lab → study}/persona.js +0 -0
|
@@ -4,18 +4,18 @@
|
|
|
4
4
|
// values, spend-cap prices and the comms catch.
|
|
5
5
|
import { resolveAutomaticAnalysis } from "../../analysis/automatic-config.js";
|
|
6
6
|
import { actorRegistry, isCuaActorDescriptor, } from "../../actors/registry.js";
|
|
7
|
-
import { isHttpUrl, isLoopbackUrl } from "../../
|
|
8
|
-
import { subjectStateInvalidReason } from "../../
|
|
9
|
-
import { brainOf, callerBrainOf, capsOf, desktopRequirements, planBase, provisionedSubject, } from "../../
|
|
10
|
-
import { computerUseParticipants, declaredTargets } from "../../
|
|
11
|
-
import {
|
|
12
|
-
import { cloneTargetValidationReason,
|
|
13
|
-
import { desktopCliProductReason } from "../../
|
|
14
|
-
import {
|
|
7
|
+
import { isHttpUrl, isLoopbackUrl } from "../../study/parse/subject.js";
|
|
8
|
+
import { subjectStateInvalidReason } from "../../study/parse/subject-state.js";
|
|
9
|
+
import { brainOf, callerBrainOf, capsOf, desktopRequirements, planBase, provisionedSubject, } from "../../study/plan-base.js";
|
|
10
|
+
import { computerUseParticipants, declaredTargets } from "../../study/plan-participants.js";
|
|
11
|
+
import { MAX_COMPUTER_USE_PARTICIPANTS } from "../../study/routing.js";
|
|
12
|
+
import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, scenarioCapsValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
13
|
+
import { desktopCliProductReason } from "../../study/composition-rules.js";
|
|
14
|
+
import { isLocalBrowserStudy } from "../../substrates/local/runtime-config.js";
|
|
15
15
|
import { boundedConcurrency, defaultSessionTimeoutMs, resolveParticipantSandboxMs, } from "./participant-runs.js";
|
|
16
16
|
import { MAX_SANDBOX_MS } from "../../substrates/e2b/lifetime.js";
|
|
17
17
|
import {} from "./types.js";
|
|
18
|
-
/** The registered descriptor for a planned actor id;
|
|
18
|
+
/** The registered descriptor for a planned actor id; planComputerUseStudy checked the registry. */
|
|
19
19
|
export function cuaDescriptorOf(actor) {
|
|
20
20
|
const descriptor = actorRegistry[actor];
|
|
21
21
|
if (!descriptor || !isCuaActorDescriptor(descriptor))
|
|
@@ -52,7 +52,7 @@ export function cuaDeclaredState(plan) {
|
|
|
52
52
|
const { subject } = plan.runner;
|
|
53
53
|
return subject.kind === "clone" || subject.kind === "local-tree" ? subject.state : undefined;
|
|
54
54
|
}
|
|
55
|
-
/** The URL a refused
|
|
55
|
+
/** The URL a refused study's result names, from its declaration: a provisioned subject's serve URL. */
|
|
56
56
|
export function declaredAppUrl(config) {
|
|
57
57
|
const { subject } = config;
|
|
58
58
|
const provisioned = subject.source === "clone" || subject.source === "local-tree";
|
|
@@ -65,15 +65,15 @@ export function plannedAppUrl(subject) {
|
|
|
65
65
|
return "appUrl" in subject ? subject.appUrl : "";
|
|
66
66
|
}
|
|
67
67
|
const invalid = (message) => ({
|
|
68
|
-
code: "
|
|
68
|
+
code: "HUMANISH_COMPUTER_USE_SUBJECT_INVALID",
|
|
69
69
|
message,
|
|
70
70
|
});
|
|
71
71
|
/**
|
|
72
|
-
* The first reason a computer-use
|
|
72
|
+
* The first reason a computer-use study cannot start, checked before any sandbox, key or provider
|
|
73
73
|
* is touched. The parser enforces most of these too; the engine repeats them for library callers
|
|
74
74
|
* that hand it a config directly. The groups run in this order, and each returns its first reason.
|
|
75
75
|
*/
|
|
76
|
-
function
|
|
76
|
+
function cuaStudyRejection(config, hasRunSession, driving, subjectRoute) {
|
|
77
77
|
return (unsupportedDeclarationReason(config, hasRunSession, driving, subjectRoute) ??
|
|
78
78
|
subjectStructureReason(config, subjectRoute) ??
|
|
79
79
|
entryTargetReason(config, subjectRoute) ??
|
|
@@ -110,17 +110,17 @@ function subjectStructureReason(config, subjectRoute) {
|
|
|
110
110
|
if (cloneRoute &&
|
|
111
111
|
(!serve || !subjectRepo || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(subjectRepo)))
|
|
112
112
|
return invalid(!serve
|
|
113
|
-
? "clone
|
|
113
|
+
? "A clone subject on the computer-use route needs `subject.serve` (start and url): humanish starts the app in the sandbox before the participant opens it."
|
|
114
114
|
: `subject.repos[0] must be an owner/repo slug (got "${subjectRepo ?? ""}").`);
|
|
115
|
-
// A library caller that skips
|
|
115
|
+
// A library caller that skips parseStudy gets the same fail-closed shape the parser
|
|
116
116
|
// enforces, naming which requirement is missing.
|
|
117
117
|
if (localTreeRoute && (!serve || config.execution?.target !== "e2b-desktop"))
|
|
118
118
|
return invalid(!serve
|
|
119
|
-
? "local-tree subjects on the computer-use route require `subject.serve` (start + url):
|
|
119
|
+
? "local-tree subjects on the computer-use route require `subject.serve` (start + url): humanish packs and serves the working tree in the sandbox."
|
|
120
120
|
: "local-tree subjects require `execution.target: e2b-desktop`: the packed working tree is provisioned and served inside a hosted desktop sandbox.");
|
|
121
121
|
if (config.subject.state) {
|
|
122
122
|
const stateReason = !provisionedRoute
|
|
123
|
-
? "`subject.state` applies only to clone subjects or local-tree subjects (
|
|
123
|
+
? "`subject.state` applies only to clone subjects or local-tree subjects (humanish seeds the state it serves)."
|
|
124
124
|
: subjectStateInvalidReason(config.subject.state, config.subject.env);
|
|
125
125
|
if (stateReason)
|
|
126
126
|
return invalid(stateReason);
|
|
@@ -146,7 +146,7 @@ function entryTargetReason(config, subjectRoute) {
|
|
|
146
146
|
if (entryTargetSafe)
|
|
147
147
|
return undefined;
|
|
148
148
|
return {
|
|
149
|
-
code: "
|
|
149
|
+
code: "HUMANISH_COMPUTER_USE_SUBJECT_UNSAFE",
|
|
150
150
|
message: provisionedRoute || localAppSubject || !allowPublicTargets
|
|
151
151
|
? "subject.appUrl and any actors[0].lanes[].target entries must be loopback (127.0.0.1 or localhost) unless policies.allowPublicTargets is set for an app-url subject."
|
|
152
152
|
: "subject.appUrl and actors[0].lanes[].target entries must be valid http(s) URLs.",
|
|
@@ -158,14 +158,14 @@ function driverReason(driving, { localAppSubject, inProcessRoute }) {
|
|
|
158
158
|
// and would fail closed against an executor that returns no screenshot.
|
|
159
159
|
if (driving.inProcess && !driving.createProvider)
|
|
160
160
|
return {
|
|
161
|
-
code: "
|
|
162
|
-
message: "RunLabOptions.inProcess
|
|
161
|
+
code: "HUMANISH_COMPUTER_USE_EXECUTOR_NO_PROVIDER",
|
|
162
|
+
message: "RunLabOptions.inProcess needs RunLabOptions.createProvider: an in-process executor returns no screenshot, so it needs a provider that does not read images. The default OpenAI computer-use provider reads screenshots and would stop the session.",
|
|
163
163
|
};
|
|
164
164
|
// There is no built-in in-process driver for a local app.
|
|
165
165
|
if (localAppSubject && !inProcessRoute)
|
|
166
166
|
return {
|
|
167
|
-
code: "
|
|
168
|
-
message: "subject.source: local-app has no built-in driver. Supply one through
|
|
167
|
+
code: "HUMANISH_COMPUTER_USE_LOCAL_APP_NO_EXECUTOR",
|
|
168
|
+
message: "subject.source: local-app has no built-in driver. Supply one through runStudy(config, { inProcess: { executor }, createProvider }); a state-driven executor needs a non-vision provider.",
|
|
169
169
|
};
|
|
170
170
|
return undefined;
|
|
171
171
|
}
|
|
@@ -173,10 +173,10 @@ function driverReason(driving, { localAppSubject, inProcessRoute }) {
|
|
|
173
173
|
function rosterShapeReason(config) {
|
|
174
174
|
// `lanes` XOR `count`/`laneFocus`, device XOR raw resolution, cap, unique ids,
|
|
175
175
|
// allowPublicTargets with more than one participant, clone.fanout.
|
|
176
|
-
const fanoutReason =
|
|
176
|
+
const fanoutReason = computerUseValidationReason(config);
|
|
177
177
|
if (fanoutReason)
|
|
178
|
-
return { code: "
|
|
179
|
-
// The sandbox deadline is derived from the session budget, so a
|
|
178
|
+
return { code: "HUMANISH_COMPUTER_USE_FANOUT_INVALID", message: fanoutReason };
|
|
179
|
+
// The sandbox deadline is derived from the session budget, so a study can ask for a session that
|
|
180
180
|
// cannot legally be provisioned. Show the arithmetic: the provider's own error names a limit
|
|
181
181
|
// but not which knob produced it.
|
|
182
182
|
const derivedSandboxMs = resolveParticipantSandboxMs(config);
|
|
@@ -192,10 +192,10 @@ function rerunPlan({ sourceRunId, participantIds, }) {
|
|
|
192
192
|
return { sourceRunId, ...(participantIds === undefined ? {} : { participantIds }) };
|
|
193
193
|
}
|
|
194
194
|
/**
|
|
195
|
-
* Plan a computer-use
|
|
195
|
+
* Plan a computer-use study. It is called for any config handed to the computer-use runner, not only
|
|
196
196
|
* one routeOf sends here, so a config for another route gets this route's refusal.
|
|
197
197
|
*/
|
|
198
|
-
export function
|
|
198
|
+
export function planComputerUseStudy(config, input) {
|
|
199
199
|
const hasRunSession = input.hasRunSession === true;
|
|
200
200
|
const driving = input.driving ?? { inProcess: false, createProvider: false };
|
|
201
201
|
const refuse = (stage, code, message, actor) => ({
|
|
@@ -210,40 +210,40 @@ export function planComputerUseLab(config, input) {
|
|
|
210
210
|
});
|
|
211
211
|
const analysis = resolveAutomaticAnalysis(config.review?.analysis);
|
|
212
212
|
if (!analysis.ok)
|
|
213
|
-
return refuse("before-scope", "
|
|
213
|
+
return refuse("before-scope", "HUMANISH_STUDY_ANALYSIS_INVALID", analysis.message);
|
|
214
214
|
const tasksReason = taskProtocolValidationReason(config, true);
|
|
215
215
|
if (tasksReason)
|
|
216
|
-
return refuse("before-scope", "
|
|
216
|
+
return refuse("before-scope", "HUMANISH_STUDY_TASKS_UNSUPPORTED", tasksReason);
|
|
217
217
|
// The parser checks the actor too; a library caller skips the parser.
|
|
218
218
|
const actorType = config.actors[0]?.type ?? "";
|
|
219
219
|
const descriptor = actorRegistry[actorType];
|
|
220
220
|
if (!descriptor || !isCuaActorDescriptor(descriptor))
|
|
221
|
-
return refuse("in-scope", "
|
|
221
|
+
return refuse("in-scope", "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED", `actors[0].type "${actorType}" is not a registered computer-use actor.`);
|
|
222
222
|
const actor = descriptor.id;
|
|
223
|
-
const rejection =
|
|
223
|
+
const rejection = cuaStudyRejection(config, hasRunSession, driving, declaredSubjectRoute(config, driving));
|
|
224
224
|
if (rejection)
|
|
225
225
|
return refuse("in-scope", rejection.code, rejection.message, actor);
|
|
226
|
-
// A shared world runs every
|
|
227
|
-
// It comes after the rules above, so a shared-world config that breaks one of them,
|
|
228
|
-
// sends here, still gets that rule's message.
|
|
226
|
+
// A shared world runs every participant against one app; this route would run them as separate
|
|
227
|
+
// participants. It comes after the rules above, so a shared-world config that breaks one of them,
|
|
228
|
+
// which runStudyWith sends here, still gets that rule's message.
|
|
229
229
|
if (config.subject.topology === "shared-world")
|
|
230
|
-
return refuse("in-scope", "
|
|
230
|
+
return refuse("in-scope", "HUMANISH_COMPUTER_USE_SUBJECT_INVALID", "subject.topology: shared-world studies run every participant against one shared app on the shared-world route; this route runs independent participants. Run the study with runStudy or runConcurrentSharedWorld.", actor);
|
|
231
231
|
// The in-process route drives subject.appUrl on this machine and creates no desktop, so it would
|
|
232
|
-
// skip the subject a clone, local-tree or desktop-cli
|
|
232
|
+
// skip the subject a clone, local-tree or desktop-cli study declares.
|
|
233
233
|
const source = config.subject.source;
|
|
234
234
|
if (driving.inProcess &&
|
|
235
235
|
(source === "clone" || source === "local-tree" || source === "desktop-cli"))
|
|
236
|
-
return refuse("in-scope", "
|
|
236
|
+
return refuse("in-scope", "HUMANISH_COMPUTER_USE_SUBJECT_INVALID", `RunLabOptions.inProcess drives subject.appUrl in this process, and a ${source} subject needs the hosted desktop the in-process route never creates. Use an app-url or local-app subject with inProcess, or remove inProcess to run on a hosted desktop.`, actor);
|
|
237
237
|
// The parser's rule; without a product the desktop study fails later.
|
|
238
238
|
const productReason = desktopCliProductReason(config);
|
|
239
239
|
if (productReason)
|
|
240
|
-
return refuse("in-scope", "
|
|
240
|
+
return refuse("in-scope", "HUMANISH_COMPUTER_USE_SUBJECT_INVALID", productReason, actor);
|
|
241
241
|
const participants = computerUseParticipants(config, input.countOverride);
|
|
242
|
-
if (participants.length >
|
|
243
|
-
return refuse("after-personas", "
|
|
242
|
+
if (participants.length > MAX_COMPUTER_USE_PARTICIPANTS)
|
|
243
|
+
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", `Computer-use fan-out is capped at ${MAX_COMPUTER_USE_PARTICIPANTS} participants (resolved ${participants.length}); N concurrent paid desktops is real spend.`, actor);
|
|
244
244
|
const [first, ...rest] = participants;
|
|
245
245
|
if (driving.inProcess && rest.length > 0)
|
|
246
|
-
return refuse("after-personas", "
|
|
246
|
+
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", "Fan-out to more than one participant is not supported on the in-process route (RunLabOptions.inProcess): fan-out provisions one independent E2B desktop per participant, which the in-process route deliberately skips. Run a single in-process participant, or fan out on the E2B route.", actor);
|
|
247
247
|
if (first === undefined)
|
|
248
248
|
throw new Error("computerUseParticipants returned no participant");
|
|
249
249
|
const appUrl = config.subject.appUrl ?? "";
|
|
@@ -267,7 +267,7 @@ export function planComputerUseLab(config, input) {
|
|
|
267
267
|
participants: [first],
|
|
268
268
|
subject: source === "local-app" ? { kind: "local-app", appUrl } : appUrlSubject,
|
|
269
269
|
}
|
|
270
|
-
:
|
|
270
|
+
: isLocalBrowserStudy(config)
|
|
271
271
|
? { desktop: "local-vm", brain, participants: [first, ...rest], subject: appUrlSubject }
|
|
272
272
|
: { desktop: "e2b-desktop", brain, participants: [first, ...rest], subject: hosted };
|
|
273
273
|
const declared = config.execution?.concurrency;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { Brain, ComputerUsePlan } from "../../
|
|
2
|
-
import type {
|
|
3
|
-
import type {
|
|
1
|
+
import type { Brain, ComputerUsePlan } from "../../study/plan-types.js";
|
|
2
|
+
import type { StudyCommsExternal } from "../../study/types.js";
|
|
3
|
+
import type { CuaActorStudyErrorCode } from "./types.js";
|
|
4
4
|
/**
|
|
5
5
|
* The first reason a live computer-use run cannot start on this machine: missing keys, a missing
|
|
6
6
|
* or signed-out local agent, missing subject env, a spend cap with no price, or an unreachable
|
|
@@ -14,8 +14,8 @@ export declare function liveCuaRejection(args: {
|
|
|
14
14
|
env: Record<string, string | undefined>;
|
|
15
15
|
/** The plan's requirements: which keys and subject env names this run needs. */
|
|
16
16
|
requirements: ComputerUsePlan["requirements"];
|
|
17
|
-
externalCommsConfig:
|
|
17
|
+
externalCommsConfig: StudyCommsExternal | undefined;
|
|
18
18
|
}): Promise<{
|
|
19
|
-
code:
|
|
19
|
+
code: CuaActorStudyErrorCode;
|
|
20
20
|
message: string;
|
|
21
21
|
} | undefined>;
|
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import { pricedModel } from "../../
|
|
2
|
-
import { missingKeys, missingSubjectEnv } from "../../
|
|
1
|
+
import { pricedModel } from "../../study/plan-base.js";
|
|
2
|
+
import { missingKeys, missingSubjectEnv } from "../../study/requirements.js";
|
|
3
3
|
import { detectLocalAgents } from "../../actors/local-agent/cli.js";
|
|
4
4
|
import { localAgentRefusal } from "../../actors/local-agent/readiness.js";
|
|
5
5
|
import { describeMissingKeys } from "../../keys/key-resolution.js";
|
|
6
6
|
import { catchTokenOf, catchTokenRefusal } from "../../comms/external-evidence.js";
|
|
7
7
|
import { externalCatchHealthy } from "../../comms/sandbox-catch.js";
|
|
8
|
-
import { MODEL_RATES } from "../../run/pricing.js";
|
|
8
|
+
import { MODEL_RATES, unpricedCapMessage } from "../../run/pricing.js";
|
|
9
9
|
/** The computer-use code for each local-agent refusal; shared-world keeps the same kinds. */
|
|
10
10
|
const LOCAL_AGENT_REFUSAL_CODES = {
|
|
11
|
-
"agent-missing": "
|
|
12
|
-
"signin-required": "
|
|
13
|
-
unsupported: "
|
|
14
|
-
"unpriced-cap": "
|
|
11
|
+
"agent-missing": "HUMANISH_COMPUTER_USE_AGENT_MISSING",
|
|
12
|
+
"signin-required": "HUMANISH_COMPUTER_USE_AGENT_SIGNIN_REQUIRED",
|
|
13
|
+
unsupported: "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED",
|
|
14
|
+
"unpriced-cap": "HUMANISH_COMPUTER_USE_UNPRICED_CAP",
|
|
15
15
|
};
|
|
16
16
|
/**
|
|
17
17
|
* The first reason a live computer-use run cannot start on this machine: missing keys, a missing
|
|
@@ -27,7 +27,7 @@ export async function liveCuaRejection(args) {
|
|
|
27
27
|
const missing = missingKeys(requirements, env);
|
|
28
28
|
if (missing.length > 0) {
|
|
29
29
|
// The moment someone new actually hits the wall. If a signed-in coding agent is sitting
|
|
30
|
-
// right there, say so
|
|
30
|
+
// right there, say so here rather than making them go and find an API key; that detour is
|
|
31
31
|
// where most people trying humanish stop.
|
|
32
32
|
const suggestion = missing.includes("OPENAI_API_KEY")
|
|
33
33
|
? await (async () => {
|
|
@@ -35,17 +35,17 @@ export async function liveCuaRejection(args) {
|
|
|
35
35
|
return ready.length === 0
|
|
36
36
|
? ""
|
|
37
37
|
: ` ${ready.map((agent) => agent.label).join(" and ")} reports authenticated on this machine` +
|
|
38
|
-
|
|
38
|
+
`. Set actors[0].type: local-agent to use ${ready.length === 1 ? "it" : "one"} instead of a key.`;
|
|
39
39
|
})()
|
|
40
40
|
: "";
|
|
41
41
|
return {
|
|
42
|
-
code: "
|
|
43
|
-
message: `Live computer-use
|
|
42
|
+
code: "HUMANISH_COMPUTER_USE_KEYS_MISSING",
|
|
43
|
+
message: `Live computer-use studies need ${missing.join(" and ")} in the environment (values are never persisted). ${describeMissingKeys(missing, env)}${suggestion}`,
|
|
44
44
|
};
|
|
45
45
|
}
|
|
46
|
-
// A caller's createProvider makes the brain `caller`, so only the
|
|
46
|
+
// A caller's createProvider makes the brain `caller`, so only the study's own local agent is checked.
|
|
47
47
|
if (localAgent) {
|
|
48
|
-
// Refuse
|
|
48
|
+
// Refuse here, before a sandbox exists. "codex is not installed" discovered after the
|
|
49
49
|
// machine is paid for is the same information delivered at the worst possible moment.
|
|
50
50
|
const refusal = await localAgentRefusal({ agent: localAgent, env, caps });
|
|
51
51
|
if (refusal)
|
|
@@ -54,38 +54,38 @@ export async function liveCuaRejection(args) {
|
|
|
54
54
|
const unsetSubjectEnv = missingSubjectEnv(requirements, env);
|
|
55
55
|
if (unsetSubjectEnv.length > 0) {
|
|
56
56
|
return {
|
|
57
|
-
code: "
|
|
57
|
+
code: "HUMANISH_COMPUTER_USE_SUBJECT_ENV_MISSING",
|
|
58
58
|
message: `subject.env declares ${unsetSubjectEnv.join(", ")} but the environment does not provide ${unsetSubjectEnv.length === 1 ? "it" : "them"} (pass via --env-file; values are never persisted).`,
|
|
59
59
|
};
|
|
60
60
|
}
|
|
61
|
-
//
|
|
61
|
+
// Fail-closed cap: a maxUsd cap needs a measurable per-turn estimate.
|
|
62
62
|
// If the operator set execution.caps.maxUsd but src/run/pricing.ts has no rate for the resolved
|
|
63
|
-
// model, the loop could not enforce the cap
|
|
64
|
-
// runaway-retry protection. Refuse at
|
|
65
|
-
// uncapped: an unenforceable cap is more dangerous than none. The operator
|
|
66
|
-
//
|
|
63
|
+
// model, the loop could not enforce the cap, and silently running uncapped would break the
|
|
64
|
+
// runaway-retry protection. Refuse at preflight (before any sandbox/spend) rather than run
|
|
65
|
+
// uncapped: an unenforceable cap is more dangerous than none. The operator picks a priced model
|
|
66
|
+
// or removes the cap; a source checkout can also add a rate to src/run/pricing.ts.
|
|
67
67
|
if (caps.maxUsd !== undefined || caps.maxTotalUsd !== undefined) {
|
|
68
68
|
const model = pricedModel(brain);
|
|
69
69
|
const capModelId = model.trim().toLowerCase();
|
|
70
70
|
if (!MODEL_RATES[capModelId]) {
|
|
71
71
|
return {
|
|
72
|
-
code: "
|
|
73
|
-
message:
|
|
72
|
+
code: "HUMANISH_COMPUTER_USE_UNPRICED_CAP",
|
|
73
|
+
message: unpricedCapMessage(model),
|
|
74
74
|
};
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
|
-
// Adopter-hosted comms catch
|
|
77
|
+
// Adopter-hosted comms catch: fail closed before any sandbox is created, because a comms study
|
|
78
78
|
// whose catch is unreachable collects nothing while every participant still spends. The probe asserts
|
|
79
|
-
//
|
|
79
|
+
// humanish's own service marker in /health, so an adopter's proxy answering 200 for everything cannot
|
|
80
80
|
// pass for a catch.
|
|
81
81
|
const tokenRefusal = externalCommsConfig === undefined
|
|
82
82
|
? undefined
|
|
83
83
|
: catchTokenRefusal(catchTokenOf(externalCommsConfig, env));
|
|
84
84
|
if (tokenRefusal !== undefined)
|
|
85
|
-
return { code: "
|
|
85
|
+
return { code: "HUMANISH_COMPUTER_USE_COMMS_TOKEN_INVALID", message: tokenRefusal };
|
|
86
86
|
if (externalCommsConfig && !(await externalCatchHealthy(externalCommsConfig))) {
|
|
87
87
|
return {
|
|
88
|
-
code: "
|
|
88
|
+
code: "HUMANISH_COMPUTER_USE_COMMS_CATCH_UNREACHABLE",
|
|
89
89
|
message: "The external comms catch or inbox is unreachable or incompatible (GET /health must identify humanish-comms-catch and advertise recipient-inbox-v1). Update humanish on the catch host and restart it with `humanish comms catch` on that host, or drop comms.email to run without the inbox funnel.",
|
|
90
90
|
};
|
|
91
91
|
}
|
|
@@ -2,7 +2,7 @@ import { type RunRerunLineage } from "../../run/bundle.js";
|
|
|
2
2
|
import type { CuaParticipantPlan, DesktopParticipantRun } from "./types.js";
|
|
3
3
|
export declare function resolveCuaRerunSelection(args: {
|
|
4
4
|
cwd: string;
|
|
5
|
-
|
|
5
|
+
studyId: string;
|
|
6
6
|
sandboxMs: number;
|
|
7
7
|
sourceRunId: string;
|
|
8
8
|
participantIds?: string[];
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import {} from "../../run/bundle.js";
|
|
4
4
|
import { loadRunBundle } from "../../run/locate.js";
|
|
5
5
|
import {} from "../../run/streams.js";
|
|
6
|
+
import { plural } from "../../run/text.js";
|
|
6
7
|
export async function resolveCuaRerunSelection(args) {
|
|
7
8
|
const source = await loadRunBundle(args.cwd, args.sourceRunId);
|
|
8
9
|
if (!source) {
|
|
@@ -36,14 +37,14 @@ export async function resolveCuaRerunSelection(args) {
|
|
|
36
37
|
if (selectedIds.length === 0) {
|
|
37
38
|
return {
|
|
38
39
|
ok: false,
|
|
39
|
-
message: `source run ${bundle.runId} has no failed, blocked, timed
|
|
40
|
+
message: `source run ${bundle.runId} has no participant to rerun: none failed, was blocked, timed out or ended without engaging.`,
|
|
40
41
|
};
|
|
41
42
|
}
|
|
42
43
|
const missingPrior = selectedIds.filter((id) => !priorById.has(id));
|
|
43
44
|
if (missingPrior.length > 0) {
|
|
44
45
|
return {
|
|
45
46
|
ok: false,
|
|
46
|
-
message:
|
|
47
|
+
message: `${plural(missingPrior.length, "selected participant id")} ${missingPrior.length === 1 ? "was" : "were"} not present in source run ${bundle.runId}: ${missingPrior.join(", ")}`,
|
|
47
48
|
};
|
|
48
49
|
}
|
|
49
50
|
const specsById = new Map(args.participantRuns.map((spec) => [spec.planned.id, spec]));
|
|
@@ -51,7 +52,7 @@ export async function resolveCuaRerunSelection(args) {
|
|
|
51
52
|
if (missingCurrent.length > 0) {
|
|
52
53
|
return {
|
|
53
54
|
ok: false,
|
|
54
|
-
message:
|
|
55
|
+
message: `${plural(missingCurrent.length, "selected participant id")} ${missingCurrent.length === 1 ? "is" : "are"} not present in the current study file ${args.studyId}: ${missingCurrent.join(", ")}`,
|
|
55
56
|
};
|
|
56
57
|
}
|
|
57
58
|
const selectedSpecs = selectedIds.map((id) => specsById.get(id));
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { runStudyParticipants } from "./live-phase.js";
|
|
2
2
|
import type { CuaFinishFacts, CuaRunSetup } from "./setup.js";
|
|
3
|
-
import { type
|
|
3
|
+
import { type CuaActorStudyResult } from "./types.js";
|
|
4
4
|
/** Builds and publishes the final bundle, runs the adapter hooks, renders the Observer and returns the result. */
|
|
5
|
-
export declare function finishCuaRun(setup: CuaRunSetup, finish: CuaFinishFacts, ran: Extract<Awaited<ReturnType<typeof
|
|
5
|
+
export declare function finishCuaRun(setup: CuaRunSetup, finish: CuaFinishFacts, ran: Extract<Awaited<ReturnType<typeof runStudyParticipants>>, {
|
|
6
6
|
ok: true;
|
|
7
|
-
}>): Promise<
|
|
7
|
+
}>): Promise<CuaActorStudyResult>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { adapterScoreFailureMessage, applyBrowserScorer } from "../../
|
|
1
|
+
import { adapterScoreFailureMessage, applyBrowserScorer } from "../../study/adapter-extension.js";
|
|
2
2
|
import { redactText } from "../../evidence/redaction.js";
|
|
3
3
|
import { round6 } from "../../run/pricing.js";
|
|
4
4
|
import { foldScorerFailures, judgeExecution, OUTCOME_POLICIES, participantHarnessFailed, resultOk, } from "../../run/judge.js";
|
|
@@ -7,8 +7,10 @@ import { summarizeCuaDiagnostics } from "./diagnostics.js";
|
|
|
7
7
|
import { toParticipantResult } from "./participant-execution.js";
|
|
8
8
|
import { buildCuaRunBundle, judgeComputerUseRun } from "./bundle.js";
|
|
9
9
|
import { aggregateCuaSubject, projectParticipantSubjects, subjectProvenanceArg, } from "./subject-projection.js";
|
|
10
|
-
import {
|
|
10
|
+
import { CUA_FANOUT_STRATEGY, participantSubjectEnv, } from "./types.js";
|
|
11
11
|
import { plannedAppUrl } from "./plan.js";
|
|
12
|
+
import { studyResultIdentity } from "../../run/study-result.js";
|
|
13
|
+
import { plural } from "../../run/text.js";
|
|
12
14
|
/** Aggregate participant counts for the result projection. */
|
|
13
15
|
function buildParticipantSummary(outcomes, participantCount, participantPlan, dryRun) {
|
|
14
16
|
if (dryRun || !outcomes) {
|
|
@@ -51,12 +53,12 @@ function buildParticipantSummary(outcomes, participantCount, participantPlan, dr
|
|
|
51
53
|
};
|
|
52
54
|
}
|
|
53
55
|
/**
|
|
54
|
-
* The computer-use
|
|
56
|
+
* The computer-use study result for a finished run. The run passes only when the Observer rendered,
|
|
55
57
|
* every participant passed (dry-run participants pass as contracts), and no adapter score or declared scorer
|
|
56
58
|
* verdict failed; otherwise the error names the first reason.
|
|
57
59
|
*/
|
|
58
|
-
function
|
|
59
|
-
const {
|
|
60
|
+
function cuaStudyResult(args) {
|
|
61
|
+
const { studyId, cwd, runId, appUrl, dryRun, participantRuns, outcomes, subjects, aggregateSubject, participantPlan, rerunLineage, bundle, observer, receivingWarnings, aggregateWarnings, adapterWarnings, } = args;
|
|
60
62
|
const participantCount = participantRuns.length;
|
|
61
63
|
// Participant-level pass: dry-run participants are contract-ok; live ones need a passed,
|
|
62
64
|
// engaged session.
|
|
@@ -83,19 +85,19 @@ function cuaLabResult(args) {
|
|
|
83
85
|
if (ok)
|
|
84
86
|
return undefined;
|
|
85
87
|
if (adapterFailure !== undefined) {
|
|
86
|
-
return { code: "
|
|
88
|
+
return { code: "HUMANISH_COMPUTER_USE_FAILED", message: adapterFailure };
|
|
87
89
|
}
|
|
88
90
|
if (participantCount === 1) {
|
|
89
91
|
const outcome = firstOutcome;
|
|
90
92
|
return {
|
|
91
|
-
code: outcome?.failureCode ?? "
|
|
93
|
+
code: outcome?.failureCode ?? "HUMANISH_COMPUTER_USE_FAILED",
|
|
92
94
|
message: outcome?.sessionError ??
|
|
93
95
|
outcome?.providerCleanupError ??
|
|
94
96
|
outcome?.providerPolicyError ??
|
|
95
97
|
(outcome?.noEngagement
|
|
96
98
|
? "Actor took no actions and produced no message (likely a blank/still-loading screen); not a credible goal_satisfied."
|
|
97
99
|
: // The participant result (toParticipantResult) named this refusal; the N=1 envelope fell through to
|
|
98
|
-
// "did not produce a terminal session", which is false
|
|
100
|
+
// "did not produce a terminal session", which is false: it produced one and refused it.
|
|
99
101
|
outcome?.selfReportedBlocker
|
|
100
102
|
? "Actor reported goal_satisfied while its final message described a blocker or asked for missing instructions; not a credible pass."
|
|
101
103
|
: observer.ok
|
|
@@ -103,25 +105,24 @@ function cuaLabResult(args) {
|
|
|
103
105
|
? `Computer-use session ended with a harness error: ${outcome.session.reason}`
|
|
104
106
|
: outcome?.session?.status !== "passed"
|
|
105
107
|
? `Computer-use session ended with ${outcome?.session?.status ?? "unknown"}: ${outcome?.session?.reason ?? "no terminal reason"}`
|
|
106
|
-
: "
|
|
107
|
-
: (observer.error?.message ?? "Observer failed for the computer-use
|
|
108
|
+
: "The computer-use run did not produce a terminal session."
|
|
109
|
+
: (observer.error?.message ?? "Observer failed for the computer-use run.")),
|
|
108
110
|
};
|
|
109
111
|
}
|
|
110
112
|
const failing = (outcomes ?? []).find((outcome) => !participantOk(outcome));
|
|
111
|
-
const geometryFailure = (outcomes ?? []).find((outcome) => outcome.failureCode === "
|
|
112
|
-
const code = geometryFailure?.failureCode ?? "
|
|
113
|
+
const geometryFailure = (outcomes ?? []).find((outcome) => outcome.failureCode === "HUMANISH_COMPUTER_USE_DEVICE_GEOMETRY");
|
|
114
|
+
const code = geometryFailure?.failureCode ?? "HUMANISH_COMPUTER_USE_FAILED";
|
|
113
115
|
return {
|
|
114
116
|
code,
|
|
115
117
|
message: observer.ok
|
|
116
|
-
? `Fan-out run failed: ${summary.passed}/${participantCount
|
|
118
|
+
? `Fan-out run failed: ${summary.passed}/${plural(participantCount, "participant")} passed (${summary.skipped} skipped, ${plural(summary.harnessErrors, "harness error")}, ${summary.hollow} without engagement)${failing?.sessionError !== undefined ? `; first failure: ${failing.sessionError}` : failing === undefined && args.execution.failures[0] !== undefined ? `; ${args.execution.failures[0].message}` : ""}.`
|
|
117
119
|
: (observer.error?.message ?? "Observer failed for the computer-use fan-out run."),
|
|
118
120
|
};
|
|
119
121
|
})();
|
|
120
122
|
return {
|
|
121
|
-
|
|
123
|
+
...studyResultIdentity("computer-use", studyId),
|
|
122
124
|
ok,
|
|
123
125
|
cwd,
|
|
124
|
-
labId,
|
|
125
126
|
actor: args.actorId,
|
|
126
127
|
appUrl,
|
|
127
128
|
dryRun,
|
|
@@ -198,7 +199,7 @@ function computerUseExecutionFailures(outcomes, observer, runId) {
|
|
|
198
199
|
: [
|
|
199
200
|
{
|
|
200
201
|
kind: "evidence",
|
|
201
|
-
message: observer.error?.message ?? "Observer failed for the computer-use
|
|
202
|
+
message: observer.error?.message ?? "Observer failed for the computer-use run.",
|
|
202
203
|
},
|
|
203
204
|
]),
|
|
204
205
|
];
|
|
@@ -214,7 +215,7 @@ function participantCapWarning(caps, participantCount) {
|
|
|
214
215
|
return undefined;
|
|
215
216
|
if (caps.maxTotalUsd !== undefined)
|
|
216
217
|
return undefined;
|
|
217
|
-
return `execution.caps.maxUsd ($${capUsd})
|
|
218
|
+
return `execution.caps.maxUsd ($${capUsd}) caps each participant, so ${participantCount} participants may spend up to ${participantCount} × $${capUsd} (about $${round6(capUsd * participantCount)}) before any of them stops. Set execution.caps.maxTotalUsd for one budget across the study.`;
|
|
218
219
|
}
|
|
219
220
|
/** Builds and publishes the final bundle, runs the adapter hooks, renders the Observer and returns the result. */
|
|
220
221
|
export async function finishCuaRun(setup, finish, ran) {
|
|
@@ -227,7 +228,7 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
227
228
|
const { runId } = run;
|
|
228
229
|
const participantCount = participantRuns.length;
|
|
229
230
|
const { outcomes, failFastReason, receiving, receivingWarnings, externalCommsWarnings } = ran;
|
|
230
|
-
// Per-participant subject projections
|
|
231
|
+
// Per-participant subject projections.
|
|
231
232
|
const subjects = projectParticipantSubjects({ ...subjectArgs, outcomes, dryRun });
|
|
232
233
|
const aggregate = aggregateCuaSubject({ subjects, outcomes, participantCount, dryRun });
|
|
233
234
|
const aggregateSubject = aggregate.subject;
|
|
@@ -256,7 +257,7 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
256
257
|
context: {
|
|
257
258
|
bundle,
|
|
258
259
|
runDir: run.paths.physicalRunRoot,
|
|
259
|
-
|
|
260
|
+
studyId: plan.studyId,
|
|
260
261
|
runId,
|
|
261
262
|
actor: descriptor.id,
|
|
262
263
|
route: "computer-use",
|
|
@@ -275,8 +276,8 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
275
276
|
const observer = await finished.renderObserver();
|
|
276
277
|
streams.attachFinal(observer);
|
|
277
278
|
const execution = judgeExecution(computerUseExecutionFailures(outcomes, observer, runId), OUTCOME_POLICIES["computer-use"]);
|
|
278
|
-
const result =
|
|
279
|
-
|
|
279
|
+
const result = cuaStudyResult({
|
|
280
|
+
studyId: plan.studyId,
|
|
280
281
|
cwd,
|
|
281
282
|
runId,
|
|
282
283
|
actorId: descriptor.id,
|
|
@@ -1,27 +1,28 @@
|
|
|
1
|
-
import type { ComputerUsePlan } from "../../
|
|
2
|
-
import type { AdmittedPlan } from "../../run-
|
|
3
|
-
import type {
|
|
1
|
+
import type { ComputerUsePlan } from "../../study/plan-types.js";
|
|
2
|
+
import type { AdmittedPlan } from "../../run-study.js";
|
|
3
|
+
import type { StudyConfig } from "../../study/types.js";
|
|
4
4
|
import { type ComputerUseRefusal } from "./plan.js";
|
|
5
|
-
import { type ComputerUseRunInput, type
|
|
5
|
+
import { type ComputerUseRunInput, type CuaActorStudyResult, type RunCuaActorStudyOptions } from "./types.js";
|
|
6
6
|
/**
|
|
7
|
-
*
|
|
7
|
+
* Plans and runs a computer-use study in one call. It is not exported from src/index.ts; tests call
|
|
8
|
+
* it. It plans the config with planComputerUseStudy and runs
|
|
8
9
|
* the plan with runComputerUsePlan, whose run scope and withTransientCommsSecrets wrapper cover the
|
|
9
10
|
* run and its analysis.
|
|
10
11
|
*/
|
|
11
|
-
export declare function
|
|
12
|
+
export declare function runCuaActorStudy(options: RunCuaActorStudyOptions): Promise<CuaActorStudyResult>;
|
|
12
13
|
/**
|
|
13
|
-
* A refused computer-use
|
|
14
|
+
* A refused computer-use study's result, at the refusal's stage: a before-scope refusal has its own
|
|
14
15
|
* envelope and no analysis record; the others come after the cwd checks, and the participant cap after
|
|
15
16
|
* the personas are read.
|
|
16
17
|
*/
|
|
17
|
-
export declare function
|
|
18
|
+
export declare function computerUseStudyRefusal(options: RunCuaActorStudyOptions, refusal: ComputerUseRefusal): Promise<CuaActorStudyResult>;
|
|
18
19
|
/**
|
|
19
|
-
*
|
|
20
|
+
* runStudyWith's step for a computer-use plan. It runs the plan's local checks (admitCuaRun: the
|
|
20
21
|
* participants, the preflight plan, the keys, the local agent, subject env and caps, and the
|
|
21
22
|
* local-tree archive) before any run scope opens, so the CLI can present their refusal before it
|
|
22
23
|
* loads a declared scorer, and returns the run that continues from them with that scorer.
|
|
23
24
|
*/
|
|
24
|
-
export declare function admitComputerUsePlan(plan: ComputerUsePlan, input: ComputerUseRunInput, config:
|
|
25
|
+
export declare function admitComputerUsePlan(plan: ComputerUsePlan, input: ComputerUseRunInput, config: StudyConfig): Promise<AdmittedPlan<"computer-use">>;
|
|
25
26
|
/**
|
|
26
27
|
* Run a computer-use plan: its local checks, then the run. The run scope in runAdmittedCuaRun
|
|
27
28
|
* finalizes any status record the run opened, on every exit, so a test or library caller does not
|
|
@@ -29,4 +30,4 @@ export declare function admitComputerUsePlan(plan: ComputerUsePlan, input: Compu
|
|
|
29
30
|
* ENOTEMPTY). Participants, the brain, the subject, the caps and the residual config come from the
|
|
30
31
|
* plan. `config` is what the caller's createProvider and inProcess executor receive.
|
|
31
32
|
*/
|
|
32
|
-
export declare function runComputerUsePlan(plan: ComputerUsePlan, input: ComputerUseRunInput, config:
|
|
33
|
+
export declare function runComputerUsePlan(plan: ComputerUsePlan, input: ComputerUseRunInput, config: StudyConfig): Promise<CuaActorStudyResult>;
|