humanish 0.106.1 → 0.108.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +38 -32
- package/CHANGELOG.md +28 -0
- package/CONTEXT.md +51 -41
- package/README.md +152 -174
- package/SECURITY.md +16 -26
- package/TELEMETRY.md +8 -6
- package/dist/actors/affordance.d.ts +4 -4
- package/dist/actors/affordance.js +12 -12
- package/dist/actors/agent-session.d.ts +2 -2
- package/dist/actors/agent-session.js +7 -7
- package/dist/actors/codex/app-server-actor-trace.d.ts +1 -1
- package/dist/actors/codex/app-server-actor-trace.js +1 -1
- package/dist/actors/codex/app-server-client.js +2 -1
- package/dist/actors/codex/app-server-trace.d.ts +0 -1
- package/dist/actors/codex/app-server-trace.js +1 -3
- package/dist/actors/codex/app-server-ui.js +1 -1
- package/dist/actors/codex/app-server.js +2 -1
- package/dist/actors/codex/codex-admission.d.ts +91 -0
- package/dist/actors/codex/codex-admission.js +150 -0
- package/dist/actors/codex/protocol-compat.d.ts +27 -0
- package/dist/actors/codex/protocol-compat.js +357 -0
- package/dist/actors/codex/protocol-contract.d.ts +61 -0
- package/dist/actors/codex/protocol-contract.js +471 -0
- package/dist/actors/codex/restricted-executable.d.ts +28 -4
- package/dist/actors/codex/restricted-executable.js +104 -29
- package/dist/actors/codex/restricted-launch.d.ts +9 -2
- package/dist/actors/codex/restricted-launch.js +43 -4
- package/dist/actors/codex/restricted-notifications.d.ts +101 -0
- package/dist/actors/codex/restricted-notifications.js +277 -0
- package/dist/actors/codex/restricted-participant.d.ts +30 -0
- package/dist/actors/codex/restricted-participant.js +66 -6
- package/dist/actors/codex/restricted-policy.d.ts +11 -3
- package/dist/actors/codex/restricted-policy.js +2 -2
- package/dist/actors/codex/restricted-session.d.ts +16 -3
- package/dist/actors/codex/restricted-session.js +60 -22
- package/dist/actors/codex/restricted-transport.d.ts +29 -0
- package/dist/actors/codex/restricted-transport.js +139 -27
- package/dist/actors/codex/restricted-turn.d.ts +19 -4
- package/dist/actors/codex/restricted-turn.js +43 -30
- package/dist/actors/computer-use/actor.d.ts +17 -17
- package/dist/actors/computer-use/actor.js +6 -6
- package/dist/actors/computer-use/admission-limit.d.ts +1 -1
- package/dist/actors/computer-use/admission-limit.js +1 -1
- package/dist/actors/computer-use/loop/actions.d.ts +3 -3
- package/dist/actors/computer-use/loop/actions.js +10 -10
- package/dist/actors/computer-use/loop/backstop.d.ts +3 -3
- package/dist/actors/computer-use/loop/backstop.js +15 -13
- package/dist/actors/computer-use/loop/ending.d.ts +7 -7
- package/dist/actors/computer-use/loop/ending.js +9 -9
- package/dist/actors/computer-use/loop/observation.d.ts +2 -2
- package/dist/actors/computer-use/loop/observation.js +9 -9
- package/dist/actors/computer-use/loop/provider-call.d.ts +1 -1
- package/dist/actors/computer-use/loop/provider-call.js +1 -1
- package/dist/actors/computer-use/loop/session.js +3 -3
- package/dist/actors/computer-use/loop/spend.d.ts +2 -2
- package/dist/actors/computer-use/loop/spend.js +4 -4
- package/dist/actors/computer-use/loop/trace.js +11 -10
- package/dist/actors/computer-use/loop/types.d.ts +37 -26
- package/dist/actors/computer-use/loop/usage.js +2 -2
- package/dist/actors/computer-use/loop.js +4 -4
- package/dist/actors/computer-use/openai-context.d.ts +37 -0
- package/dist/actors/computer-use/openai-context.js +214 -0
- package/dist/actors/computer-use/openai-provider.d.ts +12 -9
- package/dist/actors/computer-use/openai-provider.js +47 -36
- package/dist/actors/computer-use/openai-requests.d.ts +36 -10
- package/dist/actors/computer-use/openai-requests.js +82 -15
- package/dist/actors/computer-use/openai-wire.d.ts +19 -7
- package/dist/actors/computer-use/openai-wire.js +36 -34
- package/dist/actors/contract.d.ts +86 -49
- package/dist/actors/contract.js +29 -23
- package/dist/actors/local-agent/claude-session.d.ts +1 -1
- package/dist/actors/local-agent/claude-session.js +10 -9
- package/dist/actors/local-agent/cli.d.ts +8 -6
- package/dist/actors/local-agent/cli.js +24 -28
- package/dist/actors/local-agent/readiness.d.ts +2 -2
- package/dist/actors/local-agent/readiness.js +4 -4
- package/dist/actors/reasoning-effort.js +6 -6
- package/dist/actors/registry.d.ts +10 -10
- package/dist/actors/registry.js +18 -17
- package/dist/actors/scripted-browser/actor.d.ts +8 -8
- package/dist/actors/scripted-browser/actor.js +16 -16
- package/dist/actors/scripted-browser/browser-command.js +1 -1
- package/dist/actors/scripted-browser/journey.js +15 -4
- package/dist/actors/scripted-browser/steps.js +19 -1
- package/dist/actors/scripted-browser/types.d.ts +12 -2
- package/dist/actors/stop-conditions.d.ts +1 -1
- package/dist/actors/stop-conditions.js +1 -3
- package/dist/analysis/automatic-completion.d.ts +13 -8
- package/dist/analysis/automatic-completion.js +14 -8
- package/dist/analysis/automatic-config.d.ts +1 -1
- package/dist/analysis/automatic-config.js +2 -2
- package/dist/analysis/automatic.d.ts +10 -4
- package/dist/analysis/automatic.js +17 -13
- package/dist/analysis/codex-config.d.ts +6 -2
- package/dist/analysis/codex-config.js +22 -14
- package/dist/analysis/diagnostics.d.ts +23 -0
- package/dist/analysis/diagnostics.js +112 -0
- package/dist/analysis/evidence-sources.js +2 -2
- package/dist/analysis/evidence.js +6 -6
- package/dist/analysis/execute.d.ts +7 -1
- package/dist/analysis/execute.js +39 -12
- package/dist/analysis/job.d.ts +8 -5
- package/dist/analysis/job.js +15 -14
- package/dist/analysis/load.d.ts +1 -1
- package/dist/analysis/load.js +5 -5
- package/dist/analysis/provider.d.ts +9 -1
- package/dist/analysis/restricted-codex.d.ts +2 -0
- package/dist/analysis/restricted-codex.js +10 -1
- package/dist/analysis/service.d.ts +2 -0
- package/dist/analysis/service.js +35 -16
- package/dist/analysis/sharing.d.ts +2 -2
- package/dist/analysis/sharing.js +5 -5
- package/dist/analysis/store-executions.d.ts +5 -5
- package/dist/analysis/store-executions.js +12 -12
- package/dist/analysis/store.d.ts +4 -4
- package/dist/analysis/store.js +13 -13
- package/dist/cli/commands/analysis-signals.d.ts +2 -1
- package/dist/cli/commands/analysis-signals.js +4 -1
- package/dist/cli/commands/analyze.js +23 -18
- package/dist/cli/commands/codex.js +9 -9
- package/dist/cli/commands/comms.js +62 -34
- package/dist/cli/commands/feedback.js +20 -21
- package/dist/cli/commands/lab-format.d.ts +5 -4
- package/dist/cli/commands/lab-format.js +35 -23
- package/dist/cli/commands/lab-route-computer-use.d.ts +0 -2
- package/dist/cli/commands/lab-route-computer-use.js +32 -21
- package/dist/cli/commands/lab-route-preview.d.ts +1 -3
- package/dist/cli/commands/lab-route-preview.js +7 -7
- package/dist/cli/commands/lab-route-run.d.ts +6 -4
- package/dist/cli/commands/lab-route-run.js +6 -2
- package/dist/cli/commands/lab-route-scripted.d.ts +0 -2
- package/dist/cli/commands/lab-route-scripted.js +0 -1
- package/dist/cli/commands/lab-route-shared-world.d.ts +0 -2
- package/dist/cli/commands/lab-route-shared-world.js +3 -4
- package/dist/cli/commands/lab-route-terminal.d.ts +0 -2
- package/dist/cli/commands/lab-route-terminal.js +0 -1
- package/dist/cli/commands/lab-run.js +84 -54
- package/dist/cli/commands/lab-scorer.d.ts +2 -4
- package/dist/cli/commands/lab-scorer.js +1 -1
- package/dist/cli/commands/migrate.d.ts +4 -0
- package/dist/cli/commands/migrate.js +65 -0
- package/dist/cli/commands/observe.js +168 -107
- package/dist/cli/commands/run-command.d.ts +10 -0
- package/dist/cli/commands/run-command.js +85 -0
- package/dist/cli/commands/run-signals.d.ts +30 -0
- package/dist/cli/commands/run-signals.js +146 -0
- package/dist/cli/commands/runs.js +137 -136
- package/dist/cli/commands/setup.d.ts +2 -2
- package/dist/cli/commands/setup.js +26 -21
- package/dist/cli/commands/study.d.ts +3 -0
- package/dist/cli/commands/study.js +182 -0
- package/dist/cli/commands/tui.d.ts +4 -4
- package/dist/cli/commands/tui.js +21 -21
- package/dist/cli/commands/watch.js +64 -77
- package/dist/cli/deprecations.d.ts +18 -0
- package/dist/cli/deprecations.js +48 -0
- package/dist/cli/doctor.d.ts +18 -5
- package/dist/cli/doctor.js +111 -48
- package/dist/cli/first-run-path.d.ts +12 -10
- package/dist/cli/first-run-path.js +30 -28
- package/dist/cli/io.d.ts +46 -10
- package/dist/cli/io.js +109 -30
- package/dist/cli/observer-follow.d.ts +2 -2
- package/dist/cli/observer-follow.js +5 -6
- package/dist/cli/orientation.d.ts +6 -3
- package/dist/cli/orientation.js +43 -39
- package/dist/cli/product-sentence.d.ts +6 -0
- package/dist/cli/product-sentence.js +6 -0
- package/dist/cli/program.d.ts +6 -1
- package/dist/cli/program.js +112 -43
- package/dist/cli/telemetry.d.ts +17 -17
- package/dist/cli/telemetry.js +49 -41
- package/dist/cli.js +1 -1
- package/dist/comms/capture-surface.d.ts +14 -13
- package/dist/comms/capture-surface.js +23 -22
- package/dist/comms/catch-host.d.ts +8 -8
- package/dist/comms/catch-host.js +14 -14
- package/dist/comms/connections.js +4 -4
- package/dist/comms/email-catch.d.ts +3 -3
- package/dist/comms/email-catch.js +8 -8
- package/dist/comms/external-evidence.d.ts +27 -0
- package/dist/comms/external-evidence.js +93 -0
- package/dist/comms/extract.js +2 -2
- package/dist/comms/fake-inbox.d.ts +2 -2
- package/dist/comms/fake-inbox.js +2 -2
- package/dist/comms/lease-store.js +7 -7
- package/dist/comms/receiving-render.d.ts +1 -1
- package/dist/comms/receiving-render.js +2 -2
- package/dist/comms/sandbox-catch-script.d.ts +6 -6
- package/dist/comms/sandbox-catch-script.js +6 -6
- package/dist/comms/sandbox-catch.d.ts +36 -37
- package/dist/comms/sandbox-catch.js +34 -35
- package/dist/comms/setup.js +42 -8
- package/dist/comms/thread-evidence.d.ts +1 -1
- package/dist/comms/thread-evidence.js +6 -6
- package/dist/comms/types.d.ts +6 -6
- package/dist/comms/types.js +4 -4
- package/dist/evidence/browser-hygiene.d.ts +1 -1
- package/dist/evidence/browser-hygiene.js +1 -1
- package/dist/evidence/frame-signature.d.ts +1 -1
- package/dist/evidence/frame-signature.js +4 -4
- package/dist/evidence/redaction.d.ts +3 -3
- package/dist/evidence/redaction.js +5 -5
- package/dist/evidence/secret-scrub.d.ts +9 -0
- package/dist/evidence/secret-scrub.js +96 -0
- package/dist/feedback/draft.js +41 -85
- package/dist/feedback/export-bundle.js +5 -5
- package/dist/feedback/export.d.ts +5 -1
- package/dist/feedback/export.js +16 -17
- package/dist/feedback/feedback.d.ts +3 -3
- package/dist/feedback/feedback.js +21 -3
- package/dist/guest/chromium-text.js +2 -2
- package/dist/guest/desktop-native.d.ts +1 -1
- package/dist/guest/runtime.js +1 -1
- package/dist/index.d.ts +14 -15
- package/dist/index.js +10 -11
- package/dist/keys/key-resolution.d.ts +7 -5
- package/dist/keys/key-resolution.js +40 -38
- package/dist/lab/adapter-extension.d.ts +35 -13
- package/dist/lab/adapter-extension.js +50 -134
- package/dist/lab/adapter-scorer-loader.d.ts +32 -12
- package/dist/lab/adapter-scorer-loader.js +56 -32
- package/dist/lab/composition-rules.js +58 -58
- package/dist/lab/config.js +85 -43
- package/dist/lab/deprecated-fields.d.ts +10 -0
- package/dist/lab/deprecated-fields.js +24 -0
- package/dist/lab/device-presets.d.ts +10 -10
- package/dist/lab/device-presets.js +11 -11
- package/dist/lab/discover.d.ts +13 -9
- package/dist/lab/discover.js +118 -60
- package/dist/lab/doctor.d.ts +53 -11
- package/dist/lab/doctor.js +193 -47
- package/dist/lab/init-plan.js +1 -3
- package/dist/lab/init-templates.js +147 -147
- package/dist/lab/init.d.ts +1 -1
- package/dist/lab/init.js +41 -9
- package/dist/lab/keys.d.ts +2 -0
- package/dist/lab/keys.js +43 -9
- package/dist/lab/lab-deps.d.ts +80 -0
- package/dist/lab/lab-deps.js +4 -0
- package/dist/lab/parse/actors.js +17 -16
- package/dist/lab/parse/comms.js +8 -7
- package/dist/lab/parse/execution.js +15 -14
- package/dist/lab/parse/study-v3.d.ts +23 -0
- package/dist/lab/parse/study-v3.js +259 -0
- package/dist/lab/parse/subject-state.d.ts +7 -7
- package/dist/lab/parse/subject-state.js +18 -17
- package/dist/lab/parse/subject.js +30 -29
- package/dist/lab/parse/values.d.ts +1 -2
- package/dist/lab/parse/values.js +3 -6
- package/dist/lab/persona-resolve.d.ts +7 -7
- package/dist/lab/persona-resolve.js +8 -8
- package/dist/lab/persona.js +1 -3
- package/dist/lab/plan-base.d.ts +0 -2
- package/dist/lab/plan-base.js +6 -4
- package/dist/lab/plan-participants.d.ts +7 -7
- package/dist/lab/plan-participants.js +2 -2
- package/dist/lab/plan-types.d.ts +26 -8
- package/dist/lab/plan.d.ts +4 -18
- package/dist/lab/plan.js +24 -79
- package/dist/lab/preflight-probes.js +11 -11
- package/dist/lab/preflight-result.d.ts +2 -1
- package/dist/lab/preflight-result.js +5 -6
- package/dist/lab/preflight.d.ts +7 -8
- package/dist/lab/preflight.js +6 -6
- package/dist/lab/requirements.d.ts +20 -0
- package/dist/lab/requirements.js +54 -0
- package/dist/lab/route-inputs.d.ts +23 -12
- package/dist/lab/route-inputs.js +48 -51
- package/dist/lab/routing.d.ts +42 -41
- package/dist/lab/routing.js +66 -68
- package/dist/lab/run-lab-events.d.ts +3 -11
- package/dist/lab/run-lab-events.js +2 -16
- package/dist/lab/run-lab-homes.d.ts +76 -0
- package/dist/lab/run-lab-homes.js +7 -0
- package/dist/lab/run-lab-options.d.ts +12 -66
- package/dist/lab/run-lab-options.js +58 -285
- package/dist/lab/summary.d.ts +15 -9
- package/dist/lab/summary.js +24 -17
- package/dist/lab/tasks.d.ts +15 -15
- package/dist/lab/tasks.js +14 -14
- package/dist/lab/types.d.ts +201 -195
- package/dist/lab/types.js +2 -0
- package/dist/lab/validation.d.ts +21 -21
- package/dist/lab/validation.js +61 -59
- package/dist/lab/warnings.d.ts +10 -0
- package/dist/lab/warnings.js +70 -41
- package/dist/library-aliases.d.ts +56 -0
- package/dist/library-aliases.js +22 -0
- package/dist/observer/artifact.d.ts +2 -2
- package/dist/observer/artifact.js +9 -9
- package/dist/observer/data.d.ts +6 -6
- package/dist/observer/data.js +7 -6
- package/dist/observer/exposure.d.ts +1 -1
- package/dist/observer/exposure.js +11 -11
- package/dist/observer/http.js +1 -1
- package/dist/observer/library.d.ts +1 -1
- package/dist/observer/library.js +4 -2
- package/dist/observer/listen.js +5 -4
- package/dist/observer/render.d.ts +1 -1
- package/dist/observer/render.js +12 -7
- package/dist/observer/run-routes.d.ts +4 -4
- package/dist/observer/run-routes.js +6 -6
- package/dist/observer/serve.d.ts +17 -0
- package/dist/observer/serve.js +41 -6
- package/dist/observer/tunnel.js +1 -1
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle-parts.d.ts +9 -9
- package/dist/routes/computer-use/bundle-parts.js +24 -17
- package/dist/routes/computer-use/bundle.d.ts +4 -11
- package/dist/routes/computer-use/bundle.js +12 -18
- package/dist/routes/computer-use/diagnostics.d.ts +1 -1
- package/dist/routes/computer-use/diagnostics.js +1 -1
- package/dist/routes/computer-use/e2b-desktop/comms.d.ts +11 -11
- package/dist/routes/computer-use/e2b-desktop/comms.js +24 -27
- package/dist/routes/computer-use/e2b-desktop/desktop.d.ts +2 -2
- package/dist/routes/computer-use/e2b-desktop/desktop.js +8 -11
- package/dist/routes/computer-use/e2b-desktop/fidelity.d.ts +16 -15
- package/dist/routes/computer-use/e2b-desktop/fidelity.js +20 -19
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +3 -3
- package/dist/routes/computer-use/e2b-desktop/prepare.js +40 -43
- package/dist/routes/computer-use/e2b-desktop/start.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/start.js +32 -30
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +5 -4
- package/dist/routes/computer-use/e2b-desktop/state.js +5 -3
- package/dist/routes/computer-use/e2b-desktop/teardown.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/teardown.js +33 -20
- package/dist/routes/computer-use/external-comms.d.ts +4 -4
- package/dist/routes/computer-use/external-comms.js +19 -43
- package/dist/routes/computer-use/fanout-bundle.d.ts +3 -3
- package/dist/routes/computer-use/fanout-bundle.js +38 -42
- package/dist/routes/computer-use/{fanout-lanes.js → fanout-records.js} +33 -28
- package/dist/routes/computer-use/in-process-desktop.d.ts +3 -10
- package/dist/routes/computer-use/in-process-desktop.js +3 -9
- package/dist/routes/computer-use/live-flush.d.ts +7 -5
- package/dist/routes/computer-use/live-flush.js +20 -28
- package/dist/routes/computer-use/{run-lanes.d.ts → live-phase.d.ts} +4 -2
- package/dist/routes/computer-use/{run-lanes.js → live-phase.js} +17 -16
- package/dist/routes/computer-use/local-tree-pack.d.ts +3 -3
- package/dist/routes/computer-use/local-tree-pack.js +4 -5
- package/dist/routes/computer-use/local-vm.d.ts +11 -8
- package/dist/routes/computer-use/local-vm.js +40 -67
- package/dist/routes/computer-use/participant-desktop.d.ts +8 -23
- package/dist/routes/computer-use/participant-desktop.js +5 -25
- package/dist/routes/computer-use/{lanes.d.ts → participant-execution.d.ts} +11 -11
- package/dist/routes/computer-use/{lanes.js → participant-execution.js} +39 -26
- package/dist/routes/computer-use/participant-facts.d.ts +7 -1
- package/dist/routes/computer-use/participant-facts.js +12 -1
- package/dist/routes/computer-use/participant-model.d.ts +25 -11
- package/dist/routes/computer-use/participant-model.js +76 -34
- package/dist/routes/computer-use/participant-prompt.d.ts +13 -13
- package/dist/routes/computer-use/participant-prompt.js +8 -8
- package/dist/routes/computer-use/participant-runs.d.ts +11 -11
- package/dist/routes/computer-use/participant-runs.js +18 -18
- package/dist/routes/computer-use/plan.d.ts +21 -29
- package/dist/routes/computer-use/plan.js +70 -78
- package/dist/routes/computer-use/preflight.d.ts +3 -5
- package/dist/routes/computer-use/preflight.js +40 -36
- package/dist/routes/computer-use/rerun-selection.js +11 -7
- package/dist/routes/computer-use/result.d.ts +1 -1
- package/dist/routes/computer-use/result.js +48 -38
- package/dist/routes/computer-use/route.d.ts +5 -5
- package/dist/routes/computer-use/route.js +36 -33
- package/dist/routes/computer-use/self-report.d.ts +8 -8
- package/dist/routes/computer-use/self-report.js +33 -33
- package/dist/routes/computer-use/setup.d.ts +9 -15
- package/dist/routes/computer-use/setup.js +106 -83
- package/dist/routes/computer-use/single-bundle.d.ts +13 -16
- package/dist/routes/computer-use/single-bundle.js +46 -53
- package/dist/routes/computer-use/subject-projection.d.ts +6 -8
- package/dist/routes/computer-use/subject-projection.js +24 -23
- package/dist/routes/computer-use/types.d.ts +195 -238
- package/dist/routes/computer-use/types.js +21 -13
- package/dist/routes/preview.d.ts +8 -2
- package/dist/routes/preview.js +8 -5
- package/dist/routes/scripted/bundle.d.ts +5 -8
- package/dist/routes/scripted/bundle.js +23 -27
- package/dist/routes/scripted/plan.d.ts +6 -8
- package/dist/routes/scripted/plan.js +15 -13
- package/dist/routes/scripted/result.d.ts +4 -1
- package/dist/routes/scripted/result.js +15 -11
- package/dist/routes/scripted/route.js +15 -17
- package/dist/routes/scripted/scenario.d.ts +1 -1
- package/dist/routes/scripted/scenario.js +3 -3
- package/dist/routes/scripted/setup.d.ts +3 -2
- package/dist/routes/scripted/setup.js +27 -26
- package/dist/routes/scripted/subject.d.ts +8 -3
- package/dist/routes/scripted/subject.js +10 -6
- package/dist/routes/scripted/surface-records.js +4 -4
- package/dist/routes/scripted/surface-sessions.d.ts +2 -2
- package/dist/routes/scripted/surface-sessions.js +4 -4
- package/dist/routes/scripted/types.d.ts +21 -40
- package/dist/routes/scripted/types.js +2 -3
- package/dist/routes/shared-world/{seat-records.js → bundle-records.js} +17 -16
- package/dist/routes/shared-world/bundle.d.ts +7 -7
- package/dist/routes/shared-world/bundle.js +60 -62
- package/dist/routes/shared-world/checkpoints.d.ts +5 -5
- package/dist/routes/shared-world/checkpoints.js +5 -5
- package/dist/routes/shared-world/comms.d.ts +8 -7
- package/dist/routes/shared-world/comms.js +38 -51
- package/dist/routes/shared-world/external-public.d.ts +6 -5
- package/dist/routes/shared-world/external-public.js +28 -25
- package/dist/routes/shared-world/handoff.d.ts +10 -9
- package/dist/routes/shared-world/handoff.js +89 -81
- package/dist/routes/shared-world/lobby-code.d.ts +7 -7
- package/dist/routes/shared-world/lobby-code.js +22 -22
- package/dist/routes/shared-world/{seats.d.ts → participant-specs.d.ts} +15 -12
- package/dist/routes/shared-world/{seats.js → participant-specs.js} +65 -63
- package/dist/routes/shared-world/plan.d.ts +2 -6
- package/dist/routes/shared-world/plan.js +8 -9
- package/dist/routes/shared-world/provenance.d.ts +4 -4
- package/dist/routes/shared-world/provenance.js +4 -4
- package/dist/routes/shared-world/provisioned.d.ts +3 -2
- package/dist/routes/shared-world/provisioned.js +55 -48
- package/dist/routes/shared-world/result.js +47 -36
- package/dist/routes/shared-world/route.js +54 -57
- package/dist/routes/shared-world/setup.d.ts +6 -5
- package/dist/routes/shared-world/setup.js +16 -18
- package/dist/routes/shared-world/types.d.ts +70 -141
- package/dist/routes/shared-world/types.js +5 -5
- package/dist/routes/terminal/adapter.d.ts +10 -10
- package/dist/routes/terminal/adapter.js +32 -129
- package/dist/routes/terminal/bundle.d.ts +10 -15
- package/dist/routes/terminal/bundle.js +43 -52
- package/dist/routes/terminal/credentials.d.ts +4 -4
- package/dist/routes/terminal/credentials.js +20 -20
- package/dist/routes/terminal/dry-run.d.ts +1 -1
- package/dist/routes/terminal/dry-run.js +13 -15
- package/dist/routes/terminal/encoding.d.ts +1 -1
- package/dist/routes/terminal/encoding.js +4 -4
- package/dist/routes/terminal/ledger.d.ts +8 -8
- package/dist/routes/terminal/ledger.js +22 -22
- package/dist/routes/terminal/live-finish.d.ts +1 -2
- package/dist/routes/terminal/live-finish.js +24 -23
- package/dist/routes/terminal/live-sandbox.d.ts +3 -2
- package/dist/routes/terminal/live-sandbox.js +31 -32
- package/dist/routes/terminal/participant-text.d.ts +23 -0
- package/dist/routes/terminal/participant-text.js +165 -0
- package/dist/routes/terminal/persona.js +1 -1
- package/dist/routes/terminal/plan.d.ts +2 -6
- package/dist/routes/terminal/plan.js +23 -19
- package/dist/routes/terminal/recorder.d.ts +6 -1
- package/dist/routes/terminal/recorder.js +12 -4
- package/dist/routes/terminal/result.js +12 -12
- package/dist/routes/terminal/route.js +30 -31
- package/dist/routes/terminal/runtime.d.ts +3 -2
- package/dist/routes/terminal/runtime.js +3 -3
- package/dist/routes/terminal/sandbox.d.ts +4 -4
- package/dist/routes/terminal/sandbox.js +10 -10
- package/dist/routes/terminal/session.js +22 -19
- package/dist/routes/terminal/token-usage.d.ts +1 -1
- package/dist/routes/terminal/token-usage.js +10 -11
- package/dist/routes/terminal/trace.d.ts +7 -10
- package/dist/routes/terminal/trace.js +11 -66
- package/dist/routes/terminal/types.d.ts +104 -114
- package/dist/routes/terminal/types.js +12 -7
- package/dist/run/active-runs.d.ts +13 -0
- package/dist/run/active-runs.js +12 -0
- package/dist/run/bundle-shape.d.ts +2 -1
- package/dist/run/bundle-shape.js +7 -7
- package/dist/run/bundle.d.ts +99 -92
- package/dist/run/bundle.js +10 -8
- package/dist/run/cost-summary.js +6 -5
- package/dist/run/costs.d.ts +6 -6
- package/dist/run/costs.js +17 -11
- package/dist/run/detail.d.ts +9 -9
- package/dist/run/detail.js +10 -10
- package/dist/run/dry-run-inputs.js +1 -1
- package/dist/run/dry-run.js +23 -30
- package/dist/run/{study-files.d.ts → evidence-files.d.ts} +4 -4
- package/dist/run/{study-files.js → evidence-files.js} +5 -5
- package/dist/run/feedback-shape.js +1 -1
- package/dist/run/judge.d.ts +34 -16
- package/dist/run/judge.js +41 -14
- package/dist/run/locate.d.ts +2 -0
- package/dist/run/locate.js +3 -1
- package/dist/run/outcomes.d.ts +3 -3
- package/dist/run/outcomes.js +3 -3
- package/dist/run/participant-records.d.ts +2 -2
- package/dist/run/participant-records.js +2 -2
- package/dist/run/paths.d.ts +1 -1
- package/dist/run/paths.js +2 -2
- package/dist/run/pricing.d.ts +30 -15
- package/dist/run/pricing.js +49 -30
- package/dist/run/projection.d.ts +27 -27
- package/dist/run/projection.js +23 -23
- package/dist/run/reclaim.d.ts +6 -0
- package/dist/run/reclaim.js +15 -5
- package/dist/run/results.d.ts +5 -5
- package/dist/run/run-index.d.ts +9 -7
- package/dist/run/run-index.js +24 -25
- package/dist/run/run-not-found.d.ts +5 -0
- package/dist/run/run-not-found.js +15 -0
- package/dist/run/run.d.ts +7 -4
- package/dist/run/run.js +10 -6
- package/dist/run/sandbox-receipts.d.ts +5 -5
- package/dist/run/sandbox-receipts.js +6 -6
- package/dist/run/shared-world-evidence.d.ts +67 -66
- package/dist/run/shared-world-evidence.js +4 -4
- package/dist/run/shared-world-shape.d.ts +3 -3
- package/dist/run/shared-world-shape.js +16 -16
- package/dist/run/stats.d.ts +17 -11
- package/dist/run/stats.js +124 -66
- package/dist/run/status.d.ts +21 -32
- package/dist/run/status.js +34 -33
- package/dist/run/stored-runs.js +2 -1
- package/dist/run/stream-shape.js +1 -1
- package/dist/run/streams.d.ts +29 -22
- package/dist/run/study-provenance.d.ts +47 -0
- package/dist/run/study-provenance.js +63 -0
- package/dist/run/study-result.d.ts +16 -0
- package/dist/run/study-result.js +9 -0
- package/dist/run/terminal-contract.d.ts +5 -5
- package/dist/run/terminal-contract.js +4 -4
- package/dist/run/text.d.ts +2 -0
- package/dist/run/text.js +4 -0
- package/dist/run/transient-comms-secrets.d.ts +5 -0
- package/dist/run/transient-comms-secrets.js +12 -0
- package/dist/run-lab.d.ts +40 -38
- package/dist/run-lab.js +42 -40
- package/dist/study/convert.d.ts +33 -0
- package/dist/study/convert.js +432 -0
- package/dist/study/field-paths.d.ts +12 -0
- package/dist/study/field-paths.js +126 -0
- package/dist/study/files.d.ts +26 -0
- package/dist/study/files.js +43 -0
- package/dist/study/migrate.d.ts +48 -0
- package/dist/study/migrate.js +408 -0
- package/dist/study/owned-files.d.ts +38 -0
- package/dist/study/owned-files.js +182 -0
- package/dist/subject/clone.d.ts +1 -1
- package/dist/subject/desktop-cli.d.ts +2 -2
- package/dist/subject/desktop-cli.js +3 -3
- package/dist/subject/local-tree-archive.d.ts +1 -1
- package/dist/subject/local-tree-archive.js +5 -5
- package/dist/subject/local-tree.d.ts +3 -3
- package/dist/subject/local-tree.js +1 -1
- package/dist/subject/node-bootstrap.d.ts +1 -1
- package/dist/subject/node-bootstrap.js +4 -4
- package/dist/subject/runtime.js +10 -11
- package/dist/subject/serve.d.ts +1 -1
- package/dist/subject/serve.js +4 -4
- package/dist/subject/state.d.ts +1 -1
- package/dist/subject/steps.d.ts +8 -8
- package/dist/subject/steps.js +7 -7
- package/dist/substrates/command-failure.d.ts +3 -3
- package/dist/substrates/command-failure.js +3 -3
- package/dist/substrates/detached.d.ts +6 -6
- package/dist/substrates/detached.js +7 -7
- package/dist/substrates/e2b/cdp-probe.d.ts +11 -11
- package/dist/substrates/e2b/cdp-probe.js +15 -18
- package/dist/substrates/e2b/desktop-browser.d.ts +5 -5
- package/dist/substrates/e2b/desktop-browser.js +12 -12
- package/dist/substrates/e2b/desktop-cdp.d.ts +15 -15
- package/dist/substrates/e2b/desktop-cdp.js +11 -11
- package/dist/substrates/e2b/desktop-executor.d.ts +1 -1
- package/dist/substrates/e2b/desktop-executor.js +1 -1
- package/dist/substrates/e2b/desktop-geometry.d.ts +6 -6
- package/dist/substrates/e2b/desktop-geometry.js +9 -9
- package/dist/substrates/e2b/desktop-media.d.ts +1 -1
- package/dist/substrates/e2b/desktop-media.js +3 -3
- package/dist/substrates/e2b/desktop-screenshot-cleanup.d.ts +3 -3
- package/dist/substrates/e2b/desktop-screenshot-cleanup.js +3 -3
- package/dist/substrates/e2b/sandbox.d.ts +1 -1
- package/dist/substrates/e2b/sandbox.js +2 -2
- package/dist/substrates/e2b/sdk.d.ts +11 -11
- package/dist/substrates/e2b/sdk.js +8 -8
- package/dist/substrates/local/runtime-host.d.ts +2 -0
- package/dist/substrates/local/runtime-host.js +6 -0
- package/dist/tui/actions.d.ts +8 -6
- package/dist/tui/actions.js +31 -22
- package/dist/tui/contract.d.ts +16 -16
- package/dist/tui/contract.js +10 -10
- package/dist/tui/launch.d.ts +4 -4
- package/dist/tui/launch.js +18 -19
- package/dist/tui/project.js +5 -5
- package/dist/tui-app.js +83 -83
- package/dist/verify/actor.d.ts +14 -14
- package/dist/verify/actor.js +45 -45
- package/dist/verify/artifacts.d.ts +3 -3
- package/dist/verify/artifacts.js +6 -6
- package/dist/verify/context-growth.d.ts +9 -0
- package/dist/verify/context-growth.js +62 -0
- package/dist/verify/costs.d.ts +7 -7
- package/dist/verify/costs.js +9 -9
- package/dist/verify/liveness.d.ts +1 -1
- package/dist/verify/liveness.js +4 -1
- package/dist/verify/rerun.d.ts +1 -1
- package/dist/verify/rerun.js +1 -1
- package/dist/verify/shared-world-concurrent.d.ts +2 -2
- package/dist/verify/shared-world-concurrent.js +79 -76
- package/dist/verify/shared-world.d.ts +2 -2
- package/dist/verify/shared-world.js +18 -18
- package/dist/verify/subject.d.ts +4 -4
- package/dist/verify/subject.js +18 -18
- package/dist/verify/verify.js +98 -49
- package/docs/README.md +26 -33
- package/docs/architecture/actor-contract.md +105 -98
- package/docs/architecture/desktop-sessions.md +14 -14
- package/docs/architecture/external-public-shared-world.md +66 -59
- package/docs/architecture/github-feedback-loop.md +1 -4
- package/docs/architecture/guest-desktop.md +2 -2
- package/docs/architecture/observer.md +37 -37
- package/docs/architecture/restricted-codex-analysis.md +70 -30
- package/docs/architecture/serve.md +34 -37
- package/docs/architecture/state-driven-executor.md +55 -59
- package/docs/architecture/task-protocol-support.md +7 -9
- package/docs/architecture/{terminal-product-lane.md → terminal-product-route.md} +83 -115
- package/docs/contracts/README.md +24 -0
- package/docs/contracts/adapter-admission.md +3 -3
- package/docs/contracts/adapter-fixtures.md +2 -5
- package/docs/contracts/core.md +1 -3
- package/docs/contracts/errors.md +62 -0
- package/docs/contracts/feedback.md +8 -9
- package/docs/contracts/policy.md +7 -12
- package/docs/contracts/run-bundle.md +54 -51
- package/docs/contracts/schemas.md +505 -390
- package/docs/contracts/study-analysis.md +26 -2
- package/docs/contracts/study-costs.md +11 -7
- package/docs/decisions/0001-run-bundle-is-the-source-of-truth.md +1 -1
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +2 -2
- package/docs/decisions/0003-contained-paths-and-cleanup-authority.md +3 -3
- package/docs/decisions/0004-participant-desktops-stay-in-the-route.md +2 -1
- package/docs/principles/actor-fidelity.md +4 -4
- package/docs/principles/engineering.md +3 -3
- package/docs/principles/invariants-and-defaults.md +38 -38
- package/docs/principles/self-driving-harness.md +1 -5
- package/docs/principles/three-roles.md +1 -1
- package/docs/product/automatic-analysis.md +8 -7
- package/examples/participant/README.md +5 -5
- package/examples/participant/run.mjs +11 -10
- package/examples/scorer/README.md +3 -3
- package/examples/scorer/run.mjs +11 -12
- package/package.json +16 -17
- package/skills/humanish/SKILL.md +63 -60
- package/skills/humanish/agents/openai.yaml +1 -1
- package/AGENTS.md +0 -99
- package/CONTRIBUTING.md +0 -148
- package/dist/actors/codex/qualified-versions.d.ts +0 -45
- package/dist/actors/codex/qualified-versions.js +0 -87
- package/dist/cli/commands/lab.d.ts +0 -3
- package/dist/cli/commands/lab.js +0 -193
- package/dist/deprecated.d.ts +0 -90
- package/dist/deprecated.js +0 -106
- package/dist/lab/bag-overrides.d.ts +0 -18
- package/dist/lab/bag-overrides.js +0 -121
- package/dist/observer/static.d.ts +0 -39
- package/dist/observer/static.js +0 -369
- package/dist/routes/computer-use/legacy-lane-spec.d.ts +0 -62
- package/dist/routes/computer-use/legacy-lane-spec.js +0 -39
- package/docs/architecture/comms-inbox.md +0 -122
- package/docs/architecture/desktop-recording.md +0 -80
- package/docs/architecture/local-browser-runtime.md +0 -178
- package/docs/architecture/observer-review.md +0 -144
- package/docs/architecture/participant-media.md +0 -90
- package/docs/architecture/project-layout.md +0 -62
- package/docs/architecture/real-email-receiving.md +0 -136
- package/docs/assets/humanish-drawdb-hero.png +0 -0
- package/docs/assets/humanish-observer-hero.png +0 -0
- package/docs/goals/current.md +0 -207
- package/docs/product/lobby-trivia-3player-external-public.md +0 -77
- package/docs/product/open-source-install-experience.md +0 -240
- package/docs/ramp/README.md +0 -294
- package/docs/release/open-source-readiness.md +0 -222
- package/docs/release/public-readiness-standard.md +0 -210
- package/docs/roadmap/world-class-open-source-v0.md +0 -311
- /package/dist/routes/computer-use/{fanout-lanes.d.ts → fanout-records.d.ts} +0 -0
- /package/dist/routes/shared-world/{seat-records.d.ts → bundle-records.d.ts} +0 -0
|
@@ -8,7 +8,7 @@ import { redactText, toErrorMessage } from "../../evidence/redaction.js";
|
|
|
8
8
|
import { assertSafeOutputPathSegment, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
9
9
|
import { participantOutcomeOk } from "./participant-facts.js";
|
|
10
10
|
import { closeParticipantModel, judgeParticipantSession, participantSessionOptions, recordParticipantTrace, startParticipantModel, } from "./participant-model.js";
|
|
11
|
-
/** Build a
|
|
11
|
+
/** Build a participant's writeScreenshot closure: writes under screenshots/<screenshotDir>/ and records
|
|
12
12
|
* the relative path the trace references (screenshots/<name> at N=1; screenshots/<laneId>/<name>
|
|
13
13
|
* at N>1). */
|
|
14
14
|
export function makeParticipantWriteScreenshot(artifactRoot, spec, screenshots) {
|
|
@@ -30,7 +30,7 @@ export function makeParticipantWriteScreenshot(artifactRoot, spec, screenshots)
|
|
|
30
30
|
return rel;
|
|
31
31
|
};
|
|
32
32
|
}
|
|
33
|
-
/** A blocked
|
|
33
|
+
/** A blocked participant outcome (pipeline gate / fail-fast skipped it before it ran). */
|
|
34
34
|
function skippedOutcome(spec, reason) {
|
|
35
35
|
return {
|
|
36
36
|
spec,
|
|
@@ -57,6 +57,7 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
57
57
|
let session;
|
|
58
58
|
let sessionError;
|
|
59
59
|
let providerCleanupError;
|
|
60
|
+
let providerPolicyError;
|
|
60
61
|
let provisioned = false;
|
|
61
62
|
let signaled = false;
|
|
62
63
|
const signal = (ok) => {
|
|
@@ -71,7 +72,7 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
71
72
|
await desktop.prepare();
|
|
72
73
|
const ready = await desktop.openSession();
|
|
73
74
|
model = await startParticipantModel(spec, deps, ready.executor);
|
|
74
|
-
// World is ready: release the pipeline gate so the remaining
|
|
75
|
+
// World is ready: release the pipeline gate so the remaining participants may start.
|
|
75
76
|
provisioned = true;
|
|
76
77
|
signal(true);
|
|
77
78
|
session = await deps.runSession(participantSessionOptions(spec, deps, ready, model.provider, writeScreenshot));
|
|
@@ -82,25 +83,35 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
82
83
|
finally {
|
|
83
84
|
// A provider whose cleanup is unconfirmed fails the run as an execution, apart from how the
|
|
84
85
|
// session ended: a session that passed stays a passed participant.
|
|
85
|
-
|
|
86
|
+
const closed = await closeParticipantModel(model, warnings);
|
|
87
|
+
if (closed.unconfirmed)
|
|
86
88
|
providerCleanupError = "Model provider cleanup is unconfirmed.";
|
|
87
|
-
|
|
89
|
+
// The session's evidence would read clean, so a late refusal fails the run as an execution.
|
|
90
|
+
if (closed.refusal !== undefined)
|
|
91
|
+
providerPolicyError = redactText(deps.scrubKnownValues(closed.refusal === "codex_tool_call"
|
|
92
|
+
? `Codex reported a disallowed item after the participant's last request (${closed.refusal}).`
|
|
93
|
+
: `Codex output after the participant's last request could not be checked against the item policy (${closed.refusal}).`));
|
|
88
94
|
try {
|
|
89
95
|
if (!provisioned)
|
|
90
96
|
signal(false);
|
|
91
97
|
}
|
|
92
98
|
finally {
|
|
93
99
|
await desktop.finalize({
|
|
94
|
-
failed: sessionError !== undefined ||
|
|
100
|
+
failed: sessionError !== undefined ||
|
|
101
|
+
providerCleanupError !== undefined ||
|
|
102
|
+
providerPolicyError !== undefined ||
|
|
103
|
+
session === undefined,
|
|
95
104
|
});
|
|
96
105
|
}
|
|
97
106
|
}
|
|
98
107
|
if (session)
|
|
99
108
|
await recordParticipantTrace(spec, deps, session, warnings);
|
|
100
109
|
const { noEngagement, selfReportedBlocker, reportedFriction } = judgeParticipantSession(session, deps, warnings);
|
|
101
|
-
// A provider-cleanup failure still trips fail-fast and counts in the
|
|
110
|
+
// A provider-cleanup or provider-policy failure still trips fail-fast and counts in the
|
|
111
|
+
// participant summary's harness errors.
|
|
102
112
|
const harnessError = sessionError !== undefined ||
|
|
103
113
|
providerCleanupError !== undefined ||
|
|
114
|
+
providerPolicyError !== undefined ||
|
|
104
115
|
session?.completionReason === "harness_error";
|
|
105
116
|
const { released, ...desktopEvidence } = desktop.snapshot();
|
|
106
117
|
return {
|
|
@@ -108,6 +119,7 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
108
119
|
...(session ? { session } : {}),
|
|
109
120
|
...(sessionError === undefined ? {} : { sessionError }),
|
|
110
121
|
...(providerCleanupError === undefined ? {} : { providerCleanupError }),
|
|
122
|
+
...(providerPolicyError === undefined ? {} : { providerPolicyError }),
|
|
111
123
|
...desktopEvidence,
|
|
112
124
|
killed: released,
|
|
113
125
|
screenshots,
|
|
@@ -119,13 +131,13 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
119
131
|
};
|
|
120
132
|
}
|
|
121
133
|
/**
|
|
122
|
-
* Run N>1 E2B
|
|
123
|
-
* start), and session fail-fast on
|
|
124
|
-
* pinned reason + a fail-fast event; mission verdicts never trip it).
|
|
125
|
-
*
|
|
134
|
+
* Run N>1 E2B participants with bounded concurrency, a pipeline gate (the first provisions
|
|
135
|
+
* before the rest start), and session fail-fast on harness errors only (queued participants
|
|
136
|
+
* become `blocked` with a pinned reason + a fail-fast event; mission verdicts never trip it).
|
|
137
|
+
* Each participant tears down its own sandbox by id; nothing here ever enumerates.
|
|
126
138
|
*
|
|
127
|
-
* Exported for the
|
|
128
|
-
*
|
|
139
|
+
* Exported for the total-runner tests: the injectable runner lets a test make one participant
|
|
140
|
+
* throw (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
129
141
|
* the default.
|
|
130
142
|
*/
|
|
131
143
|
export async function runCuaParticipants(runs, deps, concurrency, runParticipant = runCuaParticipant) {
|
|
@@ -136,8 +148,9 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
136
148
|
resolveGate = resolve;
|
|
137
149
|
rejectGate = () => reject(new Error("gate"));
|
|
138
150
|
});
|
|
139
|
-
// The gate is rejected on
|
|
140
|
-
// later
|
|
151
|
+
// The gate is rejected on the first participant's provisioning failure; swallow the unhandled
|
|
152
|
+
// rejection if no later participant ever awaits it (concurrency could let the first finish
|
|
153
|
+
// alone).
|
|
141
154
|
gate.catch(() => undefined);
|
|
142
155
|
const outcomes = await mapWithConcurrency(runs, concurrency, async (spec, index) => {
|
|
143
156
|
if (index > 0) {
|
|
@@ -151,10 +164,10 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
151
164
|
if (failFast.tripped) {
|
|
152
165
|
return skippedOutcome(spec, `skipped: ${failFast.reason}`);
|
|
153
166
|
}
|
|
154
|
-
// The
|
|
155
|
-
// guard, one
|
|
167
|
+
// The participant runner is total: every exit path returns a recorded outcome. Without this
|
|
168
|
+
// guard, one participant's late throw (e.g. its trace write hitting ENOSPC after its own sandbox was
|
|
156
169
|
// already torn down) rejected the whole map while sibling workers kept launching sandboxes
|
|
157
|
-
// nobody would ever record
|
|
170
|
+
// nobody would ever record, so the run spent money and then reported nothing.
|
|
158
171
|
let outcome;
|
|
159
172
|
try {
|
|
160
173
|
outcome = await runParticipant(spec, {
|
|
@@ -174,7 +187,7 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
174
187
|
});
|
|
175
188
|
}
|
|
176
189
|
catch (error) {
|
|
177
|
-
//
|
|
190
|
+
// The first participant may have thrown before signaling the provisioning gate; release the followers as
|
|
178
191
|
// blocked rather than leaving them awaiting a gate that will never settle.
|
|
179
192
|
if (index === 0)
|
|
180
193
|
rejectGate?.();
|
|
@@ -202,7 +215,7 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
202
215
|
});
|
|
203
216
|
return { outcomes, ...(failFast.tripped ? { failFastReason: failFast.reason } : {}) };
|
|
204
217
|
}
|
|
205
|
-
/** Project one
|
|
218
|
+
/** Project one participant outcome (or a dry-run contract spec) into the public CuaParticipantResult. */
|
|
206
219
|
export function toParticipantResult(spec, outcome, subject, dryRun) {
|
|
207
220
|
const base = {
|
|
208
221
|
id: spec.planned.id,
|
|
@@ -234,7 +247,7 @@ export function toParticipantResult(spec, outcome, subject, dryRun) {
|
|
|
234
247
|
ok: false,
|
|
235
248
|
diagnostics: cuaParticipantDiagnostics({ dryRun, skipped: true }),
|
|
236
249
|
skippedReason: outcome.skippedReason,
|
|
237
|
-
error: { code: "
|
|
250
|
+
error: { code: "HUMANISH_COMPUTER_USE_FAILED", message: outcome.skippedReason },
|
|
238
251
|
};
|
|
239
252
|
}
|
|
240
253
|
const session = outcome.session;
|
|
@@ -286,7 +299,7 @@ export function toParticipantResult(spec, outcome, subject, dryRun) {
|
|
|
286
299
|
? {}
|
|
287
300
|
: {
|
|
288
301
|
error: {
|
|
289
|
-
code: outcome.failureCode ?? "
|
|
302
|
+
code: outcome.failureCode ?? "HUMANISH_COMPUTER_USE_FAILED",
|
|
290
303
|
message: outcome.sessionError ??
|
|
291
304
|
(outcome.noEngagement
|
|
292
305
|
? "Actor took no actions and produced no message (likely a blank/still-loading screen); not a credible goal_satisfied."
|
|
@@ -302,11 +315,11 @@ export function toParticipantResult(spec, outcome, subject, dryRun) {
|
|
|
302
315
|
};
|
|
303
316
|
}
|
|
304
317
|
/**
|
|
305
|
-
* Run every
|
|
306
|
-
* hosted
|
|
318
|
+
* Run every participant of a live run. The in-process route drives its single participant in this
|
|
319
|
+
* process; one hosted participant runs alone; a fan-out runs at the plan's concurrency and may stop early.
|
|
307
320
|
*/
|
|
308
|
-
export async function runAllCuaParticipants(runs, deps, participantPlan,
|
|
309
|
-
if (
|
|
321
|
+
export async function runAllCuaParticipants(runs, deps, participantPlan, inProcess) {
|
|
322
|
+
if (inProcess) {
|
|
310
323
|
// The caller's executor stands in for a desktop, and the shared runner drives the participant.
|
|
311
324
|
const outcome = await runCuaParticipant(runs[0], {
|
|
312
325
|
...deps,
|
|
@@ -1,6 +1,12 @@
|
|
|
1
|
-
import { type ParticipantFacts } from "../../run/judge.js";
|
|
1
|
+
import { type ExecutionFailure, type ParticipantFacts } from "../../run/judge.js";
|
|
2
2
|
import type { ParticipantRunOutcome } from "./types.js";
|
|
3
3
|
/** A participant outcome's facts for the judge. */
|
|
4
4
|
export declare function participantFactsOf(outcome: ParticipantRunOutcome): ParticipantFacts;
|
|
5
5
|
/** Whether a participant passed. A dry-run participant passes as a contract. */
|
|
6
6
|
export declare function participantOutcomeOk(outcome: ParticipantRunOutcome | undefined, dryRun: boolean): boolean;
|
|
7
|
+
/**
|
|
8
|
+
* One sandbox-cleanup failure per participant desktop not confirmed released: an E2B sandbox that
|
|
9
|
+
* was not killed, or a desktop whose release says why (a local VM names its container). A
|
|
10
|
+
* participant that never acquired a desktop (skipped, in-process) adds none.
|
|
11
|
+
*/
|
|
12
|
+
export declare function unreleasedSandboxFailures(outcomes: readonly ParticipantRunOutcome[] | undefined, runId: string): ExecutionFailure[];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// A participant run outcome's facts for the judge (src/run/judge.ts), and the pass check built on
|
|
2
2
|
// them. The bundle builders, the result, the participant runner and shared-world all read these,
|
|
3
3
|
// so they sit in a file that imports only the judge and the types.
|
|
4
|
-
import { participantPassed } from "../../run/judge.js";
|
|
4
|
+
import { participantPassed, sandboxCleanupFailure, } from "../../run/judge.js";
|
|
5
5
|
/** A participant outcome's facts for the judge. */
|
|
6
6
|
export function participantFactsOf(outcome) {
|
|
7
7
|
return {
|
|
@@ -20,3 +20,14 @@ export function participantOutcomeOk(outcome, dryRun) {
|
|
|
20
20
|
return true;
|
|
21
21
|
return outcome !== undefined && participantPassed(participantFactsOf(outcome));
|
|
22
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* One sandbox-cleanup failure per participant desktop not confirmed released: an E2B sandbox that
|
|
25
|
+
* was not killed, or a desktop whose release says why (a local VM names its container). A
|
|
26
|
+
* participant that never acquired a desktop (skipped, in-process) adds none.
|
|
27
|
+
*/
|
|
28
|
+
export function unreleasedSandboxFailures(outcomes, runId) {
|
|
29
|
+
return (outcomes ?? [])
|
|
30
|
+
.filter((outcome) => outcome.sandboxRelease !== undefined ||
|
|
31
|
+
(outcome.sandboxId !== undefined && !outcome.killed))
|
|
32
|
+
.map((outcome) => sandboxCleanupFailure(outcome.spec.planned.id, outcome.sandboxRelease?.warning, runId, outcome.sandboxRelease?.recovery));
|
|
33
|
+
}
|
|
@@ -1,28 +1,42 @@
|
|
|
1
1
|
import type { CuaExecutor, CuaLoopResult, CuaProvider } from "../../actors/computer-use/loop.js";
|
|
2
|
+
import type { ActorConversation } from "../../actors/contract.js";
|
|
2
3
|
import type { CuaActorSessionOptions } from "../../actors/computer-use/actor.js";
|
|
3
4
|
import { startClaudeSession } from "../../actors/local-agent/claude-session.js";
|
|
4
|
-
import { createRestrictedCodexParticipant } from "../../actors/codex/restricted-participant.js";
|
|
5
|
-
import type {
|
|
5
|
+
import { createRestrictedCodexParticipant, type ParticipantProviderCloseResult } from "../../actors/codex/restricted-participant.js";
|
|
6
|
+
import type { ParticipantModelDeps, CuaRunBudget, DesktopParticipantRun } from "./types.js";
|
|
6
7
|
import type { ReadyParticipantDesktop } from "./participant-desktop.js";
|
|
7
|
-
/**
|
|
8
|
+
/**
|
|
9
|
+
* Registers a caller's provider as backed by a native Codex session: the participant runner reads the session's
|
|
10
|
+
* run warnings and a refusal no request reported from `report` when it closes the provider.
|
|
11
|
+
*/
|
|
12
|
+
export declare function withCloseReport<T extends CuaProvider>(provider: T, report: () => Promise<ParticipantProviderCloseResult>): T;
|
|
13
|
+
/** The model a participant brings besides the default API client, and the handles its cleanup needs. */
|
|
8
14
|
export interface ParticipantModel {
|
|
9
15
|
provider?: CuaProvider;
|
|
10
16
|
codexParticipant?: ReturnType<typeof createRestrictedCodexParticipant>;
|
|
11
17
|
claudeSession?: Awaited<ReturnType<typeof startClaudeSession>>;
|
|
12
18
|
}
|
|
13
|
-
/** Starts the
|
|
14
|
-
export declare function startParticipantModel(spec: DesktopParticipantRun, deps:
|
|
15
|
-
/** The options the
|
|
16
|
-
export declare function participantSessionOptions(spec: DesktopParticipantRun, deps:
|
|
19
|
+
/** Starts the participant's model: a caller's provider, the operator's Codex, a Claude session, or none. */
|
|
20
|
+
export declare function startParticipantModel(spec: DesktopParticipantRun, deps: ParticipantModelDeps, executor: CuaExecutor): Promise<ParticipantModel>;
|
|
21
|
+
/** The options the participant's session runs with: prompt, model settings, spend caps and callbacks. */
|
|
22
|
+
export declare function participantSessionOptions(spec: DesktopParticipantRun, deps: ParticipantModelDeps, ready: ReadyParticipantDesktop, provider: CuaProvider | undefined, writeScreenshot: NonNullable<CuaActorSessionOptions["writeScreenshot"]>): CuaActorSessionOptions;
|
|
17
23
|
/**
|
|
18
|
-
* Closes the
|
|
24
|
+
* Closes the participant's model, in the order it was opened. Returns true when a provider's cleanup is
|
|
19
25
|
* unconfirmed; each failure also adds a warning.
|
|
20
26
|
*/
|
|
21
|
-
export declare function closeParticipantModel(model: ParticipantModel, warnings: string[]): Promise<
|
|
27
|
+
export declare function closeParticipantModel(model: ParticipantModel, warnings: string[]): Promise<{
|
|
28
|
+
unconfirmed: boolean;
|
|
29
|
+
refusal: string | undefined;
|
|
30
|
+
}>;
|
|
22
31
|
/** Prices the finished session's tokens onto its trace, writes the trace, and warns on raw screenshots. */
|
|
23
|
-
export declare function recordParticipantTrace(spec: DesktopParticipantRun, deps:
|
|
32
|
+
export declare function recordParticipantTrace(spec: DesktopParticipantRun, deps: ParticipantModelDeps, session: CuaLoopResult, warnings: string[]): Promise<void>;
|
|
33
|
+
/**
|
|
34
|
+
* The run warning for a participant whose provider kept no conversation server-side, so humanish
|
|
35
|
+
* carried it on every request. It says why, and how many early turns became a text summary.
|
|
36
|
+
*/
|
|
37
|
+
export declare function explicitContextWarning(conversation: ActorConversation): string;
|
|
24
38
|
/** The checks that keep a goal_satisfied session from counting as a pass, with their warnings. */
|
|
25
|
-
export declare function judgeParticipantSession(session: CuaLoopResult | undefined, deps:
|
|
39
|
+
export declare function judgeParticipantSession(session: CuaLoopResult | undefined, deps: ParticipantModelDeps, warnings: string[]): {
|
|
26
40
|
noEngagement: boolean;
|
|
27
41
|
selfReportedBlocker: boolean;
|
|
28
42
|
reportedFriction: boolean;
|
|
@@ -1,30 +1,36 @@
|
|
|
1
|
-
// The model side of one computer-use
|
|
1
|
+
// The model side of one computer-use participant: the provider a local agent brings, the session options
|
|
2
2
|
// the loop runs with, the study's shared spend ledger, the provider cleanup, and the checks on the
|
|
3
|
-
// finished session. The desktop side stays with the
|
|
3
|
+
// finished session. The desktop side stays with the participant runner.
|
|
4
4
|
import { startClaudeSession } from "../../actors/local-agent/claude-session.js";
|
|
5
5
|
import { createLocalAgentProvider } from "../../actors/local-agent/cli.js";
|
|
6
6
|
import { pricedModel } from "../../lab/plan-base.js";
|
|
7
7
|
import { estimateActorCostForExecution, round6 } from "../../run/pricing.js";
|
|
8
8
|
import { redactText } from "../../evidence/redaction.js";
|
|
9
|
-
import { createRestrictedCodexParticipant } from "../../actors/codex/restricted-participant.js";
|
|
9
|
+
import { createRestrictedCodexParticipant, } from "../../actors/codex/restricted-participant.js";
|
|
10
10
|
import { writeContainedOutputFile } from "../../run/contained-output.js";
|
|
11
11
|
import { withInboxMission } from "./participant-prompt.js";
|
|
12
12
|
import { resolveSelfReportedBlocker, resolveSelfReportedFriction, sessionEnding, } from "./self-report.js";
|
|
13
13
|
import { hollowCompletion } from "../../run/judge.js";
|
|
14
|
-
|
|
15
|
-
|
|
14
|
+
// Caller providers backed by a native Codex session (the local study's), with their close report.
|
|
15
|
+
const closeReports = new WeakMap();
|
|
16
|
+
/**
|
|
17
|
+
* Registers a caller's provider as backed by a native Codex session: the participant runner reads the session's
|
|
18
|
+
* run warnings and a refusal no request reported from `report` when it closes the provider.
|
|
19
|
+
*/
|
|
20
|
+
export function withCloseReport(provider, report) {
|
|
21
|
+
closeReports.set(provider, report);
|
|
22
|
+
return provider;
|
|
23
|
+
}
|
|
24
|
+
/** Starts the participant's model: a caller's provider, the operator's Codex, a Claude session, or none. */
|
|
16
25
|
export async function startParticipantModel(spec, deps, executor) {
|
|
17
|
-
const {
|
|
18
|
-
if (deps.
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
lane: legacyHookSpecOf(spec),
|
|
24
|
-
laneCount: deps.participantCount,
|
|
25
|
-
executor,
|
|
26
|
-
}),
|
|
26
|
+
const { env, brain } = deps;
|
|
27
|
+
if (deps.createProvider) {
|
|
28
|
+
const participant = {
|
|
29
|
+
id: spec.planned.id,
|
|
30
|
+
index: spec.planned.index,
|
|
31
|
+
count: deps.participantCount,
|
|
27
32
|
};
|
|
33
|
+
return { provider: await deps.createProvider(participant, executor) };
|
|
28
34
|
}
|
|
29
35
|
const localAgent = brain.kind === "local-agent" ? brain.agent : undefined;
|
|
30
36
|
if (localAgent === "codex") {
|
|
@@ -43,10 +49,10 @@ export async function startParticipantModel(spec, deps, executor) {
|
|
|
43
49
|
return { provider: codexParticipant.provider, codexParticipant };
|
|
44
50
|
}
|
|
45
51
|
if (localAgent === "claude") {
|
|
46
|
-
// One session for the whole run, like the codex thread above
|
|
52
|
+
// One session for the whole run, like the codex thread above. The one-shot
|
|
47
53
|
// provider (createLocalAgentProvider) spawned `claude -p` per turn, and every turn
|
|
48
54
|
// started with no memory of the last. HUMANISH_LOCAL_AGENT_ONE_SHOT=1 keeps that path
|
|
49
|
-
// reachable as a
|
|
55
|
+
// reachable as a measurement switch: MemTrapBench (2026-08) reports memory frameworks
|
|
50
56
|
// degrading agent performance by 10-40% on some tasks, so "remembers" has to be measured
|
|
51
57
|
// against "does not" on the same lab, not assumed. The trace records which one ran.
|
|
52
58
|
const oneShot = env.HUMANISH_LOCAL_AGENT_ONE_SHOT !== undefined &&
|
|
@@ -73,16 +79,15 @@ export async function startParticipantModel(spec, deps, executor) {
|
|
|
73
79
|
}
|
|
74
80
|
return {};
|
|
75
81
|
}
|
|
76
|
-
/** The options the
|
|
82
|
+
/** The options the participant's session runs with: prompt, model settings, spend caps and callbacks. */
|
|
77
83
|
export function participantSessionOptions(spec, deps, ready, provider, writeScreenshot) {
|
|
78
|
-
const { config } = deps;
|
|
79
84
|
const { executor, inbox } = ready;
|
|
80
|
-
// The
|
|
85
|
+
// The fail-closed spend cap (execution.caps.maxUsd) is wired into the loop as maxUsd + an
|
|
81
86
|
// injected pure per-turn estimator keyed on the resolved model. Preflight already refused a
|
|
82
87
|
// cap on an unpriced model, so the estimate is measurable whenever a cap is in force. The
|
|
83
88
|
// model id here matches provider.version (openai-responses-cu resolves the default when unset).
|
|
84
89
|
const capModelId = pricedModel(deps.brain);
|
|
85
|
-
const maxUsd =
|
|
90
|
+
const maxUsd = deps.caps.maxUsd;
|
|
86
91
|
return {
|
|
87
92
|
instructions: inbox
|
|
88
93
|
? withInboxMission(spec, inbox.url, inbox.address, inbox.receiving).instructions
|
|
@@ -90,13 +95,14 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
|
|
|
90
95
|
persona: spec.persona,
|
|
91
96
|
timeoutMs: deps.timeoutMs,
|
|
92
97
|
// The brain is either a keyed API client or a CLI the operator is already signed in to.
|
|
93
|
-
// Everything below this line
|
|
98
|
+
// Everything below this line (loop, executor, trace, affordances) is identical either
|
|
94
99
|
// way, which is what makes a local-agent run comparable to an API one.
|
|
95
100
|
...(provider === undefined ? {} : { provider: provider }),
|
|
96
101
|
openai: {
|
|
97
102
|
apiKey: deps.openaiApiKey,
|
|
98
103
|
...(deps.brain.declaredModel ? { model: deps.brain.declaredModel } : {}),
|
|
99
|
-
// Per-
|
|
104
|
+
// Per-participant, not per-actor: two participants at different efforts is the control this
|
|
105
|
+
// exists for.
|
|
100
106
|
...(spec.planned.limits.reasoningEffort === undefined
|
|
101
107
|
? {}
|
|
102
108
|
: { reasoningEffort: spec.planned.limits.reasoningEffort }),
|
|
@@ -123,8 +129,9 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
|
|
|
123
129
|
: { stopWhen: spec.planned.limits.stopWhen }),
|
|
124
130
|
...(spec.planned.limits.dwell === undefined ? {} : { dwell: spec.planned.limits.dwell }),
|
|
125
131
|
...(spec.planned.tasks === undefined ? {} : { tasks: spec.planned.tasks }),
|
|
126
|
-
// The
|
|
127
|
-
// and stops when the
|
|
132
|
+
// The study budget: this participant notes its own running estimate on the shared
|
|
133
|
+
// ledger and stops when the run total crosses the cap, independent of the per-participant
|
|
134
|
+
// maxUsd above.
|
|
128
135
|
...(deps.runBudget === undefined
|
|
129
136
|
? {}
|
|
130
137
|
: {
|
|
@@ -142,18 +149,19 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
|
|
|
142
149
|
...(deps.onTrace === undefined
|
|
143
150
|
? {}
|
|
144
151
|
: {
|
|
145
|
-
// Forwards the
|
|
146
|
-
// without it never reaches the flush
|
|
152
|
+
// Forwards the running usage as well: the participant runner is where both are known, and usage
|
|
153
|
+
// without it never reaches the flush, which is how the live cost stayed unknown.
|
|
147
154
|
onTrace: (items, usage, metadata) => deps.onTrace?.(spec.planned.id, items, usage, metadata),
|
|
148
155
|
}),
|
|
149
156
|
};
|
|
150
157
|
}
|
|
151
158
|
/**
|
|
152
|
-
* Closes the
|
|
159
|
+
* Closes the participant's model, in the order it was opened. Returns true when a provider's cleanup is
|
|
153
160
|
* unconfirmed; each failure also adds a warning.
|
|
154
161
|
*/
|
|
155
162
|
export async function closeParticipantModel(model, warnings) {
|
|
156
163
|
let unconfirmed = false;
|
|
164
|
+
let refusal;
|
|
157
165
|
try {
|
|
158
166
|
if (model.codexParticipant === undefined)
|
|
159
167
|
await model.provider?.close?.();
|
|
@@ -164,6 +172,8 @@ export async function closeParticipantModel(model, warnings) {
|
|
|
164
172
|
}
|
|
165
173
|
try {
|
|
166
174
|
const cleanup = await model.codexParticipant?.close();
|
|
175
|
+
warnings.push(...(cleanup?.warnings ?? []));
|
|
176
|
+
refusal = cleanup?.refusal;
|
|
167
177
|
if (cleanup?.status === "unconfirmed") {
|
|
168
178
|
warnings.push("Model provider cleanup is unconfirmed.");
|
|
169
179
|
unconfirmed = true;
|
|
@@ -173,37 +183,69 @@ export async function closeParticipantModel(model, warnings) {
|
|
|
173
183
|
warnings.push("Model provider cleanup is unconfirmed.");
|
|
174
184
|
unconfirmed = true;
|
|
175
185
|
}
|
|
186
|
+
const report = model.codexParticipant === undefined && model.provider !== undefined
|
|
187
|
+
? closeReports.get(model.provider)
|
|
188
|
+
: undefined;
|
|
189
|
+
if (report) {
|
|
190
|
+
// The provider's own close reported cleanup above; this reads what its session found.
|
|
191
|
+
try {
|
|
192
|
+
const found = await report();
|
|
193
|
+
warnings.push(...(found.warnings ?? []));
|
|
194
|
+
refusal ??= found.refusal;
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
/* Cleanup was already reported through the provider's close. */
|
|
198
|
+
}
|
|
199
|
+
}
|
|
176
200
|
try {
|
|
177
201
|
await model.claudeSession?.close();
|
|
178
202
|
}
|
|
179
203
|
catch {
|
|
180
204
|
warnings.push("Claude session cleanup failed; desktop cleanup will still run.");
|
|
181
205
|
}
|
|
182
|
-
return unconfirmed;
|
|
206
|
+
return { unconfirmed, refusal };
|
|
183
207
|
}
|
|
184
208
|
/** Prices the finished session's tokens onto its trace, writes the trace, and warns on raw screenshots. */
|
|
185
209
|
export async function recordParticipantTrace(spec, deps, session, warnings) {
|
|
186
|
-
// Per-
|
|
187
|
-
// id is authoritative here
|
|
210
|
+
// Per-participant model-token cost estimate, attached to the trace before it is persisted (the model
|
|
211
|
+
// id is authoritative here: provider.version). Kept at the lab boundary so the pure loop
|
|
188
212
|
// never depends on the operator rate table. estimateActorCost declares absent (null) for an
|
|
189
213
|
// unknown rate / missing usage rather than guessing.
|
|
190
214
|
session.trace.estimatedCost = estimateActorCostForExecution(session.trace.tokenUsage, session.trace.ids.model, session.trace.executionProfile);
|
|
191
215
|
await writeContainedOutputFile(deps.artifactRoot, spec.traceArtifactPath, `${JSON.stringify(session.trace, null, 2)}\n`, "utf8");
|
|
216
|
+
const conversation = session.trace.conversation;
|
|
217
|
+
if (conversation?.mode === "explicit_context")
|
|
218
|
+
warnings.push(explicitContextWarning(conversation));
|
|
192
219
|
if (session.trace.redaction.screenshots === "raw") {
|
|
193
|
-
warnings.push("Screenshots are
|
|
220
|
+
warnings.push("Screenshots are unblurred for local use: the bundle stays in the gitignored .humanish/ folder and nothing scans these pixels, so review them before sharing. Set policies.redactScreenshots: true to blur screenshots in a bundle you plan to share.");
|
|
194
221
|
}
|
|
195
222
|
}
|
|
223
|
+
/**
|
|
224
|
+
* The run warning for a participant whose provider kept no conversation server-side, so humanish
|
|
225
|
+
* carried it on every request. It says why, and how many early turns became a text summary.
|
|
226
|
+
*/
|
|
227
|
+
export function explicitContextWarning(conversation) {
|
|
228
|
+
const why = conversation.explicitReason === "zdr_rejection"
|
|
229
|
+
? `The OpenAI organization rejected server-side conversation state (zero data retention)${conversation.switchedAtRequest === undefined
|
|
230
|
+
? ""
|
|
231
|
+
: ` on request ${conversation.switchedAtRequest}`}`
|
|
232
|
+
: "zeroDataRetention is set";
|
|
233
|
+
const summarized = conversation.summarizedTurns === 0
|
|
234
|
+
? ""
|
|
235
|
+
: ` To stay within its token budget it summarized the ${conversation.summarizedTurns} oldest turns as text and dropped their screenshots.`;
|
|
236
|
+
return `${why}, so humanish carried this participant's conversation itself (explicit_context).${summarized} The trace's conversation record has each request's carried context.`;
|
|
237
|
+
}
|
|
196
238
|
/** The checks that keep a goal_satisfied session from counting as a pass, with their warnings. */
|
|
197
239
|
export function judgeParticipantSession(session, deps, warnings) {
|
|
198
240
|
const noEngagement = session !== undefined && hollowCompletion(sessionEnding(session));
|
|
199
241
|
if (noEngagement) {
|
|
200
|
-
warnings.push("
|
|
242
|
+
warnings.push("The participant returned goal_satisfied with no actions and no messages, so it likely saw a blank or loading screen and stopped. This does not count as a pass. Check the screenshot, and raise execution.timeoutMs or confirm the app had rendered before the first turn.");
|
|
201
243
|
}
|
|
202
244
|
const blockerReason = resolveSelfReportedBlocker(session);
|
|
203
245
|
const selfReportedBlocker = blockerReason !== undefined;
|
|
204
246
|
const reportedFriction = resolveSelfReportedFriction(session) !== undefined;
|
|
205
247
|
if (selfReportedBlocker) {
|
|
206
|
-
warnings.push(`
|
|
248
|
+
warnings.push(`The participant returned goal_satisfied, but its final message describes a blocker or asks for missing instructions, so this does not count as a pass: ${redactText(deps.scrubKnownValues(blockerReason))}`);
|
|
207
249
|
}
|
|
208
250
|
return { noEngagement, selfReportedBlocker, reportedFriction };
|
|
209
251
|
}
|
|
@@ -5,7 +5,7 @@ import { type LabTask } from "../../lab/tasks.js";
|
|
|
5
5
|
import type { DesktopParticipantRun } from "./types.js";
|
|
6
6
|
export declare const DEFAULT_MISSION = "You are testing a web application. The browser is already open at the subject URL. Explore it, accomplish what the scenario asks, and stop when done.";
|
|
7
7
|
/**
|
|
8
|
-
* The participant's outcome as
|
|
8
|
+
* The participant's outcome as one fixed first line of its last message. The
|
|
9
9
|
* free-text computer-use provider has no schema to fill; a fixed line is the next best thing, and
|
|
10
10
|
* the loop reads it into the trace's declaredOutcome. Prompt-only control is weak in general, so
|
|
11
11
|
* adherence is measured (declaredOutcome present or absent on the trace) and the regex over the
|
|
@@ -15,30 +15,30 @@ export declare const DEFAULT_MISSION = "You are testing a web application. The b
|
|
|
15
15
|
export declare const CLOSING_LINE_DIRECTIVE: string;
|
|
16
16
|
/** The persona id a computer-use participant gets when neither it nor its actor names one. */
|
|
17
17
|
export declare const FALLBACK_PERSONA_ID = "cua-operator";
|
|
18
|
-
/** Compose one
|
|
19
|
-
* At N=1 (homogeneous, no roster) this reproduces the prior composeInstructions byte-for-byte. */
|
|
18
|
+
/** Compose one participant's actor prompt: persona line + device line + mission + per-participant
|
|
19
|
+
* steer. At N=1 (homogeneous, no roster) this reproduces the prior composeInstructions byte-for-byte. */
|
|
20
20
|
export declare function composeParticipantInstructions(args: {
|
|
21
21
|
mission: string;
|
|
22
22
|
persona?: string;
|
|
23
23
|
instruction?: string;
|
|
24
|
-
/** The lab's declared protocol
|
|
24
|
+
/** The lab's declared protocol. Only the participant-facing `goal` halves are rendered
|
|
25
25
|
* into the prompt; the `success` criteria never appear here. */
|
|
26
26
|
tasks?: readonly LabTask[];
|
|
27
27
|
device: {
|
|
28
28
|
name: string;
|
|
29
29
|
preset: DevicePreset;
|
|
30
30
|
};
|
|
31
|
-
/** The
|
|
32
|
-
* makes the persona shape behavior
|
|
33
|
-
* traitsApplied
|
|
34
|
-
* no committed file, unparseable YAML) keeps the
|
|
31
|
+
/** The compiled persona for `args.persona`, when its committed file resolved. Supplying it
|
|
32
|
+
* makes the persona shape behavior: its traits become directives in the prompt and land in
|
|
33
|
+
* traitsApplied, instead of appearing as a bare `Persona: <id>.` label. Absent (unsafe id,
|
|
34
|
+
* no committed file, unparseable YAML) keeps the fallback: the bare line and an empty
|
|
35
35
|
* traitsApplied, never fabricated traits. Resolved by the caller so this stays pure. */
|
|
36
36
|
resolvedPersona?: ResolvedPersona;
|
|
37
37
|
/**
|
|
38
|
-
* desktop-cli
|
|
39
|
-
* because a participant whose every prior world was a browser will look for one
|
|
38
|
+
* desktop-cli: the surface under study is a terminal window, not a page. Said plainly
|
|
39
|
+
* because a participant whose every prior world was a browser will look for one, and because a
|
|
40
40
|
* capability nobody declares is one the recording cannot later be read against. It states that a
|
|
41
|
-
* terminal is open and
|
|
41
|
+
* terminal is open and does not say what to type in it: naming commands would answer the question the
|
|
42
42
|
* study is asking.
|
|
43
43
|
*/
|
|
44
44
|
surface?: "desktop-cli";
|
|
@@ -46,8 +46,8 @@ export declare function composeParticipantInstructions(args: {
|
|
|
46
46
|
instructions: string;
|
|
47
47
|
persona: ActorPersonaRef;
|
|
48
48
|
};
|
|
49
|
-
/** Runtime-inject the persona inbox instruction into a
|
|
50
|
-
* runtime loopback/getHost address (not secret), so
|
|
49
|
+
/** Runtime-inject the persona inbox instruction into a participant's prompt. The inbox URL is a
|
|
50
|
+
* runtime loopback/getHost address (not secret), so, mirroring the lobby-code runtime injection, this
|
|
51
51
|
* augments only the instructions the model receives; the authored prompt + its digest are unchanged.
|
|
52
52
|
* Returns a new spec (never mutates). Shared by the CUA + concurrent shared-world routes. */
|
|
53
53
|
export declare function withInboxMission(spec: DesktopParticipantRun, inboxUrl: string, address?: string, receiving?: boolean): DesktopParticipantRun;
|
|
@@ -7,7 +7,7 @@ import { personaBrief, personaToDirectives, renderPersonaPromptSection, } from "
|
|
|
7
7
|
import { renderTaskPrompt } from "../../lab/tasks.js";
|
|
8
8
|
export const DEFAULT_MISSION = "You are testing a web application. The browser is already open at the subject URL. Explore it, accomplish what the scenario asks, and stop when done.";
|
|
9
9
|
/**
|
|
10
|
-
* The participant's outcome as
|
|
10
|
+
* The participant's outcome as one fixed first line of its last message. The
|
|
11
11
|
* free-text computer-use provider has no schema to fill; a fixed line is the next best thing, and
|
|
12
12
|
* the loop reads it into the trace's declaredOutcome. Prompt-only control is weak in general, so
|
|
13
13
|
* adherence is measured (declaredOutcome present or absent on the trace) and the regex over the
|
|
@@ -19,19 +19,19 @@ export const CLOSING_LINE_DIRECTIVE = "When you stop, make the FIRST line of you
|
|
|
19
19
|
"Then, from the next line, say what you did, what confused you, and where you hesitated.";
|
|
20
20
|
/** The persona id a computer-use participant gets when neither it nor its actor names one. */
|
|
21
21
|
export const FALLBACK_PERSONA_ID = "cua-operator";
|
|
22
|
-
/** Compose one
|
|
23
|
-
* At N=1 (homogeneous, no roster) this reproduces the prior composeInstructions byte-for-byte. */
|
|
22
|
+
/** Compose one participant's actor prompt: persona line + device line + mission + per-participant
|
|
23
|
+
* steer. At N=1 (homogeneous, no roster) this reproduces the prior composeInstructions byte-for-byte. */
|
|
24
24
|
export function composeParticipantInstructions(args) {
|
|
25
25
|
const { name, preset } = args.device;
|
|
26
26
|
const deviceLine = preset.isMobile
|
|
27
27
|
? `You are a mobile user on a ${name} device (${preset.width}x${preset.height} @${preset.deviceScaleFactor}x). Expect a mobile/touch layout.`
|
|
28
28
|
: `You are a desktop user (${name}, ${preset.width}x${preset.height}).`;
|
|
29
|
-
// The protocol as the
|
|
30
|
-
// are the researcher's instrument and must never reach this prompt
|
|
29
|
+
// The protocol as the participant reads it: numbered goals, nothing else. The success criteria
|
|
30
|
+
// are the researcher's instrument and must never reach this prompt: a persona told how it will
|
|
31
31
|
// be measured optimizes for the measurement instead of using the product (src/lab/tasks.ts).
|
|
32
32
|
const taskLines = renderTaskPrompt(args.tasks ?? []);
|
|
33
33
|
// A resolved persona contributes its compiled directives (friction tolerance, skill bias,
|
|
34
|
-
// accessibility behavior, constraints) through the
|
|
34
|
+
// accessibility behavior, constraints) through the same persona.ts compiler the terminal route
|
|
35
35
|
// uses, so one persona file means one behavior across every route.
|
|
36
36
|
const personaLine = args.resolvedPersona
|
|
37
37
|
? renderPersonaPromptSection(args.resolvedPersona)
|
|
@@ -64,8 +64,8 @@ export function composeParticipantInstructions(args) {
|
|
|
64
64
|
},
|
|
65
65
|
};
|
|
66
66
|
}
|
|
67
|
-
/** Runtime-inject the persona inbox instruction into a
|
|
68
|
-
* runtime loopback/getHost address (not secret), so
|
|
67
|
+
/** Runtime-inject the persona inbox instruction into a participant's prompt. The inbox URL is a
|
|
68
|
+
* runtime loopback/getHost address (not secret), so, mirroring the lobby-code runtime injection, this
|
|
69
69
|
* augments only the instructions the model receives; the authored prompt + its digest are unchanged.
|
|
70
70
|
* Returns a new spec (never mutates). Shared by the CUA + concurrent shared-world routes. */
|
|
71
71
|
export function withInboxMission(spec, inboxUrl, address, receiving = false) {
|