humanish 0.106.1 → 0.107.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -4
- package/ARCHITECTURE.md +17 -16
- package/CHANGELOG.md +14 -0
- package/CONTEXT.md +42 -39
- package/CONTRIBUTING.md +8 -8
- package/README.md +8 -3
- package/dist/actors/codex/app-server-client.js +2 -1
- package/dist/actors/codex/app-server-trace.d.ts +0 -1
- package/dist/actors/codex/app-server-trace.js +1 -3
- package/dist/actors/codex/app-server.js +2 -1
- package/dist/actors/codex/protocol-compat.d.ts +27 -0
- package/dist/actors/codex/protocol-compat.js +357 -0
- package/dist/actors/codex/protocol-contract.d.ts +61 -0
- package/dist/actors/codex/protocol-contract.js +471 -0
- package/dist/actors/codex/qualified-versions.d.ts +30 -3
- package/dist/actors/codex/qualified-versions.js +50 -5
- package/dist/actors/codex/restricted-executable.d.ts +24 -0
- package/dist/actors/codex/restricted-executable.js +97 -22
- package/dist/actors/codex/restricted-launch.d.ts +7 -1
- package/dist/actors/codex/restricted-launch.js +42 -3
- package/dist/actors/codex/restricted-notifications.d.ts +101 -0
- package/dist/actors/codex/restricted-notifications.js +277 -0
- package/dist/actors/codex/restricted-participant.d.ts +30 -0
- package/dist/actors/codex/restricted-participant.js +60 -2
- package/dist/actors/codex/restricted-policy.d.ts +9 -1
- package/dist/actors/codex/restricted-session.d.ts +14 -1
- package/dist/actors/codex/restricted-session.js +50 -17
- package/dist/actors/codex/restricted-transport.d.ts +29 -0
- package/dist/actors/codex/restricted-transport.js +139 -27
- package/dist/actors/codex/restricted-turn.d.ts +19 -4
- package/dist/actors/codex/restricted-turn.js +43 -30
- package/dist/actors/computer-use/actor.d.ts +1 -1
- package/dist/actors/computer-use/actor.js +4 -4
- package/dist/actors/computer-use/loop/backstop.js +1 -1
- package/dist/actors/computer-use/loop/ending.d.ts +2 -2
- package/dist/actors/computer-use/loop/ending.js +2 -2
- package/dist/actors/computer-use/loop/spend.d.ts +1 -1
- package/dist/actors/computer-use/loop/spend.js +3 -3
- package/dist/actors/computer-use/loop/trace.js +2 -2
- package/dist/actors/computer-use/loop/types.d.ts +14 -5
- package/dist/actors/computer-use/loop.js +4 -4
- package/dist/actors/computer-use/openai-provider.d.ts +1 -1
- package/dist/actors/computer-use/openai-provider.js +2 -2
- package/dist/actors/contract.d.ts +9 -7
- package/dist/actors/local-agent/cli.js +1 -1
- package/dist/actors/scripted-browser/actor.d.ts +2 -2
- package/dist/actors/scripted-browser/journey.js +15 -4
- package/dist/actors/scripted-browser/steps.js +18 -0
- package/dist/actors/scripted-browser/types.d.ts +11 -1
- package/dist/actors/stop-conditions.js +1 -3
- package/dist/analysis/automatic-completion.d.ts +13 -8
- package/dist/analysis/automatic-completion.js +14 -8
- package/dist/analysis/automatic-config.d.ts +1 -1
- package/dist/analysis/automatic-config.js +2 -2
- package/dist/analysis/automatic.d.ts +10 -4
- package/dist/analysis/automatic.js +17 -13
- package/dist/analysis/diagnostics.d.ts +23 -0
- package/dist/analysis/diagnostics.js +112 -0
- package/dist/analysis/evidence-sources.js +2 -2
- package/dist/analysis/evidence.js +6 -6
- package/dist/analysis/execute.d.ts +7 -1
- package/dist/analysis/execute.js +39 -12
- package/dist/analysis/job.d.ts +8 -5
- package/dist/analysis/job.js +15 -14
- package/dist/analysis/load.d.ts +1 -1
- package/dist/analysis/load.js +5 -5
- package/dist/analysis/provider.d.ts +9 -1
- package/dist/analysis/restricted-codex.d.ts +2 -0
- package/dist/analysis/restricted-codex.js +10 -1
- package/dist/analysis/service.d.ts +2 -0
- package/dist/analysis/service.js +31 -12
- package/dist/analysis/sharing.d.ts +2 -2
- package/dist/analysis/sharing.js +5 -5
- package/dist/analysis/store-executions.d.ts +5 -5
- package/dist/analysis/store-executions.js +12 -12
- package/dist/analysis/store.d.ts +4 -4
- package/dist/analysis/store.js +13 -13
- package/dist/cli/commands/analysis-signals.d.ts +2 -1
- package/dist/cli/commands/analysis-signals.js +4 -1
- package/dist/cli/commands/analyze.js +4 -2
- package/dist/cli/commands/comms.js +8 -1
- package/dist/cli/commands/lab-route-computer-use.d.ts +0 -2
- package/dist/cli/commands/lab-route-computer-use.js +26 -14
- package/dist/cli/commands/lab-route-preview.d.ts +1 -3
- package/dist/cli/commands/lab-route-preview.js +3 -4
- package/dist/cli/commands/lab-route-run.d.ts +6 -4
- package/dist/cli/commands/lab-route-run.js +6 -2
- package/dist/cli/commands/lab-route-scripted.d.ts +0 -2
- package/dist/cli/commands/lab-route-scripted.js +0 -1
- package/dist/cli/commands/lab-route-shared-world.d.ts +0 -2
- package/dist/cli/commands/lab-route-shared-world.js +0 -1
- package/dist/cli/commands/lab-route-terminal.d.ts +0 -2
- package/dist/cli/commands/lab-route-terminal.js +0 -1
- package/dist/cli/commands/lab-run.js +76 -45
- package/dist/cli/commands/lab-scorer.d.ts +0 -2
- package/dist/cli/commands/lab.js +12 -6
- package/dist/cli/commands/observe.js +22 -0
- package/dist/cli/commands/run-signals.d.ts +30 -0
- package/dist/cli/commands/run-signals.js +146 -0
- package/dist/cli/commands/runs.js +10 -8
- package/dist/cli/commands/watch.js +12 -10
- package/dist/cli/doctor.d.ts +3 -0
- package/dist/cli/doctor.js +10 -2
- package/dist/cli/io.d.ts +5 -4
- package/dist/cli/io.js +12 -8
- package/dist/cli/program.js +3 -3
- package/dist/cli/renamed-options.d.ts +11 -0
- package/dist/cli/renamed-options.js +18 -0
- package/dist/cli/telemetry.d.ts +2 -2
- package/dist/cli/telemetry.js +1 -1
- package/dist/comms/capture-surface.d.ts +1 -1
- package/dist/comms/catch-host.d.ts +1 -1
- package/dist/comms/connections.js +4 -4
- package/dist/comms/external-evidence.d.ts +27 -0
- package/dist/comms/external-evidence.js +93 -0
- package/dist/comms/lease-store.js +7 -7
- package/dist/comms/types.js +1 -1
- package/dist/evidence/browser-hygiene.d.ts +1 -1
- package/dist/evidence/browser-hygiene.js +1 -1
- package/dist/evidence/redaction.js +1 -1
- package/dist/evidence/secret-scrub.d.ts +9 -0
- package/dist/evidence/secret-scrub.js +96 -0
- package/dist/feedback/draft.js +7 -9
- package/dist/feedback/export-bundle.js +4 -4
- package/dist/feedback/export.js +9 -9
- package/dist/feedback/feedback.d.ts +1 -1
- package/dist/index.d.ts +3 -6
- package/dist/index.js +3 -6
- package/dist/keys/key-resolution.d.ts +3 -1
- package/dist/keys/key-resolution.js +18 -16
- package/dist/lab/adapter-extension.d.ts +28 -6
- package/dist/lab/adapter-extension.js +65 -126
- package/dist/lab/adapter-scorer-loader.d.ts +25 -5
- package/dist/lab/adapter-scorer-loader.js +24 -0
- package/dist/lab/composition-rules.js +20 -20
- package/dist/lab/config.js +28 -26
- package/dist/lab/device-presets.d.ts +2 -2
- package/dist/lab/discover.js +1 -3
- package/dist/lab/doctor.d.ts +44 -9
- package/dist/lab/doctor.js +149 -41
- package/dist/lab/init-plan.js +1 -3
- package/dist/lab/keys.js +1 -1
- package/dist/lab/lab-deps.d.ts +85 -0
- package/dist/lab/lab-deps.js +4 -0
- package/dist/lab/parse/actors.js +8 -7
- package/dist/lab/parse/comms.js +3 -2
- package/dist/lab/parse/execution.js +3 -2
- package/dist/lab/parse/subject-state.js +2 -1
- package/dist/lab/parse/subject.js +2 -1
- package/dist/lab/parse/values.d.ts +0 -1
- package/dist/lab/parse/values.js +0 -3
- package/dist/lab/persona-resolve.d.ts +3 -3
- package/dist/lab/persona-resolve.js +4 -4
- package/dist/lab/persona.js +1 -3
- package/dist/lab/plan-base.d.ts +0 -2
- package/dist/lab/plan-base.js +6 -4
- package/dist/lab/plan-participants.d.ts +5 -5
- package/dist/lab/plan-participants.js +2 -2
- package/dist/lab/plan-types.d.ts +22 -4
- package/dist/lab/plan.d.ts +3 -11
- package/dist/lab/plan.js +18 -42
- package/dist/lab/preflight-result.js +0 -2
- package/dist/lab/preflight.d.ts +0 -3
- package/dist/lab/preflight.js +1 -1
- package/dist/lab/requirements.d.ts +20 -0
- package/dist/lab/requirements.js +54 -0
- package/dist/lab/route-inputs.d.ts +23 -12
- package/dist/lab/route-inputs.js +48 -51
- package/dist/lab/routing.d.ts +19 -24
- package/dist/lab/routing.js +14 -47
- package/dist/lab/run-lab-events.d.ts +6 -11
- package/dist/lab/run-lab-events.js +3 -11
- package/dist/lab/run-lab-homes.d.ts +64 -0
- package/dist/lab/run-lab-homes.js +3 -0
- package/dist/lab/run-lab-options.d.ts +12 -66
- package/dist/lab/run-lab-options.js +45 -275
- package/dist/lab/summary.d.ts +9 -4
- package/dist/lab/summary.js +15 -8
- package/dist/lab/types.d.ts +132 -132
- package/dist/lab/validation.d.ts +15 -15
- package/dist/lab/validation.js +28 -28
- package/dist/lab/warnings.js +20 -19
- package/dist/observer/artifact.d.ts +1 -1
- package/dist/observer/artifact.js +3 -3
- package/dist/observer/data.js +2 -2
- package/dist/observer/render.js +2 -2
- package/dist/observer/run-routes.d.ts +1 -1
- package/dist/observer/run-routes.js +5 -5
- package/dist/observer/serve.d.ts +17 -0
- package/dist/observer/serve.js +34 -3
- package/dist/routes/computer-use/bundle-parts.d.ts +1 -1
- package/dist/routes/computer-use/bundle-parts.js +10 -3
- package/dist/routes/computer-use/bundle.d.ts +4 -11
- package/dist/routes/computer-use/bundle.js +9 -15
- package/dist/routes/computer-use/diagnostics.d.ts +1 -1
- package/dist/routes/computer-use/diagnostics.js +1 -1
- package/dist/routes/computer-use/e2b-desktop/comms.d.ts +10 -10
- package/dist/routes/computer-use/e2b-desktop/comms.js +11 -14
- package/dist/routes/computer-use/e2b-desktop/desktop.d.ts +2 -2
- package/dist/routes/computer-use/e2b-desktop/desktop.js +4 -7
- package/dist/routes/computer-use/e2b-desktop/fidelity.d.ts +13 -13
- package/dist/routes/computer-use/e2b-desktop/fidelity.js +11 -11
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +3 -3
- package/dist/routes/computer-use/e2b-desktop/prepare.js +33 -36
- package/dist/routes/computer-use/e2b-desktop/start.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/start.js +24 -23
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +5 -4
- package/dist/routes/computer-use/e2b-desktop/state.js +5 -3
- package/dist/routes/computer-use/e2b-desktop/teardown.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/teardown.js +29 -17
- package/dist/routes/computer-use/external-comms.d.ts +4 -4
- package/dist/routes/computer-use/external-comms.js +19 -43
- package/dist/routes/computer-use/fanout-bundle.d.ts +3 -3
- package/dist/routes/computer-use/fanout-bundle.js +25 -29
- package/dist/routes/computer-use/{fanout-lanes.js → fanout-records.js} +25 -19
- package/dist/routes/computer-use/in-process-desktop.d.ts +3 -10
- package/dist/routes/computer-use/in-process-desktop.js +3 -9
- package/dist/routes/computer-use/live-flush.d.ts +7 -5
- package/dist/routes/computer-use/live-flush.js +19 -27
- package/dist/routes/computer-use/{run-lanes.js → live-phase.js} +12 -11
- package/dist/routes/computer-use/local-tree-pack.d.ts +3 -3
- package/dist/routes/computer-use/local-tree-pack.js +4 -5
- package/dist/routes/computer-use/local-vm.d.ts +11 -8
- package/dist/routes/computer-use/local-vm.js +39 -66
- package/dist/routes/computer-use/participant-desktop.d.ts +8 -23
- package/dist/routes/computer-use/participant-desktop.js +5 -25
- package/dist/routes/computer-use/{lanes.d.ts → participant-execution.d.ts} +10 -10
- package/dist/routes/computer-use/{lanes.js → participant-execution.js} +35 -22
- package/dist/routes/computer-use/participant-facts.d.ts +7 -1
- package/dist/routes/computer-use/participant-facts.js +12 -1
- package/dist/routes/computer-use/participant-model.d.ts +19 -11
- package/dist/routes/computer-use/participant-model.js +49 -25
- package/dist/routes/computer-use/participant-prompt.d.ts +3 -3
- package/dist/routes/computer-use/participant-prompt.js +4 -4
- package/dist/routes/computer-use/participant-runs.d.ts +6 -6
- package/dist/routes/computer-use/participant-runs.js +8 -8
- package/dist/routes/computer-use/plan.d.ts +21 -29
- package/dist/routes/computer-use/plan.js +52 -60
- package/dist/routes/computer-use/preflight.d.ts +3 -5
- package/dist/routes/computer-use/preflight.js +20 -16
- package/dist/routes/computer-use/rerun-selection.js +10 -6
- package/dist/routes/computer-use/result.d.ts +1 -1
- package/dist/routes/computer-use/result.js +40 -30
- package/dist/routes/computer-use/route.d.ts +3 -4
- package/dist/routes/computer-use/route.js +21 -19
- package/dist/routes/computer-use/self-report.d.ts +2 -2
- package/dist/routes/computer-use/self-report.js +4 -4
- package/dist/routes/computer-use/setup.d.ts +8 -14
- package/dist/routes/computer-use/setup.js +96 -72
- package/dist/routes/computer-use/single-bundle.d.ts +8 -11
- package/dist/routes/computer-use/single-bundle.js +20 -25
- package/dist/routes/computer-use/subject-projection.d.ts +5 -7
- package/dist/routes/computer-use/subject-projection.js +20 -19
- package/dist/routes/computer-use/types.d.ts +169 -210
- package/dist/routes/computer-use/types.js +13 -5
- package/dist/routes/preview.js +0 -1
- package/dist/routes/scripted/bundle.d.ts +3 -6
- package/dist/routes/scripted/bundle.js +11 -16
- package/dist/routes/scripted/plan.d.ts +5 -7
- package/dist/routes/scripted/plan.js +6 -4
- package/dist/routes/scripted/result.d.ts +4 -1
- package/dist/routes/scripted/result.js +10 -6
- package/dist/routes/scripted/route.js +10 -11
- package/dist/routes/scripted/scenario.d.ts +1 -1
- package/dist/routes/scripted/scenario.js +1 -1
- package/dist/routes/scripted/setup.d.ts +3 -2
- package/dist/routes/scripted/setup.js +22 -21
- package/dist/routes/scripted/subject.d.ts +8 -3
- package/dist/routes/scripted/subject.js +9 -5
- package/dist/routes/scripted/surface-records.js +1 -1
- package/dist/routes/scripted/surface-sessions.d.ts +2 -2
- package/dist/routes/scripted/surface-sessions.js +4 -4
- package/dist/routes/scripted/types.d.ts +19 -36
- package/dist/routes/scripted/types.js +1 -2
- package/dist/routes/shared-world/{seat-records.js → bundle-records.js} +8 -8
- package/dist/routes/shared-world/bundle.d.ts +2 -2
- package/dist/routes/shared-world/bundle.js +22 -26
- package/dist/routes/shared-world/checkpoints.d.ts +4 -4
- package/dist/routes/shared-world/checkpoints.js +4 -4
- package/dist/routes/shared-world/comms.d.ts +5 -4
- package/dist/routes/shared-world/comms.js +28 -41
- package/dist/routes/shared-world/external-public.d.ts +5 -5
- package/dist/routes/shared-world/external-public.js +19 -20
- package/dist/routes/shared-world/handoff.d.ts +4 -4
- package/dist/routes/shared-world/handoff.js +50 -50
- package/dist/routes/shared-world/lobby-code.d.ts +5 -5
- package/dist/routes/shared-world/lobby-code.js +13 -13
- package/dist/routes/shared-world/{seats.d.ts → participant-specs.d.ts} +8 -6
- package/dist/routes/shared-world/{seats.js → participant-specs.js} +43 -48
- package/dist/routes/shared-world/plan.d.ts +2 -6
- package/dist/routes/shared-world/plan.js +1 -2
- package/dist/routes/shared-world/provenance.d.ts +4 -4
- package/dist/routes/shared-world/provenance.js +4 -4
- package/dist/routes/shared-world/provisioned.d.ts +3 -2
- package/dist/routes/shared-world/provisioned.js +46 -40
- package/dist/routes/shared-world/result.js +31 -19
- package/dist/routes/shared-world/route.js +37 -41
- package/dist/routes/shared-world/setup.d.ts +5 -4
- package/dist/routes/shared-world/setup.js +12 -14
- package/dist/routes/shared-world/types.d.ts +53 -126
- package/dist/routes/shared-world/types.js +2 -2
- package/dist/routes/terminal/adapter.d.ts +8 -8
- package/dist/routes/terminal/adapter.js +28 -125
- package/dist/routes/terminal/bundle.d.ts +7 -12
- package/dist/routes/terminal/bundle.js +24 -34
- package/dist/routes/terminal/credentials.d.ts +2 -2
- package/dist/routes/terminal/credentials.js +16 -16
- package/dist/routes/terminal/dry-run.d.ts +1 -1
- package/dist/routes/terminal/dry-run.js +9 -11
- package/dist/routes/terminal/encoding.d.ts +1 -1
- package/dist/routes/terminal/encoding.js +2 -2
- package/dist/routes/terminal/ledger.d.ts +6 -6
- package/dist/routes/terminal/ledger.js +9 -9
- package/dist/routes/terminal/live-finish.d.ts +1 -2
- package/dist/routes/terminal/live-finish.js +21 -20
- package/dist/routes/terminal/live-sandbox.d.ts +3 -2
- package/dist/routes/terminal/live-sandbox.js +20 -20
- package/dist/routes/terminal/participant-text.d.ts +23 -0
- package/dist/routes/terminal/participant-text.js +165 -0
- package/dist/routes/terminal/plan.d.ts +2 -6
- package/dist/routes/terminal/plan.js +13 -9
- package/dist/routes/terminal/recorder.d.ts +6 -1
- package/dist/routes/terminal/recorder.js +12 -4
- package/dist/routes/terminal/result.js +4 -4
- package/dist/routes/terminal/route.js +26 -26
- package/dist/routes/terminal/runtime.d.ts +3 -2
- package/dist/routes/terminal/runtime.js +3 -3
- package/dist/routes/terminal/sandbox.d.ts +3 -3
- package/dist/routes/terminal/sandbox.js +9 -9
- package/dist/routes/terminal/session.js +16 -14
- package/dist/routes/terminal/token-usage.js +5 -5
- package/dist/routes/terminal/trace.d.ts +5 -8
- package/dist/routes/terminal/trace.js +9 -64
- package/dist/routes/terminal/types.d.ts +85 -93
- package/dist/routes/terminal/types.js +9 -4
- package/dist/run/active-runs.d.ts +13 -0
- package/dist/run/active-runs.js +12 -0
- package/dist/run/bundle-shape.d.ts +2 -1
- package/dist/run/bundle-shape.js +4 -4
- package/dist/run/bundle.d.ts +63 -56
- package/dist/run/bundle.js +9 -8
- package/dist/run/cost-summary.js +3 -2
- package/dist/run/costs.d.ts +6 -6
- package/dist/run/costs.js +10 -10
- package/dist/run/detail.d.ts +5 -5
- package/dist/run/detail.js +6 -6
- package/dist/run/dry-run.js +5 -12
- package/dist/run/{study-files.d.ts → evidence-files.d.ts} +4 -4
- package/dist/run/{study-files.js → evidence-files.js} +5 -5
- package/dist/run/feedback-shape.js +1 -1
- package/dist/run/judge.d.ts +16 -4
- package/dist/run/judge.js +24 -7
- package/dist/run/locate.d.ts +2 -0
- package/dist/run/locate.js +3 -1
- package/dist/run/outcomes.d.ts +3 -3
- package/dist/run/outcomes.js +3 -3
- package/dist/run/participant-records.d.ts +2 -2
- package/dist/run/participant-records.js +2 -2
- package/dist/run/paths.js +1 -1
- package/dist/run/pricing.d.ts +19 -9
- package/dist/run/pricing.js +30 -18
- package/dist/run/projection.d.ts +13 -13
- package/dist/run/projection.js +8 -8
- package/dist/run/reclaim.d.ts +6 -0
- package/dist/run/reclaim.js +13 -3
- package/dist/run/results.d.ts +1 -1
- package/dist/run/run-index.d.ts +1 -1
- package/dist/run/run-index.js +11 -11
- package/dist/run/run.d.ts +4 -2
- package/dist/run/run.js +9 -5
- package/dist/run/sandbox-receipts.d.ts +4 -4
- package/dist/run/sandbox-receipts.js +4 -4
- package/dist/run/shared-world-evidence.d.ts +47 -47
- package/dist/run/shared-world-shape.d.ts +3 -3
- package/dist/run/shared-world-shape.js +11 -11
- package/dist/run/stats.d.ts +5 -5
- package/dist/run/stats.js +8 -8
- package/dist/run/status.d.ts +16 -6
- package/dist/run/status.js +31 -8
- package/dist/run/streams.d.ts +26 -19
- package/dist/run/terminal-contract.d.ts +5 -5
- package/dist/run/terminal-contract.js +4 -4
- package/dist/run/transient-comms-secrets.d.ts +5 -0
- package/dist/run/transient-comms-secrets.js +12 -0
- package/dist/run-lab.d.ts +36 -34
- package/dist/run-lab.js +41 -37
- package/dist/subject/desktop-cli.d.ts +1 -1
- package/dist/subject/desktop-cli.js +1 -1
- package/dist/subject/local-tree-archive.d.ts +1 -1
- package/dist/subject/local-tree-archive.js +4 -4
- package/dist/subject/local-tree.d.ts +2 -2
- package/dist/subject/local-tree.js +1 -1
- package/dist/subject/node-bootstrap.d.ts +1 -1
- package/dist/subject/node-bootstrap.js +4 -4
- package/dist/subject/runtime.js +5 -5
- package/dist/subject/serve.js +1 -1
- package/dist/subject/steps.d.ts +4 -4
- package/dist/subject/steps.js +5 -5
- package/dist/substrates/e2b/cdp-probe.d.ts +2 -2
- package/dist/substrates/e2b/cdp-probe.js +1 -1
- package/dist/substrates/e2b/desktop-browser.d.ts +3 -3
- package/dist/substrates/e2b/desktop-browser.js +4 -4
- package/dist/substrates/e2b/desktop-cdp.d.ts +7 -7
- package/dist/substrates/e2b/desktop-cdp.js +4 -4
- package/dist/substrates/e2b/desktop-geometry.d.ts +4 -4
- package/dist/substrates/e2b/desktop-geometry.js +3 -3
- package/dist/substrates/e2b/desktop-media.js +2 -2
- package/dist/substrates/e2b/sandbox.js +1 -1
- package/dist/substrates/e2b/sdk.js +2 -2
- package/dist/substrates/local/runtime-host.d.ts +2 -0
- package/dist/substrates/local/runtime-host.js +6 -0
- package/dist/tui/actions.d.ts +7 -5
- package/dist/tui/actions.js +18 -9
- package/dist/tui-app.js +42 -42
- package/dist/verify/actor.d.ts +10 -10
- package/dist/verify/actor.js +15 -15
- package/dist/verify/artifacts.d.ts +1 -1
- package/dist/verify/artifacts.js +3 -3
- package/dist/verify/costs.d.ts +2 -2
- package/dist/verify/costs.js +2 -2
- package/dist/verify/liveness.d.ts +1 -1
- package/dist/verify/liveness.js +4 -1
- package/dist/verify/rerun.d.ts +1 -1
- package/dist/verify/rerun.js +1 -1
- package/dist/verify/shared-world.js +2 -2
- package/dist/verify/verify.js +6 -6
- package/docs/README.md +1 -1
- package/docs/architecture/actor-contract.md +60 -22
- package/docs/architecture/desktop-sessions.md +9 -9
- package/docs/architecture/external-public-shared-world.md +5 -0
- package/docs/architecture/observer.md +4 -4
- package/docs/architecture/restricted-codex-analysis.md +36 -2
- package/docs/architecture/state-driven-executor.md +4 -5
- package/docs/architecture/task-protocol-support.md +3 -5
- package/docs/architecture/{terminal-product-lane.md → terminal-product-route.md} +24 -16
- package/docs/contracts/run-bundle.md +10 -8
- package/docs/contracts/schemas.md +109 -53
- package/docs/contracts/study-analysis.md +24 -0
- package/docs/decisions/0004-participant-desktops-stay-in-the-route.md +2 -1
- package/docs/product/automatic-analysis.md +1 -0
- package/docs/ramp/README.md +2 -1
- package/package.json +4 -4
- package/skills/humanish/SKILL.md +3 -2
- package/skills/humanish/agents/openai.yaml +1 -1
- package/dist/deprecated.d.ts +0 -90
- package/dist/deprecated.js +0 -106
- package/dist/lab/bag-overrides.d.ts +0 -18
- package/dist/lab/bag-overrides.js +0 -121
- package/dist/routes/computer-use/legacy-lane-spec.d.ts +0 -62
- package/dist/routes/computer-use/legacy-lane-spec.js +0 -39
- /package/dist/routes/computer-use/{fanout-lanes.d.ts → fanout-records.d.ts} +0 -0
- /package/dist/routes/computer-use/{run-lanes.d.ts → live-phase.d.ts} +0 -0
- /package/dist/routes/shared-world/{seat-records.d.ts → bundle-records.d.ts} +0 -0
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { adapterScoreFailureMessage,
|
|
1
|
+
import { adapterScoreFailureMessage, applyBrowserScorer } from "../../lab/adapter-extension.js";
|
|
2
2
|
import { redactText } from "../../evidence/redaction.js";
|
|
3
3
|
import { round6 } from "../../run/pricing.js";
|
|
4
4
|
import { foldScorerFailures, judgeExecution, OUTCOME_POLICIES, participantHarnessFailed, resultOk, } from "../../run/judge.js";
|
|
5
|
-
import { participantFactsOf, participantOutcomeOk } from "./participant-facts.js";
|
|
5
|
+
import { participantFactsOf, participantOutcomeOk, unreleasedSandboxFailures, } from "./participant-facts.js";
|
|
6
6
|
import { summarizeCuaDiagnostics } from "./diagnostics.js";
|
|
7
|
-
import { toParticipantResult } from "./
|
|
7
|
+
import { toParticipantResult } from "./participant-execution.js";
|
|
8
8
|
import { buildCuaRunBundle, judgeComputerUseRun } from "./bundle.js";
|
|
9
9
|
import { aggregateCuaSubject, projectParticipantSubjects, subjectProvenanceArg, } from "./subject-projection.js";
|
|
10
|
-
import { CUA_ACTOR_LAB_SCHEMA, CUA_FANOUT_STRATEGY, } from "./types.js";
|
|
10
|
+
import { CUA_ACTOR_LAB_SCHEMA, CUA_FANOUT_STRATEGY, participantSubjectEnv, } from "./types.js";
|
|
11
|
+
import { plannedAppUrl } from "./plan.js";
|
|
11
12
|
/** Aggregate participant counts for the result projection. */
|
|
12
13
|
function buildParticipantSummary(outcomes, participantCount, participantPlan, dryRun) {
|
|
13
14
|
if (dryRun || !outcomes) {
|
|
@@ -51,13 +52,14 @@ function buildParticipantSummary(outcomes, participantCount, participantPlan, dr
|
|
|
51
52
|
}
|
|
52
53
|
/**
|
|
53
54
|
* The computer-use lab result for a finished run. The run passes only when the Observer rendered,
|
|
54
|
-
* every
|
|
55
|
+
* every participant passed (dry-run participants pass as contracts), and no adapter score or declared scorer
|
|
55
56
|
* verdict failed; otherwise the error names the first reason.
|
|
56
57
|
*/
|
|
57
58
|
function cuaLabResult(args) {
|
|
58
59
|
const { labId, cwd, runId, appUrl, dryRun, participantRuns, outcomes, subjects, aggregateSubject, participantPlan, rerunLineage, bundle, observer, receivingWarnings, aggregateWarnings, adapterWarnings, } = args;
|
|
59
60
|
const participantCount = participantRuns.length;
|
|
60
|
-
//
|
|
61
|
+
// Participant-level pass: dry-run participants are contract-ok; live ones need a passed,
|
|
62
|
+
// engaged session.
|
|
61
63
|
const participantOk = (outcome) => participantOutcomeOk(outcome, dryRun);
|
|
62
64
|
const adapterFailure = adapterScoreFailureMessage(bundle);
|
|
63
65
|
const ok = resultOk({
|
|
@@ -81,10 +83,7 @@ function cuaLabResult(args) {
|
|
|
81
83
|
if (ok)
|
|
82
84
|
return undefined;
|
|
83
85
|
if (adapterFailure !== undefined) {
|
|
84
|
-
return {
|
|
85
|
-
code: "HUMANISH_CUA_LAB_FAILED",
|
|
86
|
-
message: adapterFailure,
|
|
87
|
-
};
|
|
86
|
+
return { code: "HUMANISH_CUA_LAB_FAILED", message: adapterFailure };
|
|
88
87
|
}
|
|
89
88
|
if (participantCount === 1) {
|
|
90
89
|
const outcome = firstOutcome;
|
|
@@ -92,9 +91,10 @@ function cuaLabResult(args) {
|
|
|
92
91
|
code: outcome?.failureCode ?? "HUMANISH_CUA_LAB_FAILED",
|
|
93
92
|
message: outcome?.sessionError ??
|
|
94
93
|
outcome?.providerCleanupError ??
|
|
94
|
+
outcome?.providerPolicyError ??
|
|
95
95
|
(outcome?.noEngagement
|
|
96
96
|
? "Actor took no actions and produced no message (likely a blank/still-loading screen); not a credible goal_satisfied."
|
|
97
|
-
: // The
|
|
97
|
+
: // The participant result (toParticipantResult) named this refusal; the N=1 envelope fell through to
|
|
98
98
|
// "did not produce a terminal session", which is false — it produced one and refused it.
|
|
99
99
|
outcome?.selfReportedBlocker
|
|
100
100
|
? "Actor reported goal_satisfied while its final message described a blocker or asked for missing instructions; not a credible pass."
|
|
@@ -164,10 +164,11 @@ function cuaLabResult(args) {
|
|
|
164
164
|
};
|
|
165
165
|
}
|
|
166
166
|
/**
|
|
167
|
-
* The run's execution failures: each
|
|
168
|
-
* that
|
|
167
|
+
* The run's execution failures: each participant whose session failed in the harness, each provider whose
|
|
168
|
+
* cleanup is unconfirmed or that reported a disallowed item after its last request, each sandbox
|
|
169
|
+
* whose release is unconfirmed, and an Observer that failed.
|
|
169
170
|
*/
|
|
170
|
-
function computerUseExecutionFailures(outcomes, observer) {
|
|
171
|
+
function computerUseExecutionFailures(outcomes, observer, runId) {
|
|
171
172
|
return [
|
|
172
173
|
...(outcomes ?? [])
|
|
173
174
|
.filter((outcome) => participantHarnessFailed(participantFactsOf(outcome)))
|
|
@@ -183,6 +184,15 @@ function computerUseExecutionFailures(outcomes, observer) {
|
|
|
183
184
|
message: `${outcome.spec.planned.id}: ${outcome.providerCleanupError}`,
|
|
184
185
|
},
|
|
185
186
|
]),
|
|
187
|
+
...(outcomes ?? []).flatMap((outcome) => outcome.providerPolicyError === undefined
|
|
188
|
+
? []
|
|
189
|
+
: [
|
|
190
|
+
{
|
|
191
|
+
kind: "provider-policy",
|
|
192
|
+
message: `${outcome.spec.planned.id}: ${outcome.providerPolicyError}`,
|
|
193
|
+
},
|
|
194
|
+
]),
|
|
195
|
+
...unreleasedSandboxFailures(outcomes, runId),
|
|
186
196
|
...(observer.ok
|
|
187
197
|
? []
|
|
188
198
|
: [
|
|
@@ -194,33 +204,34 @@ function computerUseExecutionFailures(outcomes, observer) {
|
|
|
194
204
|
];
|
|
195
205
|
}
|
|
196
206
|
/**
|
|
197
|
-
* execution.caps.maxUsd is enforced inside each
|
|
198
|
-
* spend up to N × maxUsd before any
|
|
207
|
+
* execution.caps.maxUsd is enforced inside each participant's loop independently, so an
|
|
208
|
+
* N-participant fan-out can spend up to N × maxUsd before any participant aborts, while the run cost summary reports the larger
|
|
199
209
|
* aggregate. The warning names that ceiling, unless the study declared a shared maxTotalUsd budget.
|
|
200
210
|
*/
|
|
201
|
-
function participantCapWarning(
|
|
202
|
-
const capUsd =
|
|
211
|
+
function participantCapWarning(caps, participantCount) {
|
|
212
|
+
const capUsd = caps.maxUsd;
|
|
203
213
|
if (capUsd === undefined || participantCount <= 1)
|
|
204
214
|
return undefined;
|
|
205
|
-
if (
|
|
215
|
+
if (caps.maxTotalUsd !== undefined)
|
|
206
216
|
return undefined;
|
|
207
217
|
return `execution.caps.maxUsd ($${capUsd}) is a PER-PARTICIPANT cap; ${participantCount} participants may spend up to ${participantCount} × $${capUsd} (~$${round6(capUsd * participantCount)} total) before any participant aborts. Set execution.caps.maxTotalUsd for a shared study budget.`;
|
|
208
218
|
}
|
|
209
219
|
/** Builds and publishes the final bundle, runs the adapter hooks, renders the Observer and returns the result. */
|
|
210
220
|
export async function finishCuaRun(setup, finish, ran) {
|
|
211
|
-
const { plan, input,
|
|
221
|
+
const { plan, input, cwd, streams, descriptor, run } = setup;
|
|
212
222
|
const { participantRuns, participantPlan, scrubKnownValues, bundleBase } = setup;
|
|
213
|
-
const {
|
|
223
|
+
const { rerunLineage, publicRepo, subjectArgs } = finish;
|
|
214
224
|
const { dryRun } = plan;
|
|
215
|
-
const
|
|
225
|
+
const appUrl = plannedAppUrl(plan.runner.subject);
|
|
226
|
+
const subjectEnvNames = [...participantSubjectEnv(plan.runner.subject)];
|
|
216
227
|
const { runId } = run;
|
|
217
228
|
const participantCount = participantRuns.length;
|
|
218
229
|
const { outcomes, failFastReason, receiving, receivingWarnings, externalCommsWarnings } = ran;
|
|
219
|
-
// Per-
|
|
230
|
+
// Per-participant subject projections (invariant 5).
|
|
220
231
|
const subjects = projectParticipantSubjects({ ...subjectArgs, outcomes, dryRun });
|
|
221
232
|
const aggregate = aggregateCuaSubject({ subjects, outcomes, participantCount, dryRun });
|
|
222
233
|
const aggregateSubject = aggregate.subject;
|
|
223
|
-
const capWarning = participantCapWarning(
|
|
234
|
+
const capWarning = participantCapWarning(plan.caps, participantCount);
|
|
224
235
|
const aggregateWarnings = [
|
|
225
236
|
...externalCommsWarnings,
|
|
226
237
|
...(capWarning === undefined ? [] : [capWarning]),
|
|
@@ -239,8 +250,8 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
239
250
|
...(failFastReason === undefined ? {} : { failFastReason }),
|
|
240
251
|
});
|
|
241
252
|
const adapterWarnings = [];
|
|
242
|
-
const scorerResult = await
|
|
243
|
-
|
|
253
|
+
const scorerResult = await applyBrowserScorer({
|
|
254
|
+
scorer: input.scorer,
|
|
244
255
|
bundle,
|
|
245
256
|
context: {
|
|
246
257
|
bundle,
|
|
@@ -248,13 +259,12 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
248
259
|
labId: plan.labId,
|
|
249
260
|
runId,
|
|
250
261
|
actor: descriptor.id,
|
|
251
|
-
|
|
262
|
+
route: "computer-use",
|
|
252
263
|
dryRun,
|
|
253
|
-
|
|
264
|
+
participantCount,
|
|
254
265
|
},
|
|
255
266
|
sanitize: (text) => redactText(scrubKnownValues(text)),
|
|
256
267
|
warnings: adapterWarnings,
|
|
257
|
-
hookLabel: "cuaHooks",
|
|
258
268
|
...(input.scorerProvenance === undefined ? {} : { scorerProvenance: input.scorerProvenance }),
|
|
259
269
|
});
|
|
260
270
|
// The one final verdict fold: scoring can only make the judged verdict stricter.
|
|
@@ -264,7 +274,7 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
264
274
|
const finished = await run.finish(bundle);
|
|
265
275
|
const observer = await finished.renderObserver();
|
|
266
276
|
streams.attachFinal(observer);
|
|
267
|
-
const execution = judgeExecution(computerUseExecutionFailures(outcomes, observer), OUTCOME_POLICIES["computer-use"]);
|
|
277
|
+
const execution = judgeExecution(computerUseExecutionFailures(outcomes, observer, runId), OUTCOME_POLICIES["computer-use"]);
|
|
268
278
|
const result = cuaLabResult({
|
|
269
279
|
labId: plan.labId,
|
|
270
280
|
cwd,
|
|
@@ -11,7 +11,7 @@ import { type ComputerUseRunInput, type CuaActorLabResult, type RunCuaActorLabOp
|
|
|
11
11
|
export declare function runCuaActorLab(options: RunCuaActorLabOptions): Promise<CuaActorLabResult>;
|
|
12
12
|
/**
|
|
13
13
|
* A refused computer-use lab's result, at the refusal's stage: a before-scope refusal has its own
|
|
14
|
-
* envelope and no analysis record; the others come after the cwd checks, and the
|
|
14
|
+
* envelope and no analysis record; the others come after the cwd checks, and the participant cap after
|
|
15
15
|
* the personas are read.
|
|
16
16
|
*/
|
|
17
17
|
export declare function computerUseLabRefusal(options: RunCuaActorLabOptions, refusal: ComputerUseRefusal): Promise<CuaActorLabResult>;
|
|
@@ -26,8 +26,7 @@ export declare function admitComputerUsePlan(plan: ComputerUsePlan, input: Compu
|
|
|
26
26
|
* Run a computer-use plan: its local checks, then the run. The run scope in runAdmittedCuaRun
|
|
27
27
|
* finalizes any status record the run opened, on every exit, so a test or library caller does not
|
|
28
28
|
* leave the 5 s status cadence writing into a directory something else is deleting (an unrelated
|
|
29
|
-
* ENOTEMPTY). Participants
|
|
30
|
-
*
|
|
31
|
-
* per-participant cap) and by the compatibility hooks, which take the whole config.
|
|
29
|
+
* ENOTEMPTY). Participants, the brain, the subject, the caps and the residual config come from the
|
|
30
|
+
* plan. `config` is what the caller's createProvider and inProcess executor receive.
|
|
32
31
|
*/
|
|
33
32
|
export declare function runComputerUsePlan(plan: ComputerUsePlan, input: ComputerUseRunInput, config: LabConfig): Promise<CuaActorLabResult>;
|
|
@@ -26,10 +26,11 @@ import path from "node:path";
|
|
|
26
26
|
import { completeAutomaticAnalysis } from "../../analysis/automatic-completion.js";
|
|
27
27
|
import { runScope } from "../../run/run.js";
|
|
28
28
|
import { resolveAutomaticAnalysis } from "../../analysis/automatic-config.js";
|
|
29
|
-
import {
|
|
30
|
-
import {
|
|
29
|
+
import { browserRouteScorer } from "../../lab/adapter-scorer-loader.js";
|
|
30
|
+
import { withLateScorer } from "../../lab/route-inputs.js";
|
|
31
|
+
import { callerDrivingOf, planComputerUseLab } from "./plan.js";
|
|
31
32
|
import { finishCuaRun } from "./result.js";
|
|
32
|
-
import { runLabParticipants } from "./
|
|
33
|
+
import { runLabParticipants } from "./live-phase.js";
|
|
33
34
|
import { admitCuaRun, refuseCuaLab, startCuaRun } from "./setup.js";
|
|
34
35
|
import { CUA_ACTOR_LAB_SCHEMA, } from "./types.js";
|
|
35
36
|
/**
|
|
@@ -38,12 +39,12 @@ import { CUA_ACTOR_LAB_SCHEMA, } from "./types.js";
|
|
|
38
39
|
* run and its analysis.
|
|
39
40
|
*/
|
|
40
41
|
export async function runCuaActorLab(options) {
|
|
41
|
-
const { config, dryRun,
|
|
42
|
+
const { config, dryRun, ...input } = options;
|
|
42
43
|
// planComputerUseLab makes every configuration refusal, in the order this route always has.
|
|
43
44
|
const planned = planComputerUseLab(config, {
|
|
44
45
|
dryRun,
|
|
45
|
-
|
|
46
|
-
|
|
46
|
+
hasRunSession: input.deps?.runSession !== undefined,
|
|
47
|
+
driving: callerDrivingOf(input),
|
|
47
48
|
...(input.countOverride === undefined ? {} : { countOverride: input.countOverride }),
|
|
48
49
|
...(input.rerun === undefined ? {} : { rerun: input.rerun }),
|
|
49
50
|
});
|
|
@@ -53,7 +54,7 @@ export async function runCuaActorLab(options) {
|
|
|
53
54
|
}
|
|
54
55
|
/**
|
|
55
56
|
* A refused computer-use lab's result, at the refusal's stage: a before-scope refusal has its own
|
|
56
|
-
* envelope and no analysis record; the others come after the cwd checks, and the
|
|
57
|
+
* envelope and no analysis record; the others come after the cwd checks, and the participant cap after
|
|
57
58
|
* the personas are read.
|
|
58
59
|
*/
|
|
59
60
|
export async function computerUseLabRefusal(options, refusal) {
|
|
@@ -72,9 +73,9 @@ export async function computerUseLabRefusal(options, refusal) {
|
|
|
72
73
|
warnings: [],
|
|
73
74
|
error: { code: refusal.code, message: refusal.message },
|
|
74
75
|
};
|
|
75
|
-
// The other refusals come after the cwd checks, and the
|
|
76
|
+
// The other refusals come after the cwd checks, and the participant cap after the personas are read.
|
|
76
77
|
const analysis = resolveAutomaticAnalysis(config.review?.analysis);
|
|
77
|
-
return completeAutomaticAnalysis(await refuseCuaLab(options, refusal), undefined, analysis.ok ? analysis.config : undefined, options
|
|
78
|
+
return completeAutomaticAnalysis(await refuseCuaLab(options, refusal), undefined, analysis.ok ? analysis.config : undefined, options, { trigger: config.review?.analysis === undefined ? "default" : "explicit" });
|
|
78
79
|
}
|
|
79
80
|
/**
|
|
80
81
|
* runLab's step for a computer-use plan. It runs the plan's local checks (admitCuaRun: the
|
|
@@ -92,24 +93,22 @@ export async function admitComputerUsePlan(plan, input, config) {
|
|
|
92
93
|
return {
|
|
93
94
|
ok: true,
|
|
94
95
|
run: async (scorer) => {
|
|
95
|
-
|
|
96
|
-
//
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
return cuaOutcome(await runAdmittedCuaRun(plan, running, admitted));
|
|
96
|
+
// The scorer joins the input only; the runSession, provider and desktop the checks admitted
|
|
97
|
+
// stay the ones the run uses.
|
|
98
|
+
const running = withLateScorer(input, scorer, browserRouteScorer);
|
|
99
|
+
return cuaOutcome(await runAdmittedCuaRun(plan, running, admission.admitted));
|
|
100
100
|
},
|
|
101
101
|
};
|
|
102
102
|
}
|
|
103
103
|
function cuaOutcome(result) {
|
|
104
|
-
return { route: "computer-use",
|
|
104
|
+
return { route: "computer-use", result };
|
|
105
105
|
}
|
|
106
106
|
/**
|
|
107
107
|
* Run a computer-use plan: its local checks, then the run. The run scope in runAdmittedCuaRun
|
|
108
108
|
* finalizes any status record the run opened, on every exit, so a test or library caller does not
|
|
109
109
|
* leave the 5 s status cadence writing into a directory something else is deleting (an unrelated
|
|
110
|
-
* ENOTEMPTY). Participants
|
|
111
|
-
*
|
|
112
|
-
* per-participant cap) and by the compatibility hooks, which take the whole config.
|
|
110
|
+
* ENOTEMPTY). Participants, the brain, the subject, the caps and the residual config come from the
|
|
111
|
+
* plan. `config` is what the caller's createProvider and inProcess executor receive.
|
|
113
112
|
*/
|
|
114
113
|
export async function runComputerUsePlan(plan, input, config) {
|
|
115
114
|
const admission = await admitCuaRun(plan, input, config);
|
|
@@ -129,9 +128,12 @@ function runAdmittedCuaRun(plan, input, admitted) {
|
|
|
129
128
|
});
|
|
130
129
|
}
|
|
131
130
|
function completeCuaAnalysis(plan, input, result, finished) {
|
|
132
|
-
|
|
131
|
+
// A local VM study whose cleanup is unconfirmed records a skip instead of analyzing.
|
|
132
|
+
const refusal = input.localVm?.analysisRefusal;
|
|
133
|
+
return completeAutomaticAnalysis(result, finished, plan.analysis?.config, input, {
|
|
133
134
|
...(plan.analysis === undefined ? {} : { trigger: plan.analysis.trigger }),
|
|
134
135
|
preferLargerOutput: plan.analysis?.preferLargerOutput === true,
|
|
136
|
+
...(refusal === undefined ? {} : { refusal }),
|
|
135
137
|
});
|
|
136
138
|
}
|
|
137
139
|
async function runPlanInScope(plan, input, admitted, scope) {
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import type { CuaLoopResult } from "../../actors/computer-use/loop.js";
|
|
2
2
|
import { type SessionEnding } from "../../run/judge.js";
|
|
3
3
|
/**
|
|
4
|
-
* A goal_satisfied
|
|
4
|
+
* A goal_satisfied participant counts as a self-reported blocker ONLY when its final narrative contradicts
|
|
5
5
|
* the goal AND the run's own stop predicate did NOT fire. A matched stopWhen is independent,
|
|
6
6
|
* structured completion evidence, so it overrides a text scan of the free-form narrative — which can
|
|
7
7
|
* otherwise trip on the subject app's OWN quoted copy (e.g. a relayed "cannot be undone" banner).
|
|
8
8
|
* Resolved-arc segments never block the verdict (#453). Returns the offending reason, or undefined
|
|
9
|
-
* when the
|
|
9
|
+
* when the participant is a clean pass. Exported for testing.
|
|
10
10
|
*/
|
|
11
11
|
export declare function resolveSelfReportedBlocker(session: CuaLoopResult | undefined): string | undefined;
|
|
12
12
|
/** A session's ending, reduced to what the judge's engagement and blocker rules read. */
|
|
@@ -20,7 +20,7 @@ function hasBlockerLanguage(text) {
|
|
|
20
20
|
// Report-shaped language: what a participant writes when it finished AND has something to say.
|
|
21
21
|
// Every one of the day's eleven drawDB reports (2026-09-01) opened a section "What confused me"
|
|
22
22
|
// or "Accessibility defects:"; none contained a blocker word, so none became a feedback candidate
|
|
23
|
-
// and a
|
|
23
|
+
// and a participant that had just replicated a keyboard-accessibility defect three times drafted "Live
|
|
24
24
|
// study completed without a participant-reported finding". Friction is the INCLUSIVE scan; a
|
|
25
25
|
// false positive here adds a candidate a person then reads, which is the cheap direction.
|
|
26
26
|
const REPORTED_DEFECT_LANGUAGE = /\b(defects?|bugs?|accessibilit(y|ies)|inaccessible|not (keyboard|screen.?reader)[- ]?accessible|confus(ed|ing)|hesitat(ed|ion)|unexpected(ly)?|unclear|hard to (find|tell|see|read|reach)|no (visible )?focus|overlap(ped|ping|s)?|truncat(ed|es|ion)|cut off|did nothing|nothing happened|no effect)\b/;
|
|
@@ -94,7 +94,7 @@ function completionReasonBlocksVerdict(reason) {
|
|
|
94
94
|
// with a parser error. A simpler SQL import succeeded."). The lookahead demands the retry flavor
|
|
95
95
|
// on purpose: unrelated praise ("Separately, the search box worked") must never launder an
|
|
96
96
|
// unresolved failure. "Login failed so I gave up" has no recovery anywhere and stays a blocker.
|
|
97
|
-
// (#453 — the run-1 false negative: a defect report after demonstrated success failed the
|
|
97
|
+
// (#453 — the run-1 false negative: a defect report after demonstrated success failed the participant,
|
|
98
98
|
// an incentive inversion against exactly the participant behavior a study wants most.)
|
|
99
99
|
const RESOLUTION_TERMS = /\b(succeed(?:ed|s)?|success(?:ful|fully)?|worked|works around|then worked|now works?|resolved|fixed|recovered|got it working|went through)\b/;
|
|
100
100
|
const RETRY_RESOLUTION = /\b(simpler|simplified|retry(?:ing)?|retried|second (?:attempt|try)|another (?:attempt|try|approach)|different (?:approach|way|route)|instead|then|eventually|after that)\b[^.!?\n]{0,80}\b(succeed(?:ed|s)?|success(?:ful|fully)?|worked|went through|completed|passed)\b/;
|
|
@@ -152,12 +152,12 @@ function traceHasStopWhenMatch(session) {
|
|
|
152
152
|
(item.title.startsWith("stopWhen matched") || item.title === "dwell window complete"));
|
|
153
153
|
}
|
|
154
154
|
/**
|
|
155
|
-
* A goal_satisfied
|
|
155
|
+
* A goal_satisfied participant counts as a self-reported blocker ONLY when its final narrative contradicts
|
|
156
156
|
* the goal AND the run's own stop predicate did NOT fire. A matched stopWhen is independent,
|
|
157
157
|
* structured completion evidence, so it overrides a text scan of the free-form narrative — which can
|
|
158
158
|
* otherwise trip on the subject app's OWN quoted copy (e.g. a relayed "cannot be undone" banner).
|
|
159
159
|
* Resolved-arc segments never block the verdict (#453). Returns the offending reason, or undefined
|
|
160
|
-
* when the
|
|
160
|
+
* when the participant is a clean pass. Exported for testing.
|
|
161
161
|
*/
|
|
162
162
|
export function resolveSelfReportedBlocker(session) {
|
|
163
163
|
return session !== undefined && selfReportedBlocker(blockerFacts(session))
|
|
@@ -3,15 +3,15 @@ import type { CuaActorDescriptor } from "../../actors/registry.js";
|
|
|
3
3
|
import type { LabCommsEmail, LabCommsExternal, LabConfig } from "../../lab/types.js";
|
|
4
4
|
import { type RunRerunLineage } from "../../run/bundle.js";
|
|
5
5
|
import { type LocalTreeArchive } from "../../subject/local-tree-archive.js";
|
|
6
|
-
import { type ComputerUseRefusal
|
|
6
|
+
import { type ComputerUseRefusal } from "./plan.js";
|
|
7
7
|
import type { ComputerUsePlan } from "../../lab/plan-types.js";
|
|
8
8
|
import { trackRuntimeStreams, type LiveTraceFlush } from "./live-flush.js";
|
|
9
9
|
import { type CuaRunBundleBase } from "./bundle.js";
|
|
10
10
|
import { projectParticipantSubjects } from "./subject-projection.js";
|
|
11
|
-
import { type CuaActorLabErrorCode, type
|
|
11
|
+
import { type CuaActorLabErrorCode, type CuaActorLabResult, type CuaParticipantDeps, type CuaParticipantPlan, type DesktopParticipantRun, type CuaSubjectProjection, type CuaSubjectProvenanceArg, type ComputerUseRunInput, type RunCuaActorLabOptions } from "./types.js";
|
|
12
12
|
/**
|
|
13
|
-
* The envelope of a refusal the plan left for after the cwd checks. The
|
|
14
|
-
* fan-out refusals come after the committed personas are read, so a persona-file error still wins.
|
|
13
|
+
* The envelope of a refusal the plan left for after the cwd checks. The participant cap and
|
|
14
|
+
* in-process fan-out refusals come after the committed personas are read, so a persona-file error still wins.
|
|
15
15
|
*/
|
|
16
16
|
export declare function refuseCuaLab(options: RunCuaActorLabOptions, refusal: ComputerUseRefusal): Promise<CuaActorLabResult>;
|
|
17
17
|
export type AdmittedCuaRun = Extract<Awaited<ReturnType<typeof admitCuaRun>>, {
|
|
@@ -24,13 +24,9 @@ type StartedRun = Extract<Awaited<ReturnType<RunScope["startRun"]>>, {
|
|
|
24
24
|
export interface CuaRunSetup {
|
|
25
25
|
plan: ComputerUsePlan;
|
|
26
26
|
input: ComputerUseRunInput;
|
|
27
|
-
/** Read for values the plan does not carry yet, and by the compatibility hooks, which take the
|
|
28
|
-
* whole config. Participants come from the plan. */
|
|
29
|
-
config: LabConfig;
|
|
30
27
|
/** The physical project root. */
|
|
31
28
|
cwd: string;
|
|
32
29
|
descriptor: CuaActorDescriptor;
|
|
33
|
-
subjectRoute: CuaSubjectRoute;
|
|
34
30
|
run: StartedRun;
|
|
35
31
|
streams: ReturnType<typeof trackRuntimeStreams>;
|
|
36
32
|
participantRuns: DesktopParticipantRun[];
|
|
@@ -66,7 +62,6 @@ export interface CuaParticipantsSetup {
|
|
|
66
62
|
}
|
|
67
63
|
/** What only finishCuaRun reads besides the participants' outcomes. */
|
|
68
64
|
export interface CuaFinishFacts {
|
|
69
|
-
hooks: CuaActorLabHooks;
|
|
70
65
|
rerunLineage: RunRerunLineage | undefined;
|
|
71
66
|
publicRepo: string | undefined;
|
|
72
67
|
subjectArgs: Omit<Parameters<typeof projectParticipantSubjects>[0], "outcomes" | "dryRun">;
|
|
@@ -95,16 +90,16 @@ export declare function admitCuaRun(plan: ComputerUsePlan, input: ComputerUseRun
|
|
|
95
90
|
config: LabConfig;
|
|
96
91
|
dryRun: boolean;
|
|
97
92
|
cwd: string;
|
|
98
|
-
|
|
93
|
+
seams: import("../../lab/lab-deps.js").LabDeps;
|
|
99
94
|
streams: {
|
|
100
|
-
|
|
95
|
+
onStream: NonNullable<import("../../lab/run-lab-homes.js").RunLabHomes["onStream"]>;
|
|
101
96
|
showIn: (observer: import("../../index.js").ObserverResult & {
|
|
102
97
|
ok: true;
|
|
103
98
|
}) => void;
|
|
104
99
|
attachFinal: (observer: import("../../index.js").ObserverResult) => void;
|
|
105
100
|
};
|
|
106
|
-
env:
|
|
107
|
-
|
|
101
|
+
env: Record<string, string | undefined>;
|
|
102
|
+
appUrl: string;
|
|
108
103
|
fail: (code: CuaActorLabErrorCode, message: string, actorLabel?: string) => CuaActorLabResult;
|
|
109
104
|
refuse: (code: CuaActorLabErrorCode, message: string, actorLabel?: string | undefined) => {
|
|
110
105
|
ok: false;
|
|
@@ -123,7 +118,6 @@ export declare function admitCuaRun(plan: ComputerUsePlan, input: ComputerUseRun
|
|
|
123
118
|
knownSecretValues: string[];
|
|
124
119
|
scrubKnownValues: (text: string) => string;
|
|
125
120
|
publicRepo: string | undefined;
|
|
126
|
-
hasGithubToken: boolean;
|
|
127
121
|
localTreeArchive: LocalTreeArchive | undefined;
|
|
128
122
|
localTreeArchiveBuffer: ArrayBuffer | undefined;
|
|
129
123
|
};
|