humanish 0.107.0 → 0.109.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +73 -69
- package/CHANGELOG.md +27 -0
- package/CONTEXT.md +51 -44
- package/README.md +148 -175
- package/SECURITY.md +16 -26
- package/TELEMETRY.md +8 -6
- package/dist/actors/affordance.d.ts +4 -4
- package/dist/actors/affordance.js +12 -12
- package/dist/actors/agent-session.d.ts +2 -2
- package/dist/actors/agent-session.js +7 -7
- package/dist/actors/codex/app-server-actor-trace.d.ts +1 -1
- package/dist/actors/codex/app-server-actor-trace.js +1 -1
- package/dist/actors/codex/app-server-trace.js +2 -1
- package/dist/actors/codex/app-server-ui.js +1 -1
- package/dist/actors/codex/codex-admission.d.ts +91 -0
- package/dist/actors/codex/codex-admission.js +150 -0
- package/dist/actors/codex/restricted-executable.d.ts +4 -4
- package/dist/actors/codex/restricted-executable.js +8 -8
- package/dist/actors/codex/restricted-launch.d.ts +2 -1
- package/dist/actors/codex/restricted-launch.js +1 -1
- package/dist/actors/codex/restricted-participant.js +20 -18
- package/dist/actors/codex/restricted-policy.d.ts +2 -2
- package/dist/actors/codex/restricted-policy.js +2 -2
- package/dist/actors/codex/restricted-session.d.ts +3 -3
- package/dist/actors/codex/restricted-session.js +10 -5
- package/dist/actors/computer-use/actor.d.ts +18 -18
- package/dist/actors/computer-use/actor.js +4 -4
- package/dist/actors/computer-use/admission-limit.d.ts +3 -3
- package/dist/actors/computer-use/admission-limit.js +4 -4
- package/dist/actors/computer-use/executor-error.d.ts +2 -2
- package/dist/actors/computer-use/executor-error.js +3 -3
- package/dist/actors/computer-use/loop/actions.d.ts +3 -3
- package/dist/actors/computer-use/loop/actions.js +15 -15
- package/dist/actors/computer-use/loop/backstop.d.ts +3 -3
- package/dist/actors/computer-use/loop/backstop.js +14 -12
- package/dist/actors/computer-use/loop/debrief.js +2 -2
- package/dist/actors/computer-use/loop/ending.d.ts +6 -6
- package/dist/actors/computer-use/loop/ending.js +18 -17
- package/dist/actors/computer-use/loop/observation.d.ts +2 -2
- package/dist/actors/computer-use/loop/observation.js +15 -14
- package/dist/actors/computer-use/loop/provider-call.d.ts +1 -1
- package/dist/actors/computer-use/loop/provider-call.js +12 -12
- package/dist/actors/computer-use/loop/session.d.ts +4 -4
- package/dist/actors/computer-use/loop/session.js +4 -4
- package/dist/actors/computer-use/loop/spend.d.ts +1 -1
- package/dist/actors/computer-use/loop/spend.js +1 -1
- package/dist/actors/computer-use/loop/trace.js +11 -9
- package/dist/actors/computer-use/loop/types.d.ts +47 -45
- package/dist/actors/computer-use/loop/usage.d.ts +2 -2
- package/dist/actors/computer-use/loop/usage.js +2 -2
- package/dist/actors/computer-use/loop.d.ts +6 -1
- package/dist/actors/computer-use/loop.js +26 -4
- package/dist/actors/computer-use/openai-context.d.ts +37 -0
- package/dist/actors/computer-use/openai-context.js +214 -0
- package/dist/actors/computer-use/openai-provider.d.ts +13 -10
- package/dist/actors/computer-use/openai-provider.js +52 -41
- package/dist/actors/computer-use/openai-requests.d.ts +36 -10
- package/dist/actors/computer-use/openai-requests.js +82 -15
- package/dist/actors/computer-use/openai-wire.d.ts +19 -7
- package/dist/actors/computer-use/openai-wire.js +36 -34
- package/dist/actors/computer-use/provider-error.d.ts +4 -4
- package/dist/actors/computer-use/provider-error.js +6 -6
- package/dist/actors/contract.d.ts +82 -47
- package/dist/actors/contract.js +30 -24
- package/dist/actors/local-agent/claude-session.d.ts +1 -1
- package/dist/actors/local-agent/claude-session.js +13 -12
- package/dist/actors/local-agent/cli.d.ts +9 -7
- package/dist/actors/local-agent/cli.js +24 -28
- package/dist/actors/local-agent/readiness.d.ts +2 -2
- package/dist/actors/local-agent/readiness.js +4 -4
- package/dist/actors/reasoning-effort.js +8 -8
- package/dist/actors/registry.d.ts +12 -12
- package/dist/actors/registry.js +21 -20
- package/dist/actors/scripted-browser/actor.d.ts +10 -10
- package/dist/actors/scripted-browser/actor.js +17 -17
- package/dist/actors/scripted-browser/browser-command.js +1 -1
- package/dist/actors/scripted-browser/steps.js +1 -1
- package/dist/actors/scripted-browser/types.d.ts +1 -1
- package/dist/actors/stop-conditions.d.ts +1 -1
- package/dist/analysis/automatic-completion.d.ts +7 -7
- package/dist/analysis/automatic-completion.js +1 -1
- package/dist/analysis/automatic-config.d.ts +6 -6
- package/dist/analysis/automatic-config.js +1 -1
- package/dist/analysis/codex-config.d.ts +6 -2
- package/dist/analysis/codex-config.js +22 -14
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +1 -1
- package/dist/analysis/service.js +5 -5
- package/dist/analysis/types.d.ts +6 -0
- package/dist/browser-control/client.js +16 -16
- package/dist/browser-control/dispatcher.js +4 -4
- package/dist/browser-control/protocol.js +14 -14
- package/dist/browser-control/recording-transfer.js +4 -4
- package/dist/browser-control/transport.js +7 -7
- package/dist/cli/commands/analysis-signals.d.ts +2 -2
- package/dist/cli/commands/analyze.js +19 -16
- package/dist/cli/commands/codex.js +9 -9
- package/dist/cli/commands/comms.js +48 -35
- package/dist/cli/commands/feedback.js +20 -21
- package/dist/cli/commands/migrate.d.ts +4 -0
- package/dist/cli/commands/migrate.js +65 -0
- package/dist/cli/commands/observe.d.ts +0 -1
- package/dist/cli/commands/observe.js +136 -112
- package/dist/cli/commands/run-command.d.ts +10 -0
- package/dist/cli/commands/run-command.js +85 -0
- package/dist/cli/commands/runs.js +136 -139
- package/dist/cli/commands/setup.d.ts +2 -2
- package/dist/cli/commands/setup.js +27 -24
- package/dist/cli/commands/study-format.d.ts +9 -0
- package/dist/cli/commands/{lab-format.js → study-format.js} +39 -27
- package/dist/cli/commands/{lab-route-computer-use.d.ts → study-route-computer-use.d.ts} +6 -6
- package/dist/cli/commands/{lab-route-computer-use.js → study-route-computer-use.js} +17 -18
- package/dist/cli/commands/{lab-route-open.d.ts → study-route-open.d.ts} +4 -4
- package/dist/cli/commands/{lab-route-open.js → study-route-open.js} +2 -2
- package/dist/cli/commands/{lab-route-preview.d.ts → study-route-preview.d.ts} +7 -7
- package/dist/cli/commands/{lab-route-preview.js → study-route-preview.js} +7 -6
- package/dist/cli/commands/study-route-run.d.ts +19 -0
- package/dist/cli/commands/{lab-route-run.js → study-route-run.js} +6 -6
- package/dist/cli/commands/study-route-scripted.d.ts +17 -0
- package/dist/cli/commands/{lab-route-scripted.js → study-route-scripted.js} +4 -4
- package/dist/cli/commands/{lab-route-shared-world.d.ts → study-route-shared-world.d.ts} +5 -5
- package/dist/cli/commands/{lab-route-shared-world.js → study-route-shared-world.js} +10 -10
- package/dist/cli/commands/study-route-terminal.d.ts +17 -0
- package/dist/cli/commands/{lab-route-terminal.js → study-route-terminal.js} +4 -4
- package/dist/cli/commands/study-run.d.ts +9 -0
- package/dist/cli/commands/{lab-run.js → study-run.js} +44 -45
- package/dist/cli/commands/{lab-scorer.d.ts → study-scorer.d.ts} +7 -7
- package/dist/cli/commands/{lab-scorer.js → study-scorer.js} +2 -2
- package/dist/cli/commands/study.d.ts +3 -0
- package/dist/cli/commands/study.js +146 -0
- package/dist/cli/commands/tui.d.ts +4 -4
- package/dist/cli/commands/tui.js +31 -31
- package/dist/cli/commands/watch.js +71 -121
- package/dist/cli/deprecations.d.ts +6 -0
- package/dist/cli/deprecations.js +18 -0
- package/dist/cli/doctor.d.ts +18 -8
- package/dist/cli/doctor.js +114 -59
- package/dist/cli/first-run-path.d.ts +15 -13
- package/dist/cli/first-run-path.js +35 -33
- package/dist/cli/io.d.ts +44 -9
- package/dist/cli/io.js +98 -23
- package/dist/cli/observer-follow.d.ts +3 -3
- package/dist/cli/observer-follow.js +6 -7
- package/dist/cli/orientation.d.ts +10 -7
- package/dist/cli/orientation.js +89 -47
- package/dist/cli/product-sentence.d.ts +6 -0
- package/dist/cli/product-sentence.js +6 -0
- package/dist/cli/program.d.ts +6 -1
- package/dist/cli/program.js +111 -43
- package/dist/cli/telemetry.d.ts +16 -16
- package/dist/cli/telemetry.js +46 -44
- package/dist/cli.js +1 -1
- package/dist/comms/capture-surface.d.ts +13 -12
- package/dist/comms/capture-surface.js +23 -22
- package/dist/comms/catch-host.d.ts +7 -7
- package/dist/comms/catch-host.js +17 -17
- package/dist/comms/connections.d.ts +1 -1
- package/dist/comms/connections.js +2 -2
- package/dist/comms/email-catch.d.ts +3 -3
- package/dist/comms/email-catch.js +8 -8
- package/dist/comms/external-evidence.d.ts +5 -5
- package/dist/comms/external-evidence.js +5 -4
- package/dist/comms/extract.js +2 -2
- package/dist/comms/fake-inbox.d.ts +3 -3
- package/dist/comms/fake-inbox.js +3 -3
- package/dist/comms/receiving-render.d.ts +1 -1
- package/dist/comms/receiving-render.js +2 -2
- package/dist/comms/receiving-runtime.d.ts +4 -4
- package/dist/comms/sandbox-catch-script.d.ts +6 -6
- package/dist/comms/sandbox-catch-script.js +6 -6
- package/dist/comms/sandbox-catch.d.ts +38 -39
- package/dist/comms/sandbox-catch.js +36 -37
- package/dist/comms/setup.d.ts +2 -2
- package/dist/comms/setup.js +56 -25
- package/dist/comms/thread-evidence.d.ts +1 -1
- package/dist/comms/thread-evidence.js +6 -6
- package/dist/comms/types.d.ts +6 -6
- package/dist/comms/types.js +3 -3
- package/dist/evidence/frame-signature.d.ts +1 -1
- package/dist/evidence/frame-signature.js +4 -4
- package/dist/evidence/redaction.d.ts +3 -3
- package/dist/evidence/redaction.js +4 -4
- package/dist/feedback/draft.js +38 -80
- package/dist/feedback/export-bundle.js +1 -1
- package/dist/feedback/export.d.ts +5 -1
- package/dist/feedback/export.js +10 -10
- package/dist/feedback/feedback.d.ts +2 -2
- package/dist/feedback/feedback.js +21 -3
- package/dist/feedback/proof.d.ts +1 -1
- package/dist/feedback/proof.js +2 -2
- package/dist/guest/bootstrap.js +6 -6
- package/dist/guest/browser-tools.js +8 -8
- package/dist/guest/chromium-text.js +19 -19
- package/dist/guest/desktop-executor.d.ts +2 -2
- package/dist/guest/desktop-executor.js +20 -20
- package/dist/guest/desktop-keys.js +4 -4
- package/dist/guest/desktop-media.d.ts +2 -2
- package/dist/guest/desktop-media.js +16 -16
- package/dist/guest/desktop-native.d.ts +1 -1
- package/dist/guest/desktop-native.js +10 -10
- package/dist/guest/runtime-desktop.js +2 -2
- package/dist/guest/runtime-processes.d.ts +2 -2
- package/dist/guest/runtime-processes.js +2 -2
- package/dist/guest/runtime.js +6 -6
- package/dist/index.d.ts +14 -14
- package/dist/index.js +8 -8
- package/dist/keys/key-resolution.d.ts +5 -5
- package/dist/keys/key-resolution.js +23 -23
- package/dist/observer/artifact.d.ts +2 -2
- package/dist/observer/artifact.js +7 -7
- package/dist/observer/data.d.ts +6 -6
- package/dist/observer/data.js +5 -4
- package/dist/observer/exposure.d.ts +1 -1
- package/dist/observer/exposure.js +11 -11
- package/dist/observer/http.js +1 -1
- package/dist/observer/library.d.ts +3 -1
- package/dist/observer/library.js +8 -3
- package/dist/observer/listen.js +5 -4
- package/dist/observer/render.d.ts +1 -1
- package/dist/observer/render.js +17 -6
- package/dist/observer/run-routes.d.ts +4 -3
- package/dist/observer/run-routes.js +38 -13
- package/dist/observer/serve.d.ts +1 -1
- package/dist/observer/serve.js +7 -3
- package/dist/observer/tunnel.js +1 -1
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle-parts.d.ts +9 -9
- package/dist/routes/computer-use/bundle-parts.js +18 -17
- package/dist/routes/computer-use/bundle.d.ts +2 -2
- package/dist/routes/computer-use/bundle.js +10 -9
- package/dist/routes/computer-use/e2b-desktop/comms.d.ts +6 -6
- package/dist/routes/computer-use/e2b-desktop/comms.js +18 -17
- package/dist/routes/computer-use/e2b-desktop/desktop.js +4 -4
- package/dist/routes/computer-use/e2b-desktop/fidelity.d.ts +5 -4
- package/dist/routes/computer-use/e2b-desktop/fidelity.js +11 -10
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.js +10 -10
- package/dist/routes/computer-use/e2b-desktop/start.js +7 -7
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +2 -2
- package/dist/routes/computer-use/e2b-desktop/teardown.js +2 -2
- package/dist/routes/computer-use/external-comms.d.ts +4 -4
- package/dist/routes/computer-use/external-comms.js +1 -1
- package/dist/routes/computer-use/fanout-bundle.js +27 -26
- package/dist/routes/computer-use/fanout-records.js +12 -13
- package/dist/routes/computer-use/live-flush.d.ts +4 -4
- package/dist/routes/computer-use/live-flush.js +6 -6
- package/dist/routes/computer-use/live-phase.d.ts +5 -4
- package/dist/routes/computer-use/live-phase.js +8 -8
- package/dist/routes/computer-use/local-tree-pack.d.ts +5 -5
- package/dist/routes/computer-use/local-tree-pack.js +1 -1
- package/dist/routes/computer-use/local-vm.d.ts +8 -8
- package/dist/routes/computer-use/local-vm.js +3 -3
- package/dist/routes/computer-use/participant-desktop.d.ts +7 -7
- package/dist/routes/computer-use/participant-desktop.js +5 -5
- package/dist/routes/computer-use/participant-execution.d.ts +4 -4
- package/dist/routes/computer-use/participant-execution.js +10 -10
- package/dist/routes/computer-use/participant-model.d.ts +6 -0
- package/dist/routes/computer-use/participant-model.js +33 -15
- package/dist/routes/computer-use/participant-prompt.d.ts +15 -15
- package/dist/routes/computer-use/participant-prompt.js +9 -9
- package/dist/routes/computer-use/participant-runs.d.ts +16 -16
- package/dist/routes/computer-use/participant-runs.js +26 -25
- package/dist/routes/computer-use/plan.d.ts +12 -12
- package/dist/routes/computer-use/plan.js +42 -42
- package/dist/routes/computer-use/preflight.d.ts +5 -5
- package/dist/routes/computer-use/preflight.js +25 -25
- package/dist/routes/computer-use/rerun-selection.d.ts +1 -1
- package/dist/routes/computer-use/rerun-selection.js +4 -3
- package/dist/routes/computer-use/result.d.ts +4 -4
- package/dist/routes/computer-use/result.js +22 -21
- package/dist/routes/computer-use/route.d.ts +12 -11
- package/dist/routes/computer-use/route.js +31 -30
- package/dist/routes/computer-use/self-report.d.ts +7 -7
- package/dist/routes/computer-use/self-report.js +32 -32
- package/dist/routes/computer-use/setup.d.ts +23 -23
- package/dist/routes/computer-use/setup.js +28 -29
- package/dist/routes/computer-use/single-bundle.d.ts +11 -11
- package/dist/routes/computer-use/single-bundle.js +40 -41
- package/dist/routes/computer-use/subject-projection.d.ts +4 -4
- package/dist/routes/computer-use/subject-projection.js +6 -6
- package/dist/routes/computer-use/types.d.ts +71 -73
- package/dist/routes/computer-use/types.js +12 -12
- package/dist/routes/preview.d.ts +13 -7
- package/dist/routes/preview.js +11 -7
- package/dist/routes/scripted/bundle.d.ts +7 -7
- package/dist/routes/scripted/bundle.js +20 -18
- package/dist/routes/scripted/plan.d.ts +8 -8
- package/dist/routes/scripted/plan.js +13 -13
- package/dist/routes/scripted/result.d.ts +3 -3
- package/dist/routes/scripted/result.js +13 -13
- package/dist/routes/scripted/route.d.ts +9 -9
- package/dist/routes/scripted/route.js +15 -16
- package/dist/routes/scripted/scenario.d.ts +1 -1
- package/dist/routes/scripted/scenario.js +5 -5
- package/dist/routes/scripted/setup.d.ts +6 -6
- package/dist/routes/scripted/setup.js +8 -8
- package/dist/routes/scripted/subject.d.ts +5 -5
- package/dist/routes/scripted/subject.js +4 -4
- package/dist/routes/scripted/surface-records.d.ts +1 -1
- package/dist/routes/scripted/surface-records.js +4 -4
- package/dist/routes/scripted/surface-sessions.d.ts +3 -3
- package/dist/routes/scripted/types.d.ts +18 -20
- package/dist/routes/scripted/types.js +2 -2
- package/dist/routes/shared-world/bundle-records.js +11 -10
- package/dist/routes/shared-world/bundle.d.ts +5 -5
- package/dist/routes/shared-world/bundle.js +46 -43
- package/dist/routes/shared-world/checkpoints.d.ts +4 -4
- package/dist/routes/shared-world/checkpoints.js +2 -2
- package/dist/routes/shared-world/comms.d.ts +14 -14
- package/dist/routes/shared-world/comms.js +19 -18
- package/dist/routes/shared-world/external-public.d.ts +4 -3
- package/dist/routes/shared-world/external-public.js +16 -12
- package/dist/routes/shared-world/handoff.d.ts +7 -6
- package/dist/routes/shared-world/handoff.js +65 -57
- package/dist/routes/shared-world/lobby-code.d.ts +2 -2
- package/dist/routes/shared-world/lobby-code.js +9 -9
- package/dist/routes/shared-world/participant-specs.d.ts +10 -9
- package/dist/routes/shared-world/participant-specs.js +38 -31
- package/dist/routes/shared-world/plan.d.ts +9 -9
- package/dist/routes/shared-world/plan.js +16 -16
- package/dist/routes/shared-world/provenance.d.ts +2 -2
- package/dist/routes/shared-world/provenance.js +2 -2
- package/dist/routes/shared-world/provisioned.d.ts +8 -8
- package/dist/routes/shared-world/provisioned.js +18 -17
- package/dist/routes/shared-world/result.d.ts +6 -6
- package/dist/routes/shared-world/result.js +29 -29
- package/dist/routes/shared-world/route.d.ts +11 -11
- package/dist/routes/shared-world/route.js +40 -39
- package/dist/routes/shared-world/setup.d.ts +16 -16
- package/dist/routes/shared-world/setup.js +23 -23
- package/dist/routes/shared-world/types.d.ts +55 -53
- package/dist/routes/shared-world/types.js +4 -4
- package/dist/routes/terminal/adapter.d.ts +5 -5
- package/dist/routes/terminal/adapter.js +9 -9
- package/dist/routes/terminal/bundle.d.ts +13 -13
- package/dist/routes/terminal/bundle.js +28 -27
- package/dist/routes/terminal/credentials.d.ts +8 -8
- package/dist/routes/terminal/credentials.js +12 -12
- package/dist/routes/terminal/dry-run.d.ts +4 -4
- package/dist/routes/terminal/dry-run.js +12 -12
- package/dist/routes/terminal/encoding.js +2 -2
- package/dist/routes/terminal/ledger.d.ts +5 -5
- package/dist/routes/terminal/ledger.js +14 -14
- package/dist/routes/terminal/lifetime.d.ts +1 -1
- package/dist/routes/terminal/lifetime.js +1 -1
- package/dist/routes/terminal/live-finish.d.ts +2 -2
- package/dist/routes/terminal/live-finish.js +10 -10
- package/dist/routes/terminal/live-sandbox.d.ts +2 -2
- package/dist/routes/terminal/live-sandbox.js +15 -15
- package/dist/routes/terminal/participant-text.js +2 -1
- package/dist/routes/terminal/persona.d.ts +2 -2
- package/dist/routes/terminal/persona.js +3 -3
- package/dist/routes/terminal/plan.d.ts +7 -7
- package/dist/routes/terminal/plan.js +15 -15
- package/dist/routes/terminal/recorder.js +2 -2
- package/dist/routes/terminal/result.d.ts +5 -5
- package/dist/routes/terminal/result.js +14 -14
- package/dist/routes/terminal/route.d.ts +9 -9
- package/dist/routes/terminal/route.js +19 -20
- package/dist/routes/terminal/sandbox.d.ts +1 -1
- package/dist/routes/terminal/sandbox.js +1 -1
- package/dist/routes/terminal/session.d.ts +5 -5
- package/dist/routes/terminal/session.js +14 -13
- package/dist/routes/terminal/token-usage.d.ts +1 -1
- package/dist/routes/terminal/token-usage.js +7 -8
- package/dist/routes/terminal/trace.d.ts +4 -4
- package/dist/routes/terminal/trace.js +2 -2
- package/dist/routes/terminal/types.d.ts +48 -50
- package/dist/routes/terminal/types.js +5 -5
- package/dist/run/bundle-shape.js +3 -3
- package/dist/run/bundle.d.ts +49 -49
- package/dist/run/bundle.js +3 -2
- package/dist/run/cost-summary.js +3 -3
- package/dist/run/costs.d.ts +23 -0
- package/dist/run/costs.js +79 -44
- package/dist/run/detail.d.ts +6 -6
- package/dist/run/detail.js +4 -4
- package/dist/run/dry-run-inputs.d.ts +1 -1
- package/dist/run/dry-run-inputs.js +2 -2
- package/dist/run/dry-run.js +20 -20
- package/dist/run/judge.d.ts +19 -13
- package/dist/run/judge.js +18 -8
- package/dist/run/outcomes.d.ts +1 -1
- package/dist/run/participant.d.ts +1 -1
- package/dist/run/paths.d.ts +1 -1
- package/dist/run/paths.js +1 -1
- package/dist/run/preflight-receipts.js +1 -1
- package/dist/run/pricing.d.ts +11 -6
- package/dist/run/pricing.js +22 -15
- package/dist/run/projection.d.ts +46 -46
- package/dist/run/projection.js +57 -57
- package/dist/run/reclaim.d.ts +1 -1
- package/dist/run/reclaim.js +3 -3
- package/dist/run/results.d.ts +5 -5
- package/dist/run/run-cost.d.ts +53 -0
- package/dist/run/run-cost.js +105 -0
- package/dist/run/run-index-analysis.d.ts +15 -0
- package/dist/run/run-index-analysis.js +77 -0
- package/dist/run/run-index.d.ts +19 -7
- package/dist/run/run-index.js +77 -64
- package/dist/run/run-not-found.d.ts +5 -0
- package/dist/run/run-not-found.js +15 -0
- package/dist/run/run.d.ts +6 -5
- package/dist/run/run.js +3 -3
- package/dist/run/sandbox-receipts.d.ts +2 -2
- package/dist/run/sandbox-receipts.js +3 -3
- package/dist/run/shared-world-evidence.d.ts +37 -36
- package/dist/run/shared-world-evidence.js +4 -4
- package/dist/run/shared-world-shape.d.ts +1 -1
- package/dist/run/shared-world-shape.js +7 -7
- package/dist/run/stats.d.ts +14 -8
- package/dist/run/stats.js +121 -63
- package/dist/run/status.d.ts +16 -29
- package/dist/run/status.js +10 -29
- package/dist/run/stored-runs.js +2 -1
- package/dist/run/stream-shape.js +1 -1
- package/dist/run/streams.d.ts +7 -7
- package/dist/run/study-provenance.d.ts +49 -0
- package/dist/run/study-provenance.js +66 -0
- package/dist/run/study-result.d.ts +11 -0
- package/dist/run/study-result.js +9 -0
- package/dist/run/text.d.ts +2 -0
- package/dist/run/text.js +4 -0
- package/dist/run-study.d.ts +112 -0
- package/dist/{run-lab.js → run-study.js} +52 -47
- package/dist/{lab → study}/adapter-extension.d.ts +15 -27
- package/dist/{lab → study}/adapter-extension.js +10 -45
- package/dist/{lab → study}/adapter-scorer-loader.d.ts +17 -17
- package/dist/{lab → study}/adapter-scorer-loader.js +34 -34
- package/dist/study/composition-rules.d.ts +7 -0
- package/dist/{lab → study}/composition-rules.js +53 -53
- package/dist/study/config.d.ts +15 -0
- package/dist/{lab → study}/config.js +98 -41
- package/dist/study/convert.d.ts +33 -0
- package/dist/study/convert.js +432 -0
- package/dist/{lab → study}/device-presets.d.ts +10 -10
- package/dist/{lab → study}/device-presets.js +11 -11
- package/dist/study/discover.d.ts +75 -0
- package/dist/{lab → study}/discover.js +134 -95
- package/dist/{lab → study}/doctor.d.ts +19 -12
- package/dist/{lab → study}/doctor.js +58 -20
- package/dist/study/field-paths.d.ts +12 -0
- package/dist/study/field-paths.js +126 -0
- package/dist/study/files.d.ts +26 -0
- package/dist/study/files.js +44 -0
- package/dist/{lab → study}/init-templates.d.ts +2 -2
- package/dist/study/init-templates.js +421 -0
- package/dist/{lab → study}/init.d.ts +2 -2
- package/dist/{lab → study}/init.js +46 -11
- package/dist/study/keys.d.ts +4 -0
- package/dist/{lab → study}/keys.js +46 -12
- package/dist/study/migrate.d.ts +48 -0
- package/dist/study/migrate.js +408 -0
- package/dist/study/owned-files.d.ts +38 -0
- package/dist/study/owned-files.js +182 -0
- package/dist/{lab → study}/parse/actors.d.ts +5 -5
- package/dist/{lab → study}/parse/actors.js +14 -14
- package/dist/study/parse/comms.d.ts +12 -0
- package/dist/{lab → study}/parse/comms.js +5 -5
- package/dist/study/parse/execution.d.ts +19 -0
- package/dist/{lab → study}/parse/execution.js +13 -13
- package/dist/study/parse/study-v3.d.ts +21 -0
- package/dist/study/parse/study-v3.js +259 -0
- package/dist/study/parse/subject-state.d.ts +30 -0
- package/dist/{lab → study}/parse/subject-state.js +19 -19
- package/dist/{lab → study}/parse/subject.d.ts +3 -3
- package/dist/{lab → study}/parse/subject.js +34 -34
- package/dist/{lab → study}/parse/values.d.ts +3 -3
- package/dist/{lab → study}/parse/values.js +3 -3
- package/dist/{lab → study}/persona-resolve.d.ts +8 -8
- package/dist/{lab → study}/persona-resolve.js +6 -6
- package/dist/{lab → study}/plan-base.d.ts +7 -7
- package/dist/{lab → study}/plan-base.js +1 -1
- package/dist/{lab → study}/plan-participants.d.ts +12 -12
- package/dist/{lab → study}/plan-participants.js +4 -4
- package/dist/{lab → study}/plan-types.d.ts +31 -31
- package/dist/{lab → study}/plan-types.js +2 -2
- package/dist/study/plan.d.ts +13 -0
- package/dist/study/plan.js +101 -0
- package/dist/study/preflight-probes.d.ts +3 -0
- package/dist/{lab → study}/preflight-probes.js +20 -19
- package/dist/study/preflight-result.d.ts +8 -0
- package/dist/{lab → study}/preflight-result.js +5 -4
- package/dist/study/preflight.d.ts +94 -0
- package/dist/{lab → study}/preflight.js +12 -12
- package/dist/{lab → study}/requirements.d.ts +3 -3
- package/dist/{lab → study}/requirements.js +1 -1
- package/dist/study/route-inputs.d.ts +28 -0
- package/dist/{lab → study}/route-inputs.js +1 -1
- package/dist/{lab → study}/routing.d.ts +39 -33
- package/dist/{lab → study}/routing.js +60 -29
- package/dist/{lab/run-lab-events.d.ts → study/run-study-events.d.ts} +10 -13
- package/dist/{lab/run-lab-events.js → study/run-study-events.js} +3 -9
- package/dist/{lab/run-lab-homes.d.ts → study/run-study-homes.d.ts} +21 -18
- package/dist/{lab/run-lab-homes.js → study/run-study-homes.js} +1 -1
- package/dist/study/run-study-options.d.ts +28 -0
- package/dist/{lab/run-lab-options.js → study/run-study-options.js} +29 -25
- package/dist/{lab/lab-deps.d.ts → study/study-deps.d.ts} +8 -13
- package/dist/{lab/lab-deps.js → study/study-deps.js} +2 -2
- package/dist/{lab → study}/summary.d.ts +16 -15
- package/dist/{lab → study}/summary.js +21 -21
- package/dist/{lab → study}/tasks.d.ts +18 -18
- package/dist/{lab → study}/tasks.js +14 -14
- package/dist/{lab → study}/types.d.ts +202 -196
- package/dist/{lab → study}/types.js +3 -1
- package/dist/{lab → study}/validation.d.ts +27 -27
- package/dist/{lab → study}/validation.js +53 -51
- package/dist/study/warnings.d.ts +12 -0
- package/dist/{lab → study}/warnings.js +62 -34
- package/dist/subject/clone.d.ts +7 -7
- package/dist/subject/clone.js +2 -2
- package/dist/subject/desktop-cli.d.ts +1 -1
- package/dist/subject/desktop-cli.js +2 -2
- package/dist/subject/local-tree-archive.d.ts +2 -2
- package/dist/subject/local-tree-archive.js +4 -4
- package/dist/subject/local-tree.d.ts +4 -4
- package/dist/subject/node-bootstrap.js +1 -1
- package/dist/subject/runtime.js +9 -10
- package/dist/subject/serve.d.ts +5 -5
- package/dist/subject/serve.js +3 -3
- package/dist/subject/state.d.ts +4 -4
- package/dist/subject/state.js +1 -1
- package/dist/subject/steps.d.ts +6 -6
- package/dist/subject/steps.js +5 -5
- package/dist/substrates/command-failure.d.ts +3 -3
- package/dist/substrates/command-failure.js +3 -3
- package/dist/substrates/desktop-session.js +2 -2
- package/dist/substrates/detached.d.ts +6 -6
- package/dist/substrates/detached.js +7 -7
- package/dist/substrates/e2b/cdp-probe.d.ts +9 -9
- package/dist/substrates/e2b/cdp-probe.js +14 -17
- package/dist/substrates/e2b/desktop-browser.d.ts +6 -6
- package/dist/substrates/e2b/desktop-browser.js +9 -9
- package/dist/substrates/e2b/desktop-cdp.d.ts +10 -10
- package/dist/substrates/e2b/desktop-cdp.js +9 -9
- package/dist/substrates/e2b/desktop-executor.d.ts +2 -2
- package/dist/substrates/e2b/desktop-executor.js +13 -13
- package/dist/substrates/e2b/desktop-geometry.d.ts +4 -4
- package/dist/substrates/e2b/desktop-geometry.js +8 -8
- package/dist/substrates/e2b/desktop-media.d.ts +4 -4
- package/dist/substrates/e2b/desktop-media.js +1 -1
- package/dist/substrates/e2b/desktop-screenshot-cleanup.d.ts +3 -3
- package/dist/substrates/e2b/desktop-screenshot-cleanup.js +3 -3
- package/dist/substrates/e2b/lifetime.js +1 -1
- package/dist/substrates/e2b/sandbox.d.ts +4 -4
- package/dist/substrates/e2b/sandbox.js +2 -2
- package/dist/substrates/e2b/sdk.d.ts +12 -12
- package/dist/substrates/e2b/sdk.js +7 -7
- package/dist/substrates/local/runtime-config.d.ts +4 -4
- package/dist/substrates/local/runtime-config.js +4 -4
- package/dist/tui/actions.d.ts +1 -1
- package/dist/tui/actions.js +13 -13
- package/dist/tui/contract.d.ts +23 -23
- package/dist/tui/contract.js +11 -11
- package/dist/tui/launch.d.ts +7 -7
- package/dist/tui/launch.js +36 -31
- package/dist/tui/project.js +5 -5
- package/dist/tui-app.js +139 -139
- package/dist/verify/actor.d.ts +8 -8
- package/dist/verify/actor.js +36 -36
- package/dist/verify/artifacts.d.ts +2 -2
- package/dist/verify/artifacts.js +3 -3
- package/dist/verify/context-growth.d.ts +9 -0
- package/dist/verify/context-growth.js +62 -0
- package/dist/verify/costs.d.ts +6 -6
- package/dist/verify/costs.js +8 -8
- package/dist/verify/shared-world-concurrent.d.ts +2 -2
- package/dist/verify/shared-world-concurrent.js +79 -76
- package/dist/verify/shared-world.d.ts +2 -2
- package/dist/verify/shared-world.js +16 -16
- package/dist/verify/subject.d.ts +5 -5
- package/dist/verify/subject.js +19 -19
- package/dist/verify/verify.js +94 -44
- package/docs/README.md +25 -32
- package/docs/architecture/actor-contract.md +60 -91
- package/docs/architecture/browser-control.md +1 -1
- package/docs/architecture/desktop-sessions.md +7 -7
- package/docs/architecture/external-public-shared-world.md +68 -65
- package/docs/architecture/github-feedback-loop.md +2 -5
- package/docs/architecture/guest-desktop.md +2 -2
- package/docs/architecture/observer.md +34 -34
- package/docs/architecture/restricted-codex-analysis.md +34 -28
- package/docs/architecture/serve.md +34 -37
- package/docs/architecture/state-driven-executor.md +55 -58
- package/docs/architecture/task-protocol-support.md +6 -6
- package/docs/architecture/terminal-product-route.md +70 -110
- package/docs/contracts/README.md +24 -0
- package/docs/contracts/adapter-admission.md +3 -3
- package/docs/contracts/adapter-fixtures.md +4 -7
- package/docs/contracts/core.md +1 -3
- package/docs/contracts/errors.md +62 -0
- package/docs/contracts/feedback.md +8 -9
- package/docs/contracts/policy.md +7 -12
- package/docs/contracts/run-bundle.md +42 -44
- package/docs/contracts/schemas.md +446 -368
- package/docs/contracts/study-analysis.md +7 -4
- package/docs/contracts/study-costs.md +28 -9
- package/docs/decisions/0001-run-bundle-is-the-source-of-truth.md +1 -1
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +2 -2
- package/docs/decisions/0003-contained-paths-and-cleanup-authority.md +3 -3
- package/docs/principles/actor-fidelity.md +4 -4
- package/docs/principles/engineering.md +3 -3
- package/docs/principles/invariants-and-defaults.md +38 -38
- package/docs/principles/self-driving-harness.md +1 -5
- package/docs/principles/three-roles.md +1 -1
- package/docs/product/automatic-analysis.md +7 -7
- package/examples/README.md +1 -1
- package/examples/participant/README.md +5 -5
- package/examples/participant/run.mjs +11 -10
- package/examples/scorer/README.md +3 -3
- package/examples/scorer/run.mjs +11 -12
- package/examples/scorer/scorer.mjs +3 -3
- package/package.json +16 -17
- package/skills/humanish/SKILL.md +64 -60
- package/skills/humanish/agents/openai.yaml +1 -1
- package/AGENTS.md +0 -99
- package/CONTRIBUTING.md +0 -148
- package/dist/actors/codex/qualified-versions.d.ts +0 -72
- package/dist/actors/codex/qualified-versions.js +0 -132
- package/dist/cli/commands/lab-format.d.ts +0 -8
- package/dist/cli/commands/lab-route-run.d.ts +0 -19
- package/dist/cli/commands/lab-route-scripted.d.ts +0 -17
- package/dist/cli/commands/lab-route-terminal.d.ts +0 -17
- package/dist/cli/commands/lab-run.d.ts +0 -9
- package/dist/cli/commands/lab.d.ts +0 -3
- package/dist/cli/commands/lab.js +0 -199
- package/dist/cli/renamed-options.d.ts +0 -11
- package/dist/cli/renamed-options.js +0 -18
- package/dist/lab/composition-rules.d.ts +0 -7
- package/dist/lab/config.d.ts +0 -7
- package/dist/lab/discover.d.ts +0 -65
- package/dist/lab/init-templates.js +0 -416
- package/dist/lab/keys.d.ts +0 -2
- package/dist/lab/parse/comms.d.ts +0 -12
- package/dist/lab/parse/execution.d.ts +0 -19
- package/dist/lab/parse/subject-state.d.ts +0 -30
- package/dist/lab/plan.d.ts +0 -19
- package/dist/lab/plan.js +0 -132
- package/dist/lab/preflight-probes.d.ts +0 -3
- package/dist/lab/preflight-result.d.ts +0 -7
- package/dist/lab/preflight.d.ts +0 -92
- package/dist/lab/route-inputs.d.ts +0 -28
- package/dist/lab/run-lab-options.d.ts +0 -28
- package/dist/lab/warnings.d.ts +0 -2
- package/dist/observer/static.d.ts +0 -39
- package/dist/observer/static.js +0 -369
- package/dist/run-lab.d.ts +0 -112
- package/docs/architecture/comms-inbox.md +0 -122
- package/docs/architecture/desktop-recording.md +0 -80
- package/docs/architecture/local-browser-runtime.md +0 -178
- package/docs/architecture/observer-review.md +0 -144
- package/docs/architecture/participant-media.md +0 -90
- package/docs/architecture/project-layout.md +0 -62
- package/docs/architecture/real-email-receiving.md +0 -136
- package/docs/assets/humanish-drawdb-hero.png +0 -0
- package/docs/assets/humanish-observer-hero.png +0 -0
- package/docs/goals/current.md +0 -207
- package/docs/product/lobby-trivia-3player-external-public.md +0 -77
- package/docs/product/open-source-install-experience.md +0 -240
- package/docs/ramp/README.md +0 -295
- package/docs/release/open-source-readiness.md +0 -222
- package/docs/release/public-readiness-standard.md +0 -210
- package/docs/roadmap/world-class-open-source-v0.md +0 -311
- /package/dist/{lab → study}/init-paths.d.ts +0 -0
- /package/dist/{lab → study}/init-paths.js +0 -0
- /package/dist/{lab → study}/init-plan.d.ts +0 -0
- /package/dist/{lab → study}/init-plan.js +0 -0
- /package/dist/{lab → study}/participant-assignment.d.ts +0 -0
- /package/dist/{lab → study}/participant-assignment.js +0 -0
- /package/dist/{lab → study}/persona.d.ts +0 -0
- /package/dist/{lab → study}/persona.js +0 -0
|
@@ -4,9 +4,9 @@ import { actionFingerprint, isIdleTurn } from "./actions.js";
|
|
|
4
4
|
// streak (turns that neither change the UI state nor try something new). It is a pure fold over
|
|
5
5
|
// turns, never a turn budget.
|
|
6
6
|
const IDLE_PROGRESS_FORGIVENESS_STEPS = 2;
|
|
7
|
-
/** How many recent turns the repetition check looks back over
|
|
7
|
+
/** How many recent turns the repetition check looks back over. */
|
|
8
8
|
const ACTION_REPEAT_WINDOW = 3;
|
|
9
|
-
/** Pixels of vertical scroll per progress bucket
|
|
9
|
+
/** Pixels of vertical scroll per progress bucket: a real scroll step (typically >=100px)
|
|
10
10
|
* crosses a bucket and counts as progress; sub-bucket jiggle does not, so an actor nudging the
|
|
11
11
|
* same dead panel cannot stay "progressing" forever. */
|
|
12
12
|
const SCROLL_PROGRESS_BUCKET_PX = 200;
|
|
@@ -22,10 +22,10 @@ const STABLE_KEY_MAX_TOTAL = 8192;
|
|
|
22
22
|
/**
|
|
23
23
|
* A deterministic, bounded, sorted-key projection of an appState object, used as the friction
|
|
24
24
|
* loop's progress key. Two structurally-equal states (regardless of key insertion order) map
|
|
25
|
-
* to the
|
|
25
|
+
* to the same string, so key reordering can never fabricate a progress delta; two different
|
|
26
26
|
* states map to different strings (within the caps).
|
|
27
27
|
*
|
|
28
|
-
* Correctness
|
|
28
|
+
* Correctness depends on this: it must not throw on a cyclic or huge input. Cycles are detected
|
|
29
29
|
* with a seen-set (a back-edge degrades to the marker "[Circular]"); depth, key count, array
|
|
30
30
|
* length, string length, and total output length are all capped so an adversarial or merely
|
|
31
31
|
* large appState degrades to a bounded value rather than crashing the loop. Pure: it never
|
|
@@ -107,8 +107,8 @@ function progressKeyOf(observation) {
|
|
|
107
107
|
const base = observation.appState !== undefined
|
|
108
108
|
? stableProgressKey(observation.appState)
|
|
109
109
|
: observation.stateSignature;
|
|
110
|
-
// Scroll position is state
|
|
111
|
-
// the participant genuinely advances, so the offset rides the key
|
|
110
|
+
// Scroll position is state: a scroll-pinned section keeps the frame hash constant while
|
|
111
|
+
// the participant genuinely advances, so the offset rides the key, bucketed, never raw.
|
|
112
112
|
return observation.scrollY === undefined
|
|
113
113
|
? base
|
|
114
114
|
: `${base}#s${Math.round(observation.scrollY / SCROLL_PROGRESS_BUCKET_PX)}`;
|
|
@@ -127,23 +127,23 @@ export function advanceBackstop(previous, turn, limits) {
|
|
|
127
127
|
const idle = isIdleTurn(turn.actions);
|
|
128
128
|
const progressKey = progressKeyOf(turn.observation);
|
|
129
129
|
const frameChanged = progressKey !== previous.progressKey;
|
|
130
|
-
//
|
|
130
|
+
// Corroboration. A stale frame alone is not evidence of a stuck agent. The frame hash is
|
|
131
131
|
// a coarse whole-screen measure, and on a light-themed web app it can miss a renamed row, a new
|
|
132
|
-
// list item, or an opened panel
|
|
132
|
+
// list item, or an opened panel: a measured run had 9 visibly different consecutive frames hash
|
|
133
133
|
// identically while the agent was a foreign key away from finishing. Ending a participant on that
|
|
134
134
|
// signal alone recorded working sessions as `gave_up`, capping every browser run at roughly
|
|
135
135
|
// noProgressSteps turns and writing harness artifacts into evidence as actor behavior.
|
|
136
136
|
//
|
|
137
|
-
// So a no-progress turn now requires
|
|
137
|
+
// So a no-progress turn now requires both a stale frame and the agent repeating something it
|
|
138
138
|
// just tried. An agent doing varied work is never counted stuck, however blind the hash is;
|
|
139
|
-
// an agent re-clicking the same dead control trips it as fast as it did before
|
|
139
|
+
// an agent re-clicking the same dead control trips it as fast as it did before, arguably
|
|
140
140
|
// faster, since that is the actual signature of being stuck.
|
|
141
141
|
const fingerprint = actionFingerprint(turn.actions);
|
|
142
142
|
const repeatingRecentAction = fingerprint.length > 0 && previous.recentFingerprints.includes(fingerprint);
|
|
143
143
|
const recentFingerprints = [...previous.recentFingerprints, fingerprint].slice(-ACTION_REPEAT_WINDOW);
|
|
144
|
-
// Corroboration governs the
|
|
144
|
+
// Corroboration governs the frame-staleness backstop only. The idle backstop below is a direct
|
|
145
145
|
// behavioral signal already (the agent took nothing but screenshots and waits), so it keeps
|
|
146
|
-
// reading the frame on its own
|
|
146
|
+
// reading the frame on its own: a repeated screenshot is exactly what an idle streak is, and
|
|
147
147
|
// feeding repetition into it would grant an extra forgiveness step for being idle.
|
|
148
148
|
const observedProgress = frameChanged || turn.heardNewSpeech;
|
|
149
149
|
const progressed = observedProgress || !repeatingRecentAction;
|
|
@@ -172,11 +172,13 @@ export function advanceBackstop(previous, turn, limits) {
|
|
|
172
172
|
const hints = [];
|
|
173
173
|
if (consecutiveNoProgress >= noProgressRecoverySteps &&
|
|
174
174
|
consecutiveNoProgress < limits.noProgressSteps) {
|
|
175
|
+
// prose-check: model prompt (the participant model reads this hint, not a person)
|
|
175
176
|
hints.push(`No visible progress for ${consecutiveNoProgress} step(s). ` +
|
|
176
177
|
"If your task calls for waiting for another participant or a pending transition, you may continue waiting. " +
|
|
177
178
|
"Choose whether to continue or stop based on your situation and what you observe.");
|
|
178
179
|
}
|
|
179
180
|
if (consecutiveIdle >= idleRecoverySteps && consecutiveIdle < limits.idleSteps) {
|
|
181
|
+
// prose-check: model prompt (the participant model reads this hint, not a person)
|
|
180
182
|
hints.push(`You are only waiting or taking screenshots for ${consecutiveIdle} step(s). ` +
|
|
181
183
|
"If your task calls for waiting, you may continue within the remaining session time. " +
|
|
182
184
|
"When the relevant controls become actionable, continue your task; describe any blocker you actually encounter.");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { isComputerUseAdmissionLimitError } from "../admission-limit.js";
|
|
2
2
|
import { singleDispatch } from "./provider-call.js";
|
|
3
3
|
import { CuaDeadlineError, raceSessionDeadline, requestScope } from "./race.js";
|
|
4
4
|
import { notice } from "./trace.js";
|
|
@@ -110,7 +110,7 @@ async function debriefExchange(session, provider, context, trigger, boundMs, rec
|
|
|
110
110
|
}
|
|
111
111
|
catch (error) {
|
|
112
112
|
// A failed optional report cannot rewrite the already observed structured completion.
|
|
113
|
-
if (
|
|
113
|
+
if (isComputerUseAdmissionLimitError(error)) {
|
|
114
114
|
return record("skipped", "the adapter reported a local admission limit before provider dispatch; no closing request was sent");
|
|
115
115
|
}
|
|
116
116
|
const detail = signal?.aborted
|
|
@@ -18,12 +18,12 @@ export interface Stop {
|
|
|
18
18
|
readonly debriefTrigger?: DebriefTrigger;
|
|
19
19
|
}
|
|
20
20
|
/**
|
|
21
|
-
* Read the fixed closing line the prompt asks for
|
|
21
|
+
* Read the fixed closing line the prompt asks for: the first non-empty line of the
|
|
22
22
|
* participant's last message, exactly one of three phrases, punctuation and case forgiven. Anything
|
|
23
23
|
* else is absence, never a guess. Exported for tests.
|
|
24
24
|
*/
|
|
25
25
|
export declare function declaredOutcomeFromClosingLine(message: string | undefined): ParticipantDeclaredOutcome | undefined;
|
|
26
|
-
/** The participant's own word for how it ended: a schema field first, then the closing line
|
|
26
|
+
/** The participant's own word for how it ended: a schema field first, then the closing line. */
|
|
27
27
|
export declare function declaredOutcomeOf(turn: CuaTurn): ParticipantDeclaredOutcome | undefined;
|
|
28
28
|
/**
|
|
29
29
|
* The participant reported a natural endpoint. "not_reached" is a participant who stopped without
|
|
@@ -35,8 +35,8 @@ export declare function participantEnded(turn: CuaTurn, declaredOutcome: Partici
|
|
|
35
35
|
export declare const harnessAborted: Stop;
|
|
36
36
|
/**
|
|
37
37
|
* The wall-clock deadline. A session that took at least one material (non-idle) action before the
|
|
38
|
-
* cap reached its
|
|
39
|
-
*
|
|
38
|
+
* cap reached its budget rather than stalling; a deadline hit with zero material actions is still
|
|
39
|
+
* recorded as a failure (timed_out).
|
|
40
40
|
*/
|
|
41
41
|
export declare function timeLimit(session: LoopSession): Stop;
|
|
42
42
|
/**
|
|
@@ -67,9 +67,9 @@ export declare function providerStalledTwice(turnNumber: number, afterMs: number
|
|
|
67
67
|
export declare function providerInterrupted(interruption: NonNullable<CuaTurn["interruption"]>): Stop;
|
|
68
68
|
/**
|
|
69
69
|
* Safety-check categories are provider-defined enums (e.g. "malicious_instructions"), not free
|
|
70
|
-
* text; the evidence records them (redacted for defense-in-depth) so it shows
|
|
70
|
+
* text; the evidence records them (redacted for defense-in-depth) so it shows why the run paused.
|
|
71
71
|
*/
|
|
72
|
-
export declare function blockedOnSafetyChecks(checks: string): Stop;
|
|
72
|
+
export declare function blockedOnSafetyChecks(checks: string, count: number): Stop;
|
|
73
73
|
/**
|
|
74
74
|
* A vision provider against a screenshot-less observation is a fail-closed harness error, not a
|
|
75
75
|
* silent crash. The provider sets requiresFrame; a state-reasoning provider omits it.
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { isComputerUseExecutorError } from "../executor-error.js";
|
|
2
|
+
import { isComputerUsePromptRefusedError, isComputerUseProviderError } from "../provider-error.js";
|
|
3
3
|
import { CuaAbortError, CuaDeadlineError } from "./race.js";
|
|
4
4
|
import { notice } from "./trace.js";
|
|
5
|
+
import { plural } from "../../../run/text.js";
|
|
5
6
|
/**
|
|
6
|
-
* Read the fixed closing line the prompt asks for
|
|
7
|
+
* Read the fixed closing line the prompt asks for: the first non-empty line of the
|
|
7
8
|
* participant's last message, exactly one of three phrases, punctuation and case forgiven. Anything
|
|
8
9
|
* else is absence, never a guess. Exported for tests.
|
|
9
10
|
*/
|
|
@@ -25,7 +26,7 @@ export function declaredOutcomeFromClosingLine(message) {
|
|
|
25
26
|
return "blocked";
|
|
26
27
|
return undefined;
|
|
27
28
|
}
|
|
28
|
-
/** The participant's own word for how it ended: a schema field first, then the closing line
|
|
29
|
+
/** The participant's own word for how it ended: a schema field first, then the closing line. */
|
|
29
30
|
export function declaredOutcomeOf(turn) {
|
|
30
31
|
return turn.outcome ?? declaredOutcomeFromClosingLine(turn.message);
|
|
31
32
|
}
|
|
@@ -51,8 +52,8 @@ export const harnessAborted = {
|
|
|
51
52
|
};
|
|
52
53
|
/**
|
|
53
54
|
* The wall-clock deadline. A session that took at least one material (non-idle) action before the
|
|
54
|
-
* cap reached its
|
|
55
|
-
*
|
|
55
|
+
* cap reached its budget rather than stalling; a deadline hit with zero material actions is still
|
|
56
|
+
* recorded as a failure (timed_out).
|
|
56
57
|
*/
|
|
57
58
|
export function timeLimit(session) {
|
|
58
59
|
const { timeoutMs } = session;
|
|
@@ -67,8 +68,8 @@ export function timeLimit(session) {
|
|
|
67
68
|
return {
|
|
68
69
|
completionReason: "budget_reached",
|
|
69
70
|
reason: session.actionHistory.interruptedActionOutcome
|
|
70
|
-
? `reached the ${timeoutMs}ms time budget with ${materialActions
|
|
71
|
-
: `reached the ${timeoutMs}ms time budget after productive activity (${materialActions
|
|
71
|
+
? `reached the ${timeoutMs}ms time budget with ${plural(materialActions, "material action attempt")}, ${plural(turns, "turn")}; the latest action outcome is uncertain`
|
|
72
|
+
: `reached the ${timeoutMs}ms time budget after productive activity (${plural(materialActions, "material action")}, ${plural(turns, "turn")})`,
|
|
72
73
|
stopCause: "time_limit",
|
|
73
74
|
};
|
|
74
75
|
}
|
|
@@ -82,12 +83,12 @@ export function spendLimit(session, estimate, maxUsd) {
|
|
|
82
83
|
const booked = session.usage.reservedRequests;
|
|
83
84
|
const spend = booked === 0
|
|
84
85
|
? `estimated spend $${estimate}`
|
|
85
|
-
: `estimated spend $${estimate}, including the worst case of ${booked
|
|
86
|
+
: `estimated spend $${estimate}, including the worst case of ${plural(booked, "lost request")},`;
|
|
86
87
|
return {
|
|
87
88
|
completionReason: "budget_reached",
|
|
88
89
|
reason: materialActions > 0
|
|
89
|
-
?
|
|
90
|
-
:
|
|
90
|
+
? `Stopped before the next model turn: ${spend} passed execution.caps.maxUsd=$${maxUsd} after productive activity (${plural(materialActions, "material action")}, ${plural(turns, "turn")})`
|
|
91
|
+
: `Stopped before the next model turn: ${spend} passed execution.caps.maxUsd=$${maxUsd} with no material progress`,
|
|
91
92
|
stopCause: "spend_limit",
|
|
92
93
|
};
|
|
93
94
|
}
|
|
@@ -178,12 +179,12 @@ export function providerInterrupted(interruption) {
|
|
|
178
179
|
}
|
|
179
180
|
/**
|
|
180
181
|
* Safety-check categories are provider-defined enums (e.g. "malicious_instructions"), not free
|
|
181
|
-
* text; the evidence records them (redacted for defense-in-depth) so it shows
|
|
182
|
+
* text; the evidence records them (redacted for defense-in-depth) so it shows why the run paused.
|
|
182
183
|
*/
|
|
183
|
-
export function blockedOnSafetyChecks(checks) {
|
|
184
|
+
export function blockedOnSafetyChecks(checks, count) {
|
|
184
185
|
return {
|
|
185
186
|
completionReason: "blocked_approval",
|
|
186
|
-
reason: `paused on model safety check
|
|
187
|
+
reason: `paused on ${count === 1 ? "a model safety check" : `${count} model safety checks`}: ${checks}; not acknowledged`,
|
|
187
188
|
evidence: {
|
|
188
189
|
kind: "approval",
|
|
189
190
|
body: () => ({ lifecycle: "completed", status: "blocked", title: `safety check: ${checks}` }),
|
|
@@ -258,7 +259,7 @@ export function stopForError(session, error) {
|
|
|
258
259
|
const { lastActionTitle } = session.actionHistory;
|
|
259
260
|
const lastAction = () => lastActionTitle === undefined ? undefined : `last action: ${redact(lastActionTitle)}`;
|
|
260
261
|
const screenshot = session.lastScreenshotRef === undefined ? {} : { screenshotRef: session.lastScreenshotRef };
|
|
261
|
-
if (
|
|
262
|
+
if (isComputerUsePromptRefusedError(error)) {
|
|
262
263
|
// The provider's policy decision about the prompt, not a harness fault; never resent.
|
|
263
264
|
const reason = `${error.message}; the prompt was not sent again`;
|
|
264
265
|
return {
|
|
@@ -268,7 +269,7 @@ export function stopForError(session, error) {
|
|
|
268
269
|
evidence: noticeEvidence("error", "provider refused the prompt", `phase: ${redact(session.phase)}; ${reason}`),
|
|
269
270
|
};
|
|
270
271
|
}
|
|
271
|
-
if (
|
|
272
|
+
if (isComputerUseProviderError(error)) {
|
|
272
273
|
const reason = `participant provider error: ${error.code}${error.failurePhase ? ` during ${error.failurePhase}` : ""}; cleanup: ${error.receipt.cleanup}`;
|
|
273
274
|
return {
|
|
274
275
|
completionReason: "harness_error",
|
|
@@ -276,7 +277,7 @@ export function stopForError(session, error) {
|
|
|
276
277
|
evidence: noticeEvidence("error", "participant provider error", reason),
|
|
277
278
|
};
|
|
278
279
|
}
|
|
279
|
-
if (
|
|
280
|
+
if (isComputerUseExecutorError(error)) {
|
|
280
281
|
const detail = () => [
|
|
281
282
|
`phase: ${redact(session.phase)}`,
|
|
282
283
|
`code: ${error.code}`,
|
|
@@ -23,8 +23,8 @@ export declare class DesktopObserver {
|
|
|
23
23
|
checkpoint(turnNumber: number): Promise<Checkpoint>;
|
|
24
24
|
/**
|
|
25
25
|
* Hand an observation the next turn may react to to the runtime hooks, and note whether it
|
|
26
|
-
* carried app state. Runtime-only: the
|
|
27
|
-
* and is never persisted.
|
|
26
|
+
* carried app state. Runtime-only: the participant's live location.href goes back to the
|
|
27
|
+
* orchestrator and is never persisted.
|
|
28
28
|
*/
|
|
29
29
|
private handToHooks;
|
|
30
30
|
/**
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { evaluateStopWhen } from "../../stop-conditions.js";
|
|
2
|
-
import {
|
|
2
|
+
import { ComputerUseExecutorError, isComputerUseExecutorError } from "../executor-error.js";
|
|
3
3
|
import { CUA_SPEECH_LIMITS } from "../speech.js";
|
|
4
4
|
import { dwellCompleted, missingFrame, stopWhenMatched } from "./ending.js";
|
|
5
5
|
import { CuaAbortError, CuaStallError, raceCallBound } from "./race.js";
|
|
6
6
|
import { notice } from "./trace.js";
|
|
7
|
-
|
|
7
|
+
import { plural } from "../../../run/text.js";
|
|
8
|
+
// One projection feeds both the stop guard and the task tracker, so a criterion that would stop
|
|
8
9
|
// the run and a criterion that completes a task can never see different evidence for one turn.
|
|
9
10
|
function stopObservationOf(observation) {
|
|
10
11
|
return {
|
|
@@ -72,8 +73,8 @@ export class DesktopObserver {
|
|
|
72
73
|
}
|
|
73
74
|
/**
|
|
74
75
|
* Hand an observation the next turn may react to to the runtime hooks, and note whether it
|
|
75
|
-
* carried app state. Runtime-only: the
|
|
76
|
-
* and is never persisted.
|
|
76
|
+
* carried app state. Runtime-only: the participant's live location.href goes back to the
|
|
77
|
+
* orchestrator and is never persisted.
|
|
77
78
|
*/
|
|
78
79
|
handToHooks(observation) {
|
|
79
80
|
const { onObservedUrl, onScreenshot } = this.session.settings;
|
|
@@ -114,11 +115,11 @@ export class DesktopObserver {
|
|
|
114
115
|
this.observeTasks(final, turnNumber);
|
|
115
116
|
}
|
|
116
117
|
catch (error) {
|
|
117
|
-
if (
|
|
118
|
+
if (isComputerUseExecutorError(error))
|
|
118
119
|
throw error;
|
|
119
120
|
}
|
|
120
121
|
}
|
|
121
|
-
// An executor without stallRecovery "fail_closed" gets one retry of a stalled observe
|
|
122
|
+
// An executor without stallRecovery "fail_closed" gets one retry of a stalled observe.
|
|
122
123
|
// One that cannot safely replay a pending request opts out, so even a shorter outer bound stops
|
|
123
124
|
// without retrying.
|
|
124
125
|
async observeBounded(label) {
|
|
@@ -133,7 +134,7 @@ export class DesktopObserver {
|
|
|
133
134
|
if (executor.stallRecovery === "fail_closed") {
|
|
134
135
|
// The outer bound says nothing about whether the still-pending request completed.
|
|
135
136
|
// The owning session will close it; this loop must not send a replacement request.
|
|
136
|
-
throw new
|
|
137
|
+
throw new ComputerUseExecutorError("deadline_exceeded", "outcome_uncertain");
|
|
137
138
|
}
|
|
138
139
|
session.trace.record("notice", () => notice("warn", "observation stalled; retrying once", `${error.what} produced nothing within ${error.afterMs}ms; asking the desktop again`));
|
|
139
140
|
return await raceCallBound(`observe (${label}, retry)`, executor.observe(), session.remaining(), session.observationTimeoutMs, session.signal);
|
|
@@ -153,7 +154,7 @@ export class DesktopObserver {
|
|
|
153
154
|
this.session.trace.record("notice", () => notice("ok", "speech heard", this.session.redactNarration(utterance.text)));
|
|
154
155
|
}
|
|
155
156
|
if (this.pendingHeardSpeech.length > CUA_SPEECH_LIMITS.utterances) {
|
|
156
|
-
throw new
|
|
157
|
+
throw new ComputerUseExecutorError("invalid_response", "outcome_uncertain");
|
|
157
158
|
}
|
|
158
159
|
return this.pendingHeardSpeech.length === 0
|
|
159
160
|
? observation
|
|
@@ -185,19 +186,19 @@ export class DesktopObserver {
|
|
|
185
186
|
}));
|
|
186
187
|
session.trace.bump("screenshots");
|
|
187
188
|
}
|
|
188
|
-
// Evaluated
|
|
189
|
+
// Evaluated before the stopWhen check each turn so a final task whose criterion coincides with
|
|
189
190
|
// the stop condition still lands in the funnel of the very turn that ends the session.
|
|
190
191
|
observeTasks(observation, turnNumber) {
|
|
191
192
|
const { taskTracker } = this.session;
|
|
192
193
|
if (taskTracker === undefined)
|
|
193
194
|
return;
|
|
194
195
|
for (const completion of taskTracker.observe(stopObservationOf(observation), turnNumber)) {
|
|
195
|
-
// The id is researcher-authored config and the kinds are rule-type names; the matched
|
|
196
|
-
// (a URL, page text) never appear here
|
|
196
|
+
// The id is researcher-authored config and the kinds are rule-type names; the matched values
|
|
197
|
+
// (a URL, page text) never appear here: the same discipline as the stopWhen notice.
|
|
197
198
|
this.session.trace.record("notice", () => notice("matched", `task completed: ${this.session.redactNarration(completion.id)}`, this.session.redactNarration(`turn ${turnNumber}; matched rule ${completion.matchedRuleIndex} (${completion.matchedKinds.join("+")})`)));
|
|
198
199
|
}
|
|
199
200
|
}
|
|
200
|
-
// The declared observation window
|
|
201
|
+
// The declared observation window. The harness holds, looks, and takes nothing back to
|
|
201
202
|
// the model: no action, no turn, no tokens. It runs once, cut to whatever session budget is
|
|
202
203
|
// left, and says in the trace that the time was deliberate.
|
|
203
204
|
async dwellIfDue(observation, turnNumber) {
|
|
@@ -238,9 +239,9 @@ export class DesktopObserver {
|
|
|
238
239
|
}
|
|
239
240
|
const heldMs = session.now() - dwellStartedAtMs;
|
|
240
241
|
session.trace.record("notice", () => notice(
|
|
241
|
-
// A window that
|
|
242
|
+
// A window that ends the session is structured, harness-owned completion evidence, the
|
|
242
243
|
// same class as a matched stopWhen, and the verdict resolver reads it that way.
|
|
243
|
-
dwell.then === "stop" ? "matched" : "ok", "dwell window complete", `${frames
|
|
244
|
+
dwell.then === "stop" ? "matched" : "ok", "dwell window complete", `${plural(frames, "frame")} over ${heldMs}ms; no model turn was requested during the window`));
|
|
244
245
|
return { next: dwell.then, heldMs };
|
|
245
246
|
}
|
|
246
247
|
}
|
|
@@ -8,7 +8,7 @@ export type TurnReply = {
|
|
|
8
8
|
};
|
|
9
9
|
/**
|
|
10
10
|
* Ask the provider for one turn. A fail_closed provider gets one single dispatch; any other is
|
|
11
|
-
* raced against turnTimeoutMs
|
|
11
|
+
* raced against turnTimeoutMs and retried once when it stalls.
|
|
12
12
|
*/
|
|
13
13
|
export declare function requestTurn(session: LoopSession, request: CuaTurnRequest, turnNumber: number): Promise<TurnReply>;
|
|
14
14
|
/**
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { isComputerUseAdmissionLimitError } from "../admission-limit.js";
|
|
2
|
+
import { ComputerUseProviderError, isComputerUseProviderError } from "../provider-error.js";
|
|
3
3
|
import { adapterLimit, providerStalledTwice, usageUnreported } from "./ending.js";
|
|
4
4
|
import { CuaAbortError, CuaDeadlineError, CuaStallError, raceCallBound, requestScope, } from "./race.js";
|
|
5
5
|
import { bookLostRequest, LostRequestRefused, spendGate } from "./spend.js";
|
|
@@ -12,7 +12,7 @@ import { reportedCounts, settledReceipt } from "./usage.js";
|
|
|
12
12
|
const CUA_PROVIDER_CLEANUP_GRACE_MS = 5000;
|
|
13
13
|
/**
|
|
14
14
|
* Ask the provider for one turn. A fail_closed provider gets one single dispatch; any other is
|
|
15
|
-
* raced against turnTimeoutMs
|
|
15
|
+
* raced against turnTimeoutMs and retried once when it stalls.
|
|
16
16
|
*/
|
|
17
17
|
export async function requestTurn(session, request, turnNumber) {
|
|
18
18
|
const { provider } = session;
|
|
@@ -32,7 +32,7 @@ export async function requestTurn(session, request, turnNumber) {
|
|
|
32
32
|
catch (error) {
|
|
33
33
|
// Stops are concluded here, before `finally` ends the request scope, so their notices are
|
|
34
34
|
// recorded before any abort listener runs.
|
|
35
|
-
if (
|
|
35
|
+
if (isComputerUseAdmissionLimitError(error))
|
|
36
36
|
return { stop: session.conclude(adapterLimit) };
|
|
37
37
|
if (error instanceof LostRequestRefused)
|
|
38
38
|
return { stop: session.conclude(error.stop) };
|
|
@@ -73,7 +73,7 @@ async function retryStalledTurn(session, request, turnNumber, stall, stalled) {
|
|
|
73
73
|
};
|
|
74
74
|
}
|
|
75
75
|
catch (retryError) {
|
|
76
|
-
if (
|
|
76
|
+
if (isComputerUseAdmissionLimitError(retryError))
|
|
77
77
|
return { stop: session.conclude(adapterLimit) };
|
|
78
78
|
if (retryError instanceof LostRequestRefused)
|
|
79
79
|
return { stop: session.conclude(retryError.stop) };
|
|
@@ -101,7 +101,7 @@ export async function singleDispatch(session, kind, dispatch, boundMs) {
|
|
|
101
101
|
const pending = Promise.resolve()
|
|
102
102
|
.then(() => {
|
|
103
103
|
if (scope.signal.aborted)
|
|
104
|
-
throw new
|
|
104
|
+
throw new ComputerUseProviderError("cancelled", {
|
|
105
105
|
dispatched: false,
|
|
106
106
|
usageComplete: false,
|
|
107
107
|
cleanup: "confirmed",
|
|
@@ -135,11 +135,11 @@ export async function singleDispatch(session, kind, dispatch, boundMs) {
|
|
|
135
135
|
}
|
|
136
136
|
const failure = "error" in outcome ? outcome.error : undefined;
|
|
137
137
|
const requestSettlement = settlementRef.current;
|
|
138
|
-
const providerError =
|
|
138
|
+
const providerError = isComputerUseProviderError(failure)
|
|
139
139
|
? failure
|
|
140
140
|
: requestSettlement &&
|
|
141
141
|
"error" in requestSettlement &&
|
|
142
|
-
|
|
142
|
+
isComputerUseProviderError(requestSettlement.error)
|
|
143
143
|
? requestSettlement.error
|
|
144
144
|
: undefined;
|
|
145
145
|
const turn = requestSettlement && "turn" in requestSettlement ? requestSettlement.turn : undefined;
|
|
@@ -157,8 +157,8 @@ export async function singleDispatch(session, kind, dispatch, boundMs) {
|
|
|
157
157
|
if (failure instanceof CuaAbortError || failure instanceof CuaDeadlineError)
|
|
158
158
|
throw failure;
|
|
159
159
|
if (failure instanceof CuaStallError)
|
|
160
|
-
throw new
|
|
161
|
-
throw providerError ?? new
|
|
160
|
+
throw new ComputerUseProviderError("timeout", receipt, usage, providerError?.failurePhase);
|
|
161
|
+
throw providerError ?? new ComputerUseProviderError("process_failed", receipt, usage);
|
|
162
162
|
}
|
|
163
163
|
/**
|
|
164
164
|
* Whether a single-dispatch turn is a valid turn from a continuing request (see
|
|
@@ -175,7 +175,7 @@ function isContinuingTurn(kind, turn) {
|
|
|
175
175
|
turn.usage !== undefined ||
|
|
176
176
|
turn.interruption !== undefined ||
|
|
177
177
|
turn.closingReport !== undefined) {
|
|
178
|
-
throw new
|
|
178
|
+
throw new ComputerUseProviderError("invalid_response", {
|
|
179
179
|
dispatched: "unknown",
|
|
180
180
|
usageComplete: false,
|
|
181
181
|
cleanup: "unconfirmed",
|
|
@@ -188,7 +188,7 @@ function isContinuingTurn(kind, turn) {
|
|
|
188
188
|
r.dispatched !== true ||
|
|
189
189
|
typeof r.usageComplete !== "boolean" ||
|
|
190
190
|
r.cleanup !== "confirmed") {
|
|
191
|
-
throw new
|
|
191
|
+
throw new ComputerUseProviderError("invalid_response", { dispatched: "unknown", usageComplete: false, cleanup: "unconfirmed" }, turn.usage);
|
|
192
192
|
}
|
|
193
193
|
return false;
|
|
194
194
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { ActorTraceItem, ParticipantDeclaredOutcome } from "../../contract.js";
|
|
2
2
|
import type { AffordanceObservation } from "../../affordance.js";
|
|
3
|
-
import { TaskTracker } from "../../../
|
|
3
|
+
import { TaskTracker } from "../../../study/tasks.js";
|
|
4
4
|
import type { DebriefTrigger, Stop } from "./ending.js";
|
|
5
5
|
import { TraceRecorder } from "./trace.js";
|
|
6
|
-
import type { CuaExecutor,
|
|
6
|
+
import type { CuaExecutor, CuaProvider, CuaSafetyCheck, LoopRunOptions } from "./types.js";
|
|
7
7
|
import { UsageLedger } from "./usage.js";
|
|
8
8
|
export type ScreenshotRef = NonNullable<ActorTraceItem["screenshotRef"]>;
|
|
9
9
|
/**
|
|
@@ -11,7 +11,7 @@ export type ScreenshotRef = NonNullable<ActorTraceItem["screenshotRef"]>;
|
|
|
11
11
|
* wrapped callbacks are fields on LoopSession instead. Callers destructure hooks to call them.
|
|
12
12
|
*/
|
|
13
13
|
export type LoopSettings = {
|
|
14
|
-
readonly [K in "instructions" | "persona" | "redaction" | "stopWhen" | "dwell" | "tasks" | "maxUsd" | "overRunBudget" | "estimateTurnCostUsd" | "onObservedUrl" | "onMessage" | "onScreenshot" | "onTrace"]:
|
|
14
|
+
readonly [K in "instructions" | "persona" | "redaction" | "stopWhen" | "dwell" | "tasks" | "maxUsd" | "overRunBudget" | "estimateTurnCostUsd" | "onObservedUrl" | "onMessage" | "onScreenshot" | "onTrace"]: LoopRunOptions[K];
|
|
15
15
|
};
|
|
16
16
|
/** The participant's dispatched actions, as the backstop and failure notices describe them. */
|
|
17
17
|
export interface ActionHistory {
|
|
@@ -63,7 +63,7 @@ export declare class LoopSession {
|
|
|
63
63
|
debriefTrigger: DebriefTrigger | undefined;
|
|
64
64
|
private stopCause;
|
|
65
65
|
private readonly scrubText;
|
|
66
|
-
constructor(options:
|
|
66
|
+
constructor(options: LoopRunOptions);
|
|
67
67
|
remaining(): number;
|
|
68
68
|
redactNarration(text: string): string;
|
|
69
69
|
/** Hand a watcher the trace so far with the running usage, without waiting on it. */
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TaskTracker } from "../../../
|
|
1
|
+
import { TaskTracker } from "../../../study/tasks.js";
|
|
2
2
|
import { TraceRecorder } from "./trace.js";
|
|
3
3
|
import { UsageLedger } from "./usage.js";
|
|
4
4
|
// The state one loop session shares across its phases: the options as read at entry, the session
|
|
@@ -7,7 +7,7 @@ import { UsageLedger } from "./usage.js";
|
|
|
7
7
|
// (screenshot, wait, screenshot, wait), and a short idle limit ends that session as gave_up before
|
|
8
8
|
// the mail can arrive.
|
|
9
9
|
const DEFAULT_IDLE_STEPS = 24;
|
|
10
|
-
// A no-progress turn also needs a repeated action
|
|
10
|
+
// A no-progress turn also needs a repeated action, so this needs less headroom than the
|
|
11
11
|
// idle limit.
|
|
12
12
|
const DEFAULT_NO_PROGRESS_STEPS = 20;
|
|
13
13
|
const DEFAULT_TURN_TIMEOUT_MS = 180_000;
|
|
@@ -105,9 +105,9 @@ export class LoopSession {
|
|
|
105
105
|
remaining() {
|
|
106
106
|
return this.timeoutMs - (this.now() - this.startedAtMs);
|
|
107
107
|
}
|
|
108
|
-
// Model-authored narration: literal-scrub known provisioned values,
|
|
108
|
+
// Model-authored narration: literal-scrub known provisioned values, then pattern-redact.
|
|
109
109
|
// A value the model transcribes (a DB password it read on screen) has no shape, so redactText
|
|
110
|
-
// alone cannot catch it
|
|
110
|
+
// alone cannot catch it; the run's scrubKnownValues, injected as scrubText, closes that.
|
|
111
111
|
redactNarration(text) {
|
|
112
112
|
return this.settings.redaction.redactText(this.scrubText(text));
|
|
113
113
|
}
|
|
@@ -9,7 +9,7 @@ export declare class LostRequestRefused extends Error {
|
|
|
9
9
|
/**
|
|
10
10
|
* The spend caps, checked before the next provider request so a model stuck retrying cannot keep
|
|
11
11
|
* spending. The per-participant cap stops the session once the running estimate crosses maxUsd; a null
|
|
12
|
-
* estimate cannot trip it, because preflight guaranteed a rate. The study budget
|
|
12
|
+
* estimate cannot trip it, because preflight guaranteed a rate. The study budget is checked
|
|
13
13
|
* next.
|
|
14
14
|
*/
|
|
15
15
|
export declare function spendStop(session: LoopSession): Stop | undefined;
|
|
@@ -14,7 +14,7 @@ export class LostRequestRefused extends Error {
|
|
|
14
14
|
/**
|
|
15
15
|
* The spend caps, checked before the next provider request so a model stuck retrying cannot keep
|
|
16
16
|
* spending. The per-participant cap stops the session once the running estimate crosses maxUsd; a null
|
|
17
|
-
* estimate cannot trip it, because preflight guaranteed a rate. The study budget
|
|
17
|
+
* estimate cannot trip it, because preflight guaranteed a rate. The study budget is checked
|
|
18
18
|
* next.
|
|
19
19
|
*/
|
|
20
20
|
export function spendStop(session) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { summarizeAffordanceUse } from "../../affordance.js";
|
|
2
2
|
import { ACTOR_TRACE_SCHEMA, } from "../../contract.js";
|
|
3
|
+
import { plural } from "../../../run/text.js";
|
|
3
4
|
export function notice(status, title, text) {
|
|
4
5
|
return { lifecycle: "completed", status, title, text };
|
|
5
6
|
}
|
|
@@ -20,7 +21,7 @@ export class TraceRecorder {
|
|
|
20
21
|
constructor(now) {
|
|
21
22
|
this.now = now;
|
|
22
23
|
}
|
|
23
|
-
// The
|
|
24
|
+
// The single recording choke point: every trace item is stamped `at` from the
|
|
24
25
|
// loop's injected clock as it is recorded, so timed playback reads recorded facts
|
|
25
26
|
// (deterministic in tests via the injected `now`). The id is allocated before the body is
|
|
26
27
|
// built, so a body that fails to build (a redaction error) still consumes its id.
|
|
@@ -40,7 +41,7 @@ export function statusForCompletionReason(reason) {
|
|
|
40
41
|
case "turn_completed": // turn_completed is a Codex reason; this loop emits goal_satisfied
|
|
41
42
|
return "passed";
|
|
42
43
|
// A session that ran out of time or budget did not reach its goal, whatever it achieved along
|
|
43
|
-
// the way. Calling that `passed` is how a truncated study came to be reported as a green one
|
|
44
|
+
// the way. Calling that `passed` is how a truncated study came to be reported as a green one,
|
|
44
45
|
// and why "raise the timeout" kept landing on the operator instead of on the tool. This switch
|
|
45
46
|
// is exhaustive with no default, so a new completion reason forces a compile error here.
|
|
46
47
|
case "budget_reached":
|
|
@@ -51,7 +52,7 @@ export function statusForCompletionReason(reason) {
|
|
|
51
52
|
return "blocked";
|
|
52
53
|
// A participant who stopped trying is the single most valuable thing a usability study
|
|
53
54
|
// produces. Recording it as `failed` said the instrument broke, which is a different claim and
|
|
54
|
-
// a false one
|
|
55
|
+
// a false one; see docs/principles/three-roles.md.
|
|
55
56
|
case "gave_up":
|
|
56
57
|
return "abandoned";
|
|
57
58
|
// Only the harness failing is a harness failure.
|
|
@@ -76,13 +77,13 @@ export function loopResult(session, stop, debrief) {
|
|
|
76
77
|
const screenshotNote = !(counts.screenshots > 0)
|
|
77
78
|
? "no screenshots captured"
|
|
78
79
|
: session.redactScreenshots
|
|
79
|
-
? `${counts.screenshots
|
|
80
|
-
: `${counts.screenshots
|
|
81
|
-
// Self-describing artifact
|
|
80
|
+
? `${plural(counts.screenshots, "screenshot")} redacted to blurred thumbnails via RedactionHooks`
|
|
81
|
+
: `${plural(counts.screenshots, "unblurred screenshot")} kept for local use and not redacted for publishing; set redactScreenshots to blur screenshots in a bundle you plan to share`;
|
|
82
|
+
// Self-describing artifact: when any observation carried structured app state,
|
|
82
83
|
// the trace says how the loop handled it: it fed progress and task checks and was not written to
|
|
83
84
|
// the trace. The appState itself never appears in this bundle.
|
|
84
85
|
const notes = session.observedAppState
|
|
85
|
-
? `${screenshotNote}. App state was
|
|
86
|
+
? `${screenshotNote}. App state was read each turn to detect progress (an in-process executor) and was not written to the trace: it is used only while the run is live and is never stored as evidence.`
|
|
86
87
|
: screenshotNote;
|
|
87
88
|
const tokenUsage = usage.tokenUsage();
|
|
88
89
|
const trace = {
|
|
@@ -90,6 +91,7 @@ export function loopResult(session, stop, debrief) {
|
|
|
90
91
|
provider: provider.id,
|
|
91
92
|
...usage.liveMetadata(),
|
|
92
93
|
...(provider.version === undefined ? {} : { providerVersion: provider.version }),
|
|
94
|
+
...(provider.conversation === undefined ? {} : { conversation: provider.conversation }),
|
|
93
95
|
protocol: "cua-loop",
|
|
94
96
|
lane: "computer-use",
|
|
95
97
|
persona: settings.persona,
|
|
@@ -130,8 +132,8 @@ export function loopResult(session, stop, debrief) {
|
|
|
130
132
|
...(usage.interactionUsageIncomplete(session.capDeclared)
|
|
131
133
|
? { interactionUsageIncomplete: true }
|
|
132
134
|
: {}),
|
|
133
|
-
// The funnel is present exactly when a protocol was declared
|
|
134
|
-
// on turn 0, whose funnel
|
|
135
|
+
// The funnel is present exactly when a protocol was declared, including a session that ended
|
|
136
|
+
// on turn 0, whose funnel reads 0/N. No tasks declared means no funnel, not an empty one.
|
|
135
137
|
...(session.taskTracker === undefined ? {} : { taskFunnel: session.taskTracker.funnel() }),
|
|
136
138
|
...(tokenUsage === undefined ? {} : { tokenUsage }),
|
|
137
139
|
capabilities: provider.capabilities,
|