humanish 0.107.0 → 0.109.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +73 -69
- package/CHANGELOG.md +27 -0
- package/CONTEXT.md +51 -44
- package/README.md +148 -175
- package/SECURITY.md +16 -26
- package/TELEMETRY.md +8 -6
- package/dist/actors/affordance.d.ts +4 -4
- package/dist/actors/affordance.js +12 -12
- package/dist/actors/agent-session.d.ts +2 -2
- package/dist/actors/agent-session.js +7 -7
- package/dist/actors/codex/app-server-actor-trace.d.ts +1 -1
- package/dist/actors/codex/app-server-actor-trace.js +1 -1
- package/dist/actors/codex/app-server-trace.js +2 -1
- package/dist/actors/codex/app-server-ui.js +1 -1
- package/dist/actors/codex/codex-admission.d.ts +91 -0
- package/dist/actors/codex/codex-admission.js +150 -0
- package/dist/actors/codex/restricted-executable.d.ts +4 -4
- package/dist/actors/codex/restricted-executable.js +8 -8
- package/dist/actors/codex/restricted-launch.d.ts +2 -1
- package/dist/actors/codex/restricted-launch.js +1 -1
- package/dist/actors/codex/restricted-participant.js +20 -18
- package/dist/actors/codex/restricted-policy.d.ts +2 -2
- package/dist/actors/codex/restricted-policy.js +2 -2
- package/dist/actors/codex/restricted-session.d.ts +3 -3
- package/dist/actors/codex/restricted-session.js +10 -5
- package/dist/actors/computer-use/actor.d.ts +18 -18
- package/dist/actors/computer-use/actor.js +4 -4
- package/dist/actors/computer-use/admission-limit.d.ts +3 -3
- package/dist/actors/computer-use/admission-limit.js +4 -4
- package/dist/actors/computer-use/executor-error.d.ts +2 -2
- package/dist/actors/computer-use/executor-error.js +3 -3
- package/dist/actors/computer-use/loop/actions.d.ts +3 -3
- package/dist/actors/computer-use/loop/actions.js +15 -15
- package/dist/actors/computer-use/loop/backstop.d.ts +3 -3
- package/dist/actors/computer-use/loop/backstop.js +14 -12
- package/dist/actors/computer-use/loop/debrief.js +2 -2
- package/dist/actors/computer-use/loop/ending.d.ts +6 -6
- package/dist/actors/computer-use/loop/ending.js +18 -17
- package/dist/actors/computer-use/loop/observation.d.ts +2 -2
- package/dist/actors/computer-use/loop/observation.js +15 -14
- package/dist/actors/computer-use/loop/provider-call.d.ts +1 -1
- package/dist/actors/computer-use/loop/provider-call.js +12 -12
- package/dist/actors/computer-use/loop/session.d.ts +4 -4
- package/dist/actors/computer-use/loop/session.js +4 -4
- package/dist/actors/computer-use/loop/spend.d.ts +1 -1
- package/dist/actors/computer-use/loop/spend.js +1 -1
- package/dist/actors/computer-use/loop/trace.js +11 -9
- package/dist/actors/computer-use/loop/types.d.ts +47 -45
- package/dist/actors/computer-use/loop/usage.d.ts +2 -2
- package/dist/actors/computer-use/loop/usage.js +2 -2
- package/dist/actors/computer-use/loop.d.ts +6 -1
- package/dist/actors/computer-use/loop.js +26 -4
- package/dist/actors/computer-use/openai-context.d.ts +37 -0
- package/dist/actors/computer-use/openai-context.js +214 -0
- package/dist/actors/computer-use/openai-provider.d.ts +13 -10
- package/dist/actors/computer-use/openai-provider.js +52 -41
- package/dist/actors/computer-use/openai-requests.d.ts +36 -10
- package/dist/actors/computer-use/openai-requests.js +82 -15
- package/dist/actors/computer-use/openai-wire.d.ts +19 -7
- package/dist/actors/computer-use/openai-wire.js +36 -34
- package/dist/actors/computer-use/provider-error.d.ts +4 -4
- package/dist/actors/computer-use/provider-error.js +6 -6
- package/dist/actors/contract.d.ts +82 -47
- package/dist/actors/contract.js +30 -24
- package/dist/actors/local-agent/claude-session.d.ts +1 -1
- package/dist/actors/local-agent/claude-session.js +13 -12
- package/dist/actors/local-agent/cli.d.ts +9 -7
- package/dist/actors/local-agent/cli.js +24 -28
- package/dist/actors/local-agent/readiness.d.ts +2 -2
- package/dist/actors/local-agent/readiness.js +4 -4
- package/dist/actors/reasoning-effort.js +8 -8
- package/dist/actors/registry.d.ts +12 -12
- package/dist/actors/registry.js +21 -20
- package/dist/actors/scripted-browser/actor.d.ts +10 -10
- package/dist/actors/scripted-browser/actor.js +17 -17
- package/dist/actors/scripted-browser/browser-command.js +1 -1
- package/dist/actors/scripted-browser/steps.js +1 -1
- package/dist/actors/scripted-browser/types.d.ts +1 -1
- package/dist/actors/stop-conditions.d.ts +1 -1
- package/dist/analysis/automatic-completion.d.ts +7 -7
- package/dist/analysis/automatic-completion.js +1 -1
- package/dist/analysis/automatic-config.d.ts +6 -6
- package/dist/analysis/automatic-config.js +1 -1
- package/dist/analysis/codex-config.d.ts +6 -2
- package/dist/analysis/codex-config.js +22 -14
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +1 -1
- package/dist/analysis/service.js +5 -5
- package/dist/analysis/types.d.ts +6 -0
- package/dist/browser-control/client.js +16 -16
- package/dist/browser-control/dispatcher.js +4 -4
- package/dist/browser-control/protocol.js +14 -14
- package/dist/browser-control/recording-transfer.js +4 -4
- package/dist/browser-control/transport.js +7 -7
- package/dist/cli/commands/analysis-signals.d.ts +2 -2
- package/dist/cli/commands/analyze.js +19 -16
- package/dist/cli/commands/codex.js +9 -9
- package/dist/cli/commands/comms.js +48 -35
- package/dist/cli/commands/feedback.js +20 -21
- package/dist/cli/commands/migrate.d.ts +4 -0
- package/dist/cli/commands/migrate.js +65 -0
- package/dist/cli/commands/observe.d.ts +0 -1
- package/dist/cli/commands/observe.js +136 -112
- package/dist/cli/commands/run-command.d.ts +10 -0
- package/dist/cli/commands/run-command.js +85 -0
- package/dist/cli/commands/runs.js +136 -139
- package/dist/cli/commands/setup.d.ts +2 -2
- package/dist/cli/commands/setup.js +27 -24
- package/dist/cli/commands/study-format.d.ts +9 -0
- package/dist/cli/commands/{lab-format.js → study-format.js} +39 -27
- package/dist/cli/commands/{lab-route-computer-use.d.ts → study-route-computer-use.d.ts} +6 -6
- package/dist/cli/commands/{lab-route-computer-use.js → study-route-computer-use.js} +17 -18
- package/dist/cli/commands/{lab-route-open.d.ts → study-route-open.d.ts} +4 -4
- package/dist/cli/commands/{lab-route-open.js → study-route-open.js} +2 -2
- package/dist/cli/commands/{lab-route-preview.d.ts → study-route-preview.d.ts} +7 -7
- package/dist/cli/commands/{lab-route-preview.js → study-route-preview.js} +7 -6
- package/dist/cli/commands/study-route-run.d.ts +19 -0
- package/dist/cli/commands/{lab-route-run.js → study-route-run.js} +6 -6
- package/dist/cli/commands/study-route-scripted.d.ts +17 -0
- package/dist/cli/commands/{lab-route-scripted.js → study-route-scripted.js} +4 -4
- package/dist/cli/commands/{lab-route-shared-world.d.ts → study-route-shared-world.d.ts} +5 -5
- package/dist/cli/commands/{lab-route-shared-world.js → study-route-shared-world.js} +10 -10
- package/dist/cli/commands/study-route-terminal.d.ts +17 -0
- package/dist/cli/commands/{lab-route-terminal.js → study-route-terminal.js} +4 -4
- package/dist/cli/commands/study-run.d.ts +9 -0
- package/dist/cli/commands/{lab-run.js → study-run.js} +44 -45
- package/dist/cli/commands/{lab-scorer.d.ts → study-scorer.d.ts} +7 -7
- package/dist/cli/commands/{lab-scorer.js → study-scorer.js} +2 -2
- package/dist/cli/commands/study.d.ts +3 -0
- package/dist/cli/commands/study.js +146 -0
- package/dist/cli/commands/tui.d.ts +4 -4
- package/dist/cli/commands/tui.js +31 -31
- package/dist/cli/commands/watch.js +71 -121
- package/dist/cli/deprecations.d.ts +6 -0
- package/dist/cli/deprecations.js +18 -0
- package/dist/cli/doctor.d.ts +18 -8
- package/dist/cli/doctor.js +114 -59
- package/dist/cli/first-run-path.d.ts +15 -13
- package/dist/cli/first-run-path.js +35 -33
- package/dist/cli/io.d.ts +44 -9
- package/dist/cli/io.js +98 -23
- package/dist/cli/observer-follow.d.ts +3 -3
- package/dist/cli/observer-follow.js +6 -7
- package/dist/cli/orientation.d.ts +10 -7
- package/dist/cli/orientation.js +89 -47
- package/dist/cli/product-sentence.d.ts +6 -0
- package/dist/cli/product-sentence.js +6 -0
- package/dist/cli/program.d.ts +6 -1
- package/dist/cli/program.js +111 -43
- package/dist/cli/telemetry.d.ts +16 -16
- package/dist/cli/telemetry.js +46 -44
- package/dist/cli.js +1 -1
- package/dist/comms/capture-surface.d.ts +13 -12
- package/dist/comms/capture-surface.js +23 -22
- package/dist/comms/catch-host.d.ts +7 -7
- package/dist/comms/catch-host.js +17 -17
- package/dist/comms/connections.d.ts +1 -1
- package/dist/comms/connections.js +2 -2
- package/dist/comms/email-catch.d.ts +3 -3
- package/dist/comms/email-catch.js +8 -8
- package/dist/comms/external-evidence.d.ts +5 -5
- package/dist/comms/external-evidence.js +5 -4
- package/dist/comms/extract.js +2 -2
- package/dist/comms/fake-inbox.d.ts +3 -3
- package/dist/comms/fake-inbox.js +3 -3
- package/dist/comms/receiving-render.d.ts +1 -1
- package/dist/comms/receiving-render.js +2 -2
- package/dist/comms/receiving-runtime.d.ts +4 -4
- package/dist/comms/sandbox-catch-script.d.ts +6 -6
- package/dist/comms/sandbox-catch-script.js +6 -6
- package/dist/comms/sandbox-catch.d.ts +38 -39
- package/dist/comms/sandbox-catch.js +36 -37
- package/dist/comms/setup.d.ts +2 -2
- package/dist/comms/setup.js +56 -25
- package/dist/comms/thread-evidence.d.ts +1 -1
- package/dist/comms/thread-evidence.js +6 -6
- package/dist/comms/types.d.ts +6 -6
- package/dist/comms/types.js +3 -3
- package/dist/evidence/frame-signature.d.ts +1 -1
- package/dist/evidence/frame-signature.js +4 -4
- package/dist/evidence/redaction.d.ts +3 -3
- package/dist/evidence/redaction.js +4 -4
- package/dist/feedback/draft.js +38 -80
- package/dist/feedback/export-bundle.js +1 -1
- package/dist/feedback/export.d.ts +5 -1
- package/dist/feedback/export.js +10 -10
- package/dist/feedback/feedback.d.ts +2 -2
- package/dist/feedback/feedback.js +21 -3
- package/dist/feedback/proof.d.ts +1 -1
- package/dist/feedback/proof.js +2 -2
- package/dist/guest/bootstrap.js +6 -6
- package/dist/guest/browser-tools.js +8 -8
- package/dist/guest/chromium-text.js +19 -19
- package/dist/guest/desktop-executor.d.ts +2 -2
- package/dist/guest/desktop-executor.js +20 -20
- package/dist/guest/desktop-keys.js +4 -4
- package/dist/guest/desktop-media.d.ts +2 -2
- package/dist/guest/desktop-media.js +16 -16
- package/dist/guest/desktop-native.d.ts +1 -1
- package/dist/guest/desktop-native.js +10 -10
- package/dist/guest/runtime-desktop.js +2 -2
- package/dist/guest/runtime-processes.d.ts +2 -2
- package/dist/guest/runtime-processes.js +2 -2
- package/dist/guest/runtime.js +6 -6
- package/dist/index.d.ts +14 -14
- package/dist/index.js +8 -8
- package/dist/keys/key-resolution.d.ts +5 -5
- package/dist/keys/key-resolution.js +23 -23
- package/dist/observer/artifact.d.ts +2 -2
- package/dist/observer/artifact.js +7 -7
- package/dist/observer/data.d.ts +6 -6
- package/dist/observer/data.js +5 -4
- package/dist/observer/exposure.d.ts +1 -1
- package/dist/observer/exposure.js +11 -11
- package/dist/observer/http.js +1 -1
- package/dist/observer/library.d.ts +3 -1
- package/dist/observer/library.js +8 -3
- package/dist/observer/listen.js +5 -4
- package/dist/observer/render.d.ts +1 -1
- package/dist/observer/render.js +17 -6
- package/dist/observer/run-routes.d.ts +4 -3
- package/dist/observer/run-routes.js +38 -13
- package/dist/observer/serve.d.ts +1 -1
- package/dist/observer/serve.js +7 -3
- package/dist/observer/tunnel.js +1 -1
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle-parts.d.ts +9 -9
- package/dist/routes/computer-use/bundle-parts.js +18 -17
- package/dist/routes/computer-use/bundle.d.ts +2 -2
- package/dist/routes/computer-use/bundle.js +10 -9
- package/dist/routes/computer-use/e2b-desktop/comms.d.ts +6 -6
- package/dist/routes/computer-use/e2b-desktop/comms.js +18 -17
- package/dist/routes/computer-use/e2b-desktop/desktop.js +4 -4
- package/dist/routes/computer-use/e2b-desktop/fidelity.d.ts +5 -4
- package/dist/routes/computer-use/e2b-desktop/fidelity.js +11 -10
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.js +10 -10
- package/dist/routes/computer-use/e2b-desktop/start.js +7 -7
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +2 -2
- package/dist/routes/computer-use/e2b-desktop/teardown.js +2 -2
- package/dist/routes/computer-use/external-comms.d.ts +4 -4
- package/dist/routes/computer-use/external-comms.js +1 -1
- package/dist/routes/computer-use/fanout-bundle.js +27 -26
- package/dist/routes/computer-use/fanout-records.js +12 -13
- package/dist/routes/computer-use/live-flush.d.ts +4 -4
- package/dist/routes/computer-use/live-flush.js +6 -6
- package/dist/routes/computer-use/live-phase.d.ts +5 -4
- package/dist/routes/computer-use/live-phase.js +8 -8
- package/dist/routes/computer-use/local-tree-pack.d.ts +5 -5
- package/dist/routes/computer-use/local-tree-pack.js +1 -1
- package/dist/routes/computer-use/local-vm.d.ts +8 -8
- package/dist/routes/computer-use/local-vm.js +3 -3
- package/dist/routes/computer-use/participant-desktop.d.ts +7 -7
- package/dist/routes/computer-use/participant-desktop.js +5 -5
- package/dist/routes/computer-use/participant-execution.d.ts +4 -4
- package/dist/routes/computer-use/participant-execution.js +10 -10
- package/dist/routes/computer-use/participant-model.d.ts +6 -0
- package/dist/routes/computer-use/participant-model.js +33 -15
- package/dist/routes/computer-use/participant-prompt.d.ts +15 -15
- package/dist/routes/computer-use/participant-prompt.js +9 -9
- package/dist/routes/computer-use/participant-runs.d.ts +16 -16
- package/dist/routes/computer-use/participant-runs.js +26 -25
- package/dist/routes/computer-use/plan.d.ts +12 -12
- package/dist/routes/computer-use/plan.js +42 -42
- package/dist/routes/computer-use/preflight.d.ts +5 -5
- package/dist/routes/computer-use/preflight.js +25 -25
- package/dist/routes/computer-use/rerun-selection.d.ts +1 -1
- package/dist/routes/computer-use/rerun-selection.js +4 -3
- package/dist/routes/computer-use/result.d.ts +4 -4
- package/dist/routes/computer-use/result.js +22 -21
- package/dist/routes/computer-use/route.d.ts +12 -11
- package/dist/routes/computer-use/route.js +31 -30
- package/dist/routes/computer-use/self-report.d.ts +7 -7
- package/dist/routes/computer-use/self-report.js +32 -32
- package/dist/routes/computer-use/setup.d.ts +23 -23
- package/dist/routes/computer-use/setup.js +28 -29
- package/dist/routes/computer-use/single-bundle.d.ts +11 -11
- package/dist/routes/computer-use/single-bundle.js +40 -41
- package/dist/routes/computer-use/subject-projection.d.ts +4 -4
- package/dist/routes/computer-use/subject-projection.js +6 -6
- package/dist/routes/computer-use/types.d.ts +71 -73
- package/dist/routes/computer-use/types.js +12 -12
- package/dist/routes/preview.d.ts +13 -7
- package/dist/routes/preview.js +11 -7
- package/dist/routes/scripted/bundle.d.ts +7 -7
- package/dist/routes/scripted/bundle.js +20 -18
- package/dist/routes/scripted/plan.d.ts +8 -8
- package/dist/routes/scripted/plan.js +13 -13
- package/dist/routes/scripted/result.d.ts +3 -3
- package/dist/routes/scripted/result.js +13 -13
- package/dist/routes/scripted/route.d.ts +9 -9
- package/dist/routes/scripted/route.js +15 -16
- package/dist/routes/scripted/scenario.d.ts +1 -1
- package/dist/routes/scripted/scenario.js +5 -5
- package/dist/routes/scripted/setup.d.ts +6 -6
- package/dist/routes/scripted/setup.js +8 -8
- package/dist/routes/scripted/subject.d.ts +5 -5
- package/dist/routes/scripted/subject.js +4 -4
- package/dist/routes/scripted/surface-records.d.ts +1 -1
- package/dist/routes/scripted/surface-records.js +4 -4
- package/dist/routes/scripted/surface-sessions.d.ts +3 -3
- package/dist/routes/scripted/types.d.ts +18 -20
- package/dist/routes/scripted/types.js +2 -2
- package/dist/routes/shared-world/bundle-records.js +11 -10
- package/dist/routes/shared-world/bundle.d.ts +5 -5
- package/dist/routes/shared-world/bundle.js +46 -43
- package/dist/routes/shared-world/checkpoints.d.ts +4 -4
- package/dist/routes/shared-world/checkpoints.js +2 -2
- package/dist/routes/shared-world/comms.d.ts +14 -14
- package/dist/routes/shared-world/comms.js +19 -18
- package/dist/routes/shared-world/external-public.d.ts +4 -3
- package/dist/routes/shared-world/external-public.js +16 -12
- package/dist/routes/shared-world/handoff.d.ts +7 -6
- package/dist/routes/shared-world/handoff.js +65 -57
- package/dist/routes/shared-world/lobby-code.d.ts +2 -2
- package/dist/routes/shared-world/lobby-code.js +9 -9
- package/dist/routes/shared-world/participant-specs.d.ts +10 -9
- package/dist/routes/shared-world/participant-specs.js +38 -31
- package/dist/routes/shared-world/plan.d.ts +9 -9
- package/dist/routes/shared-world/plan.js +16 -16
- package/dist/routes/shared-world/provenance.d.ts +2 -2
- package/dist/routes/shared-world/provenance.js +2 -2
- package/dist/routes/shared-world/provisioned.d.ts +8 -8
- package/dist/routes/shared-world/provisioned.js +18 -17
- package/dist/routes/shared-world/result.d.ts +6 -6
- package/dist/routes/shared-world/result.js +29 -29
- package/dist/routes/shared-world/route.d.ts +11 -11
- package/dist/routes/shared-world/route.js +40 -39
- package/dist/routes/shared-world/setup.d.ts +16 -16
- package/dist/routes/shared-world/setup.js +23 -23
- package/dist/routes/shared-world/types.d.ts +55 -53
- package/dist/routes/shared-world/types.js +4 -4
- package/dist/routes/terminal/adapter.d.ts +5 -5
- package/dist/routes/terminal/adapter.js +9 -9
- package/dist/routes/terminal/bundle.d.ts +13 -13
- package/dist/routes/terminal/bundle.js +28 -27
- package/dist/routes/terminal/credentials.d.ts +8 -8
- package/dist/routes/terminal/credentials.js +12 -12
- package/dist/routes/terminal/dry-run.d.ts +4 -4
- package/dist/routes/terminal/dry-run.js +12 -12
- package/dist/routes/terminal/encoding.js +2 -2
- package/dist/routes/terminal/ledger.d.ts +5 -5
- package/dist/routes/terminal/ledger.js +14 -14
- package/dist/routes/terminal/lifetime.d.ts +1 -1
- package/dist/routes/terminal/lifetime.js +1 -1
- package/dist/routes/terminal/live-finish.d.ts +2 -2
- package/dist/routes/terminal/live-finish.js +10 -10
- package/dist/routes/terminal/live-sandbox.d.ts +2 -2
- package/dist/routes/terminal/live-sandbox.js +15 -15
- package/dist/routes/terminal/participant-text.js +2 -1
- package/dist/routes/terminal/persona.d.ts +2 -2
- package/dist/routes/terminal/persona.js +3 -3
- package/dist/routes/terminal/plan.d.ts +7 -7
- package/dist/routes/terminal/plan.js +15 -15
- package/dist/routes/terminal/recorder.js +2 -2
- package/dist/routes/terminal/result.d.ts +5 -5
- package/dist/routes/terminal/result.js +14 -14
- package/dist/routes/terminal/route.d.ts +9 -9
- package/dist/routes/terminal/route.js +19 -20
- package/dist/routes/terminal/sandbox.d.ts +1 -1
- package/dist/routes/terminal/sandbox.js +1 -1
- package/dist/routes/terminal/session.d.ts +5 -5
- package/dist/routes/terminal/session.js +14 -13
- package/dist/routes/terminal/token-usage.d.ts +1 -1
- package/dist/routes/terminal/token-usage.js +7 -8
- package/dist/routes/terminal/trace.d.ts +4 -4
- package/dist/routes/terminal/trace.js +2 -2
- package/dist/routes/terminal/types.d.ts +48 -50
- package/dist/routes/terminal/types.js +5 -5
- package/dist/run/bundle-shape.js +3 -3
- package/dist/run/bundle.d.ts +49 -49
- package/dist/run/bundle.js +3 -2
- package/dist/run/cost-summary.js +3 -3
- package/dist/run/costs.d.ts +23 -0
- package/dist/run/costs.js +79 -44
- package/dist/run/detail.d.ts +6 -6
- package/dist/run/detail.js +4 -4
- package/dist/run/dry-run-inputs.d.ts +1 -1
- package/dist/run/dry-run-inputs.js +2 -2
- package/dist/run/dry-run.js +20 -20
- package/dist/run/judge.d.ts +19 -13
- package/dist/run/judge.js +18 -8
- package/dist/run/outcomes.d.ts +1 -1
- package/dist/run/participant.d.ts +1 -1
- package/dist/run/paths.d.ts +1 -1
- package/dist/run/paths.js +1 -1
- package/dist/run/preflight-receipts.js +1 -1
- package/dist/run/pricing.d.ts +11 -6
- package/dist/run/pricing.js +22 -15
- package/dist/run/projection.d.ts +46 -46
- package/dist/run/projection.js +57 -57
- package/dist/run/reclaim.d.ts +1 -1
- package/dist/run/reclaim.js +3 -3
- package/dist/run/results.d.ts +5 -5
- package/dist/run/run-cost.d.ts +53 -0
- package/dist/run/run-cost.js +105 -0
- package/dist/run/run-index-analysis.d.ts +15 -0
- package/dist/run/run-index-analysis.js +77 -0
- package/dist/run/run-index.d.ts +19 -7
- package/dist/run/run-index.js +77 -64
- package/dist/run/run-not-found.d.ts +5 -0
- package/dist/run/run-not-found.js +15 -0
- package/dist/run/run.d.ts +6 -5
- package/dist/run/run.js +3 -3
- package/dist/run/sandbox-receipts.d.ts +2 -2
- package/dist/run/sandbox-receipts.js +3 -3
- package/dist/run/shared-world-evidence.d.ts +37 -36
- package/dist/run/shared-world-evidence.js +4 -4
- package/dist/run/shared-world-shape.d.ts +1 -1
- package/dist/run/shared-world-shape.js +7 -7
- package/dist/run/stats.d.ts +14 -8
- package/dist/run/stats.js +121 -63
- package/dist/run/status.d.ts +16 -29
- package/dist/run/status.js +10 -29
- package/dist/run/stored-runs.js +2 -1
- package/dist/run/stream-shape.js +1 -1
- package/dist/run/streams.d.ts +7 -7
- package/dist/run/study-provenance.d.ts +49 -0
- package/dist/run/study-provenance.js +66 -0
- package/dist/run/study-result.d.ts +11 -0
- package/dist/run/study-result.js +9 -0
- package/dist/run/text.d.ts +2 -0
- package/dist/run/text.js +4 -0
- package/dist/run-study.d.ts +112 -0
- package/dist/{run-lab.js → run-study.js} +52 -47
- package/dist/{lab → study}/adapter-extension.d.ts +15 -27
- package/dist/{lab → study}/adapter-extension.js +10 -45
- package/dist/{lab → study}/adapter-scorer-loader.d.ts +17 -17
- package/dist/{lab → study}/adapter-scorer-loader.js +34 -34
- package/dist/study/composition-rules.d.ts +7 -0
- package/dist/{lab → study}/composition-rules.js +53 -53
- package/dist/study/config.d.ts +15 -0
- package/dist/{lab → study}/config.js +98 -41
- package/dist/study/convert.d.ts +33 -0
- package/dist/study/convert.js +432 -0
- package/dist/{lab → study}/device-presets.d.ts +10 -10
- package/dist/{lab → study}/device-presets.js +11 -11
- package/dist/study/discover.d.ts +75 -0
- package/dist/{lab → study}/discover.js +134 -95
- package/dist/{lab → study}/doctor.d.ts +19 -12
- package/dist/{lab → study}/doctor.js +58 -20
- package/dist/study/field-paths.d.ts +12 -0
- package/dist/study/field-paths.js +126 -0
- package/dist/study/files.d.ts +26 -0
- package/dist/study/files.js +44 -0
- package/dist/{lab → study}/init-templates.d.ts +2 -2
- package/dist/study/init-templates.js +421 -0
- package/dist/{lab → study}/init.d.ts +2 -2
- package/dist/{lab → study}/init.js +46 -11
- package/dist/study/keys.d.ts +4 -0
- package/dist/{lab → study}/keys.js +46 -12
- package/dist/study/migrate.d.ts +48 -0
- package/dist/study/migrate.js +408 -0
- package/dist/study/owned-files.d.ts +38 -0
- package/dist/study/owned-files.js +182 -0
- package/dist/{lab → study}/parse/actors.d.ts +5 -5
- package/dist/{lab → study}/parse/actors.js +14 -14
- package/dist/study/parse/comms.d.ts +12 -0
- package/dist/{lab → study}/parse/comms.js +5 -5
- package/dist/study/parse/execution.d.ts +19 -0
- package/dist/{lab → study}/parse/execution.js +13 -13
- package/dist/study/parse/study-v3.d.ts +21 -0
- package/dist/study/parse/study-v3.js +259 -0
- package/dist/study/parse/subject-state.d.ts +30 -0
- package/dist/{lab → study}/parse/subject-state.js +19 -19
- package/dist/{lab → study}/parse/subject.d.ts +3 -3
- package/dist/{lab → study}/parse/subject.js +34 -34
- package/dist/{lab → study}/parse/values.d.ts +3 -3
- package/dist/{lab → study}/parse/values.js +3 -3
- package/dist/{lab → study}/persona-resolve.d.ts +8 -8
- package/dist/{lab → study}/persona-resolve.js +6 -6
- package/dist/{lab → study}/plan-base.d.ts +7 -7
- package/dist/{lab → study}/plan-base.js +1 -1
- package/dist/{lab → study}/plan-participants.d.ts +12 -12
- package/dist/{lab → study}/plan-participants.js +4 -4
- package/dist/{lab → study}/plan-types.d.ts +31 -31
- package/dist/{lab → study}/plan-types.js +2 -2
- package/dist/study/plan.d.ts +13 -0
- package/dist/study/plan.js +101 -0
- package/dist/study/preflight-probes.d.ts +3 -0
- package/dist/{lab → study}/preflight-probes.js +20 -19
- package/dist/study/preflight-result.d.ts +8 -0
- package/dist/{lab → study}/preflight-result.js +5 -4
- package/dist/study/preflight.d.ts +94 -0
- package/dist/{lab → study}/preflight.js +12 -12
- package/dist/{lab → study}/requirements.d.ts +3 -3
- package/dist/{lab → study}/requirements.js +1 -1
- package/dist/study/route-inputs.d.ts +28 -0
- package/dist/{lab → study}/route-inputs.js +1 -1
- package/dist/{lab → study}/routing.d.ts +39 -33
- package/dist/{lab → study}/routing.js +60 -29
- package/dist/{lab/run-lab-events.d.ts → study/run-study-events.d.ts} +10 -13
- package/dist/{lab/run-lab-events.js → study/run-study-events.js} +3 -9
- package/dist/{lab/run-lab-homes.d.ts → study/run-study-homes.d.ts} +21 -18
- package/dist/{lab/run-lab-homes.js → study/run-study-homes.js} +1 -1
- package/dist/study/run-study-options.d.ts +28 -0
- package/dist/{lab/run-lab-options.js → study/run-study-options.js} +29 -25
- package/dist/{lab/lab-deps.d.ts → study/study-deps.d.ts} +8 -13
- package/dist/{lab/lab-deps.js → study/study-deps.js} +2 -2
- package/dist/{lab → study}/summary.d.ts +16 -15
- package/dist/{lab → study}/summary.js +21 -21
- package/dist/{lab → study}/tasks.d.ts +18 -18
- package/dist/{lab → study}/tasks.js +14 -14
- package/dist/{lab → study}/types.d.ts +202 -196
- package/dist/{lab → study}/types.js +3 -1
- package/dist/{lab → study}/validation.d.ts +27 -27
- package/dist/{lab → study}/validation.js +53 -51
- package/dist/study/warnings.d.ts +12 -0
- package/dist/{lab → study}/warnings.js +62 -34
- package/dist/subject/clone.d.ts +7 -7
- package/dist/subject/clone.js +2 -2
- package/dist/subject/desktop-cli.d.ts +1 -1
- package/dist/subject/desktop-cli.js +2 -2
- package/dist/subject/local-tree-archive.d.ts +2 -2
- package/dist/subject/local-tree-archive.js +4 -4
- package/dist/subject/local-tree.d.ts +4 -4
- package/dist/subject/node-bootstrap.js +1 -1
- package/dist/subject/runtime.js +9 -10
- package/dist/subject/serve.d.ts +5 -5
- package/dist/subject/serve.js +3 -3
- package/dist/subject/state.d.ts +4 -4
- package/dist/subject/state.js +1 -1
- package/dist/subject/steps.d.ts +6 -6
- package/dist/subject/steps.js +5 -5
- package/dist/substrates/command-failure.d.ts +3 -3
- package/dist/substrates/command-failure.js +3 -3
- package/dist/substrates/desktop-session.js +2 -2
- package/dist/substrates/detached.d.ts +6 -6
- package/dist/substrates/detached.js +7 -7
- package/dist/substrates/e2b/cdp-probe.d.ts +9 -9
- package/dist/substrates/e2b/cdp-probe.js +14 -17
- package/dist/substrates/e2b/desktop-browser.d.ts +6 -6
- package/dist/substrates/e2b/desktop-browser.js +9 -9
- package/dist/substrates/e2b/desktop-cdp.d.ts +10 -10
- package/dist/substrates/e2b/desktop-cdp.js +9 -9
- package/dist/substrates/e2b/desktop-executor.d.ts +2 -2
- package/dist/substrates/e2b/desktop-executor.js +13 -13
- package/dist/substrates/e2b/desktop-geometry.d.ts +4 -4
- package/dist/substrates/e2b/desktop-geometry.js +8 -8
- package/dist/substrates/e2b/desktop-media.d.ts +4 -4
- package/dist/substrates/e2b/desktop-media.js +1 -1
- package/dist/substrates/e2b/desktop-screenshot-cleanup.d.ts +3 -3
- package/dist/substrates/e2b/desktop-screenshot-cleanup.js +3 -3
- package/dist/substrates/e2b/lifetime.js +1 -1
- package/dist/substrates/e2b/sandbox.d.ts +4 -4
- package/dist/substrates/e2b/sandbox.js +2 -2
- package/dist/substrates/e2b/sdk.d.ts +12 -12
- package/dist/substrates/e2b/sdk.js +7 -7
- package/dist/substrates/local/runtime-config.d.ts +4 -4
- package/dist/substrates/local/runtime-config.js +4 -4
- package/dist/tui/actions.d.ts +1 -1
- package/dist/tui/actions.js +13 -13
- package/dist/tui/contract.d.ts +23 -23
- package/dist/tui/contract.js +11 -11
- package/dist/tui/launch.d.ts +7 -7
- package/dist/tui/launch.js +36 -31
- package/dist/tui/project.js +5 -5
- package/dist/tui-app.js +139 -139
- package/dist/verify/actor.d.ts +8 -8
- package/dist/verify/actor.js +36 -36
- package/dist/verify/artifacts.d.ts +2 -2
- package/dist/verify/artifacts.js +3 -3
- package/dist/verify/context-growth.d.ts +9 -0
- package/dist/verify/context-growth.js +62 -0
- package/dist/verify/costs.d.ts +6 -6
- package/dist/verify/costs.js +8 -8
- package/dist/verify/shared-world-concurrent.d.ts +2 -2
- package/dist/verify/shared-world-concurrent.js +79 -76
- package/dist/verify/shared-world.d.ts +2 -2
- package/dist/verify/shared-world.js +16 -16
- package/dist/verify/subject.d.ts +5 -5
- package/dist/verify/subject.js +19 -19
- package/dist/verify/verify.js +94 -44
- package/docs/README.md +25 -32
- package/docs/architecture/actor-contract.md +60 -91
- package/docs/architecture/browser-control.md +1 -1
- package/docs/architecture/desktop-sessions.md +7 -7
- package/docs/architecture/external-public-shared-world.md +68 -65
- package/docs/architecture/github-feedback-loop.md +2 -5
- package/docs/architecture/guest-desktop.md +2 -2
- package/docs/architecture/observer.md +34 -34
- package/docs/architecture/restricted-codex-analysis.md +34 -28
- package/docs/architecture/serve.md +34 -37
- package/docs/architecture/state-driven-executor.md +55 -58
- package/docs/architecture/task-protocol-support.md +6 -6
- package/docs/architecture/terminal-product-route.md +70 -110
- package/docs/contracts/README.md +24 -0
- package/docs/contracts/adapter-admission.md +3 -3
- package/docs/contracts/adapter-fixtures.md +4 -7
- package/docs/contracts/core.md +1 -3
- package/docs/contracts/errors.md +62 -0
- package/docs/contracts/feedback.md +8 -9
- package/docs/contracts/policy.md +7 -12
- package/docs/contracts/run-bundle.md +42 -44
- package/docs/contracts/schemas.md +446 -368
- package/docs/contracts/study-analysis.md +7 -4
- package/docs/contracts/study-costs.md +28 -9
- package/docs/decisions/0001-run-bundle-is-the-source-of-truth.md +1 -1
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +2 -2
- package/docs/decisions/0003-contained-paths-and-cleanup-authority.md +3 -3
- package/docs/principles/actor-fidelity.md +4 -4
- package/docs/principles/engineering.md +3 -3
- package/docs/principles/invariants-and-defaults.md +38 -38
- package/docs/principles/self-driving-harness.md +1 -5
- package/docs/principles/three-roles.md +1 -1
- package/docs/product/automatic-analysis.md +7 -7
- package/examples/README.md +1 -1
- package/examples/participant/README.md +5 -5
- package/examples/participant/run.mjs +11 -10
- package/examples/scorer/README.md +3 -3
- package/examples/scorer/run.mjs +11 -12
- package/examples/scorer/scorer.mjs +3 -3
- package/package.json +16 -17
- package/skills/humanish/SKILL.md +64 -60
- package/skills/humanish/agents/openai.yaml +1 -1
- package/AGENTS.md +0 -99
- package/CONTRIBUTING.md +0 -148
- package/dist/actors/codex/qualified-versions.d.ts +0 -72
- package/dist/actors/codex/qualified-versions.js +0 -132
- package/dist/cli/commands/lab-format.d.ts +0 -8
- package/dist/cli/commands/lab-route-run.d.ts +0 -19
- package/dist/cli/commands/lab-route-scripted.d.ts +0 -17
- package/dist/cli/commands/lab-route-terminal.d.ts +0 -17
- package/dist/cli/commands/lab-run.d.ts +0 -9
- package/dist/cli/commands/lab.d.ts +0 -3
- package/dist/cli/commands/lab.js +0 -199
- package/dist/cli/renamed-options.d.ts +0 -11
- package/dist/cli/renamed-options.js +0 -18
- package/dist/lab/composition-rules.d.ts +0 -7
- package/dist/lab/config.d.ts +0 -7
- package/dist/lab/discover.d.ts +0 -65
- package/dist/lab/init-templates.js +0 -416
- package/dist/lab/keys.d.ts +0 -2
- package/dist/lab/parse/comms.d.ts +0 -12
- package/dist/lab/parse/execution.d.ts +0 -19
- package/dist/lab/parse/subject-state.d.ts +0 -30
- package/dist/lab/plan.d.ts +0 -19
- package/dist/lab/plan.js +0 -132
- package/dist/lab/preflight-probes.d.ts +0 -3
- package/dist/lab/preflight-result.d.ts +0 -7
- package/dist/lab/preflight.d.ts +0 -92
- package/dist/lab/route-inputs.d.ts +0 -28
- package/dist/lab/run-lab-options.d.ts +0 -28
- package/dist/lab/warnings.d.ts +0 -2
- package/dist/observer/static.d.ts +0 -39
- package/dist/observer/static.js +0 -369
- package/dist/run-lab.d.ts +0 -112
- package/docs/architecture/comms-inbox.md +0 -122
- package/docs/architecture/desktop-recording.md +0 -80
- package/docs/architecture/local-browser-runtime.md +0 -178
- package/docs/architecture/observer-review.md +0 -144
- package/docs/architecture/participant-media.md +0 -90
- package/docs/architecture/project-layout.md +0 -62
- package/docs/architecture/real-email-receiving.md +0 -136
- package/docs/assets/humanish-drawdb-hero.png +0 -0
- package/docs/assets/humanish-observer-hero.png +0 -0
- package/docs/goals/current.md +0 -207
- package/docs/product/lobby-trivia-3player-external-public.md +0 -77
- package/docs/product/open-source-install-experience.md +0 -240
- package/docs/ramp/README.md +0 -295
- package/docs/release/open-source-readiness.md +0 -222
- package/docs/release/public-readiness-standard.md +0 -210
- package/docs/roadmap/world-class-open-source-v0.md +0 -311
- /package/dist/{lab → study}/init-paths.d.ts +0 -0
- /package/dist/{lab → study}/init-paths.js +0 -0
- /package/dist/{lab → study}/init-plan.d.ts +0 -0
- /package/dist/{lab → study}/init-plan.js +0 -0
- /package/dist/{lab → study}/participant-assignment.d.ts +0 -0
- /package/dist/{lab → study}/participant-assignment.js +0 -0
- /package/dist/{lab → study}/persona.d.ts +0 -0
- /package/dist/{lab → study}/persona.js +0 -0
|
@@ -1,22 +1,19 @@
|
|
|
1
|
-
// The typed homes on
|
|
1
|
+
// The typed homes on RunStudyOptions. runStudyWith calls normalizeRunStudyOptions first: it refuses an option
|
|
2
2
|
// the route cannot honor and otherwise passes the typed options to the route, with env copied and
|
|
3
|
-
// onEvent turned into the emitter. The package's
|
|
3
|
+
// onEvent turned into the emitter. The package's runStudy refuses the removed bags
|
|
4
4
|
// (removedOptionRefusal).
|
|
5
5
|
import path from "node:path";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { TERMINAL_PRODUCT_LAB_SCHEMA } from "../routes/terminal/types.js";
|
|
10
|
-
import { isLocalBrowserLab } from "../substrates/local/runtime-config.js";
|
|
11
|
-
import { resolveLabDryRun } from "./plan.js";
|
|
12
|
-
import { knownSecretValues, labEventEmitter } from "./run-lab-events.js";
|
|
6
|
+
import { isLocalBrowserStudy } from "../substrates/local/runtime-config.js";
|
|
7
|
+
import { resolveStudyDryRun } from "./plan.js";
|
|
8
|
+
import { knownSecretValues, studyEventEmitter } from "./run-study-events.js";
|
|
13
9
|
import { rosterOf } from "./parse/actors.js";
|
|
10
|
+
import { studyResultIdentity } from "../run/study-result.js";
|
|
14
11
|
const unsupported = (option, route, reason) => ({
|
|
15
12
|
ok: false,
|
|
16
|
-
code: "
|
|
13
|
+
code: "HUMANISH_STUDY_OPTION_UNSUPPORTED",
|
|
17
14
|
message: `RunLabOptions.${option} is not supported on the ${route} route: ${reason}`,
|
|
18
15
|
});
|
|
19
|
-
// Fields
|
|
16
|
+
// Fields RunStudyOptions no longer has, and where each one's job went.
|
|
20
17
|
const REMOVED_OPTIONS = {
|
|
21
18
|
cuaHooks: "Use scorer, createProvider, inProcess, prepareDesktop, env, onEvent and onStream.",
|
|
22
19
|
scriptedHooks: "Use prepareDesktop and env.",
|
|
@@ -24,16 +21,17 @@ const REMOVED_OPTIONS = {
|
|
|
24
21
|
sharedWorldHooks: "Use scorer, prepareDesktop, env, onEvent and onStream.",
|
|
25
22
|
automaticAnalysis: "Use onEvent (analysis-started, analysis-finished) and analysisSignal.",
|
|
26
23
|
lab: "The humanish CLI sets it.",
|
|
24
|
+
study: "The humanish CLI sets it.",
|
|
27
25
|
scorerProvenance: "The humanish CLI sets it.",
|
|
28
26
|
};
|
|
29
|
-
/** The refusal for a field a JavaScript caller passed that
|
|
27
|
+
/** The refusal for a field a JavaScript caller passed that RunStudyOptions no longer has. */
|
|
30
28
|
export function removedOptionRefusal(options) {
|
|
31
29
|
// rerun.laneIds, the older name of rerun.participantIds, would otherwise be ignored, so the rerun
|
|
32
30
|
// would select every failed participant instead of the ones the caller named.
|
|
33
31
|
if (options.rerun !== undefined && Reflect.get(options.rerun, "laneIds") !== undefined)
|
|
34
32
|
return {
|
|
35
33
|
ok: false,
|
|
36
|
-
code: "
|
|
34
|
+
code: "HUMANISH_STUDY_OPTION_UNSUPPORTED",
|
|
37
35
|
message: 'RunLabOptions.rerun.laneIds was removed. Use rerun.participantIds. See docs/contracts/schemas.md, "Library options".',
|
|
38
36
|
};
|
|
39
37
|
const field = Object.keys(REMOVED_OPTIONS).find((key) => Reflect.get(options, key) !== undefined);
|
|
@@ -41,7 +39,7 @@ export function removedOptionRefusal(options) {
|
|
|
41
39
|
return undefined;
|
|
42
40
|
return {
|
|
43
41
|
ok: false,
|
|
44
|
-
code: "
|
|
42
|
+
code: "HUMANISH_STUDY_OPTION_UNSUPPORTED",
|
|
45
43
|
message: `RunLabOptions.${field} was removed. ${REMOVED_OPTIONS[field]} See docs/contracts/schemas.md, "Library options".`,
|
|
46
44
|
};
|
|
47
45
|
}
|
|
@@ -55,7 +53,7 @@ function unsupportedOption(config, route, options) {
|
|
|
55
53
|
return unsupported("scorer", route, "only computer use, shared world and terminal score runs.");
|
|
56
54
|
if (createProvider !== undefined && route !== "computer-use")
|
|
57
55
|
return unsupported("createProvider", route, route === "shared-world"
|
|
58
|
-
? "shared-world participants run the
|
|
56
|
+
? "shared-world participants run the study's own brain."
|
|
59
57
|
: "only computer use takes a caller brain.");
|
|
60
58
|
if (route === "computer-use") {
|
|
61
59
|
const source = config.subject.source;
|
|
@@ -64,7 +62,7 @@ function unsupportedOption(config, route, options) {
|
|
|
64
62
|
return unsupported("inProcess", route, "it drives an app-url or local-app subject.");
|
|
65
63
|
}
|
|
66
64
|
if (prepareDesktop !== undefined) {
|
|
67
|
-
if (
|
|
65
|
+
if (isLocalBrowserStudy(config))
|
|
68
66
|
return unsupported("prepareDesktop", route, "a local VM study has no E2B desktop.");
|
|
69
67
|
// An in-process run has no desktop to prepare.
|
|
70
68
|
if (inProcess !== undefined || source === "local-app")
|
|
@@ -86,7 +84,7 @@ function unsupportedOption(config, route, options) {
|
|
|
86
84
|
* Refuse what the route cannot honor, copy env, and build the emitter onEvent receives through.
|
|
87
85
|
* Nothing here touches the filesystem, so a refusal leaves no run directory, receipt or sandbox.
|
|
88
86
|
*/
|
|
89
|
-
export function
|
|
87
|
+
export function normalizeRunStudyOptions(config, route, options) {
|
|
90
88
|
const refused = unsupportedOption(config, route, options);
|
|
91
89
|
if (refused)
|
|
92
90
|
return refused;
|
|
@@ -95,7 +93,7 @@ export function normalizeRunLabOptions(config, route, options) {
|
|
|
95
93
|
// The route gets this copy, so it is the env a warning is scrubbed against, whatever the caller
|
|
96
94
|
// does to its own object afterwards.
|
|
97
95
|
const forwardedEnv = env === undefined ? undefined : { ...env };
|
|
98
|
-
const emit =
|
|
96
|
+
const emit = studyEventEmitter(onEvent, warnings, () => knownSecretValues(config, options, forwardedEnv));
|
|
99
97
|
// Every route reads its typed options directly, and onEvent reaches it as `emit`. unsupportedOption
|
|
100
98
|
// already refused an option no route reads; the preview route reads no env.
|
|
101
99
|
const normalized = { ...forwarded };
|
|
@@ -108,20 +106,26 @@ export function optionRefusalOutcome(config, route, options, refusal) {
|
|
|
108
106
|
const cwd = path.resolve(options.cwd);
|
|
109
107
|
const error = { code: refusal.code, message: refusal.message };
|
|
110
108
|
const actor = config.actors[0]?.type ?? "";
|
|
111
|
-
const dryRun =
|
|
109
|
+
const dryRun = resolveStudyDryRun(config, options.dryRun, true) ?? true;
|
|
112
110
|
const runId = options.runId ?? "not-created";
|
|
113
|
-
const common = { ok: false, cwd,
|
|
111
|
+
const common = { ok: false, cwd, actor, dryRun, runId, warnings: [], error };
|
|
114
112
|
switch (route) {
|
|
115
113
|
case "preview":
|
|
116
114
|
return {
|
|
117
115
|
route: "preview",
|
|
118
|
-
result: {
|
|
116
|
+
result: {
|
|
117
|
+
...studyResultIdentity("preview", config.id),
|
|
118
|
+
ok: false,
|
|
119
|
+
cwd,
|
|
120
|
+
warnings: [],
|
|
121
|
+
error,
|
|
122
|
+
},
|
|
119
123
|
};
|
|
120
124
|
case "computer-use":
|
|
121
125
|
return {
|
|
122
126
|
route: "computer-use",
|
|
123
127
|
result: {
|
|
124
|
-
|
|
128
|
+
...studyResultIdentity("computer-use", config.id),
|
|
125
129
|
...common,
|
|
126
130
|
ok: false,
|
|
127
131
|
appUrl: config.subject.appUrl ?? config.subject.serve?.url ?? "",
|
|
@@ -132,7 +136,7 @@ export function optionRefusalOutcome(config, route, options, refusal) {
|
|
|
132
136
|
return {
|
|
133
137
|
route: "scripted",
|
|
134
138
|
result: {
|
|
135
|
-
|
|
139
|
+
...studyResultIdentity("scripted", config.id),
|
|
136
140
|
...common,
|
|
137
141
|
ok: false,
|
|
138
142
|
appUrl: config.subject.appUrl ?? "",
|
|
@@ -143,7 +147,7 @@ export function optionRefusalOutcome(config, route, options, refusal) {
|
|
|
143
147
|
return {
|
|
144
148
|
route: "terminal",
|
|
145
149
|
result: {
|
|
146
|
-
|
|
150
|
+
...studyResultIdentity("terminal", config.id),
|
|
147
151
|
...common,
|
|
148
152
|
ok: false,
|
|
149
153
|
product: config.subject.product?.name ?? "",
|
|
@@ -154,7 +158,7 @@ export function optionRefusalOutcome(config, route, options, refusal) {
|
|
|
154
158
|
return {
|
|
155
159
|
route: "shared-world",
|
|
156
160
|
result: {
|
|
157
|
-
|
|
161
|
+
...studyResultIdentity("shared-world", config.id),
|
|
158
162
|
...common,
|
|
159
163
|
ok: false,
|
|
160
164
|
topology: "shared-world",
|
|
@@ -9,13 +9,8 @@ import type { DetachedTimers } from "../substrates/detached.js";
|
|
|
9
9
|
import type { E2BDesktopModule } from "../substrates/e2b/sdk.js";
|
|
10
10
|
import type { LocalTreeArchive } from "../subject/local-tree-archive.js";
|
|
11
11
|
import type { SubjectPhaseEvent } from "../subject/steps.js";
|
|
12
|
-
|
|
13
|
-
export interface
|
|
14
|
-
readonly laneId: string;
|
|
15
|
-
readonly laneIndex: number;
|
|
16
|
-
readonly laneCount: number;
|
|
17
|
-
}
|
|
18
|
-
export interface LabDeps {
|
|
12
|
+
import type { ParticipantRef } from "./run-study-events.js";
|
|
13
|
+
export interface StudyDeps {
|
|
19
14
|
/** Loads the E2B SDK. Defaults to loadE2BDesktopModule. */
|
|
20
15
|
readonly desktopModule?: () => Promise<E2BDesktopModule>;
|
|
21
16
|
/** Renders the Observer. Defaults to renderObserver. */
|
|
@@ -29,10 +24,10 @@ export interface LabDeps {
|
|
|
29
24
|
* computer-use phase names its participant; a shared-world plane's names none. onEvent still
|
|
30
25
|
* receives every phase.
|
|
31
26
|
*/
|
|
32
|
-
readonly subjectPhaseSink?: (event: SubjectPhaseEvent, participant?:
|
|
27
|
+
readonly subjectPhaseSink?: (event: SubjectPhaseEvent, participant?: ParticipantRef) => void;
|
|
33
28
|
/**
|
|
34
|
-
* Shared world: runs each
|
|
35
|
-
* reads whether it is set: a custom runner cannot enforce actors[0].maxOutputTokens.
|
|
29
|
+
* Shared world: runs each participant's computer-use session in place of the actor's own. The
|
|
30
|
+
* planner reads whether it is set: a custom runner cannot enforce actors[0].maxOutputTokens.
|
|
36
31
|
*/
|
|
37
32
|
readonly runSession?: (options: CuaActorSessionOptions) => Promise<CuaLoopResult>;
|
|
38
33
|
/**
|
|
@@ -50,13 +45,13 @@ export interface LabDeps {
|
|
|
50
45
|
/** Shared world: the background stateSeries prober's cadence in milliseconds. Defaults to 1000. */
|
|
51
46
|
readonly proberCadenceMs?: number;
|
|
52
47
|
/**
|
|
53
|
-
* Shared world, external-public plane: how long the host
|
|
48
|
+
* Shared world, external-public plane: how long the host has to surface a /lobby/CODE URL
|
|
54
49
|
* before the run fails closed. Defaults to 120000, capped by execution.timeoutMs.
|
|
55
50
|
*/
|
|
56
51
|
readonly handoffDeadlineMs?: number;
|
|
57
52
|
/**
|
|
58
|
-
* Shared world, external-public plane: reads a /lobby/CODE off a
|
|
59
|
-
* the single-frame OpenAI read.
|
|
53
|
+
* Shared world, external-public plane: reads a /lobby/CODE off a participant's screenshot.
|
|
54
|
+
* Defaults to the single-frame OpenAI read.
|
|
60
55
|
*/
|
|
61
56
|
readonly readLobbyCodeFromFrame?: (frame: Buffer, apiKey: string) => Promise<string | undefined>;
|
|
62
57
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Test seams: the implementations a test puts in place of the real ones. The internal
|
|
2
|
-
// them as its third argument and each route runner as `deps`; the package's
|
|
1
|
+
// Test seams: the implementations a test puts in place of the real ones. The internal runStudyWith takes
|
|
2
|
+
// them as its third argument and each route runner as `deps`; the package's runStudy takes none.
|
|
3
3
|
// Each field defaults to the real implementation.
|
|
4
4
|
export {};
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { type LocalRuntimeStatus } from "../substrates/local/runtime.js";
|
|
2
2
|
import { type KeyResolutionDeps } from "../keys/key-resolution.js";
|
|
3
|
-
export declare const
|
|
4
|
-
export interface
|
|
3
|
+
export declare const STUDY_SUMMARY_SCHEMA = "humanish.study-summary.v1";
|
|
4
|
+
export interface StudyCaps {
|
|
5
5
|
/** Per-participant blast-radius budget. */
|
|
6
6
|
laneUsd?: number;
|
|
7
7
|
/** Shared study budget across every participant. */
|
|
8
8
|
studyUsd?: number;
|
|
9
9
|
}
|
|
10
|
-
export interface
|
|
10
|
+
export interface StudySummary {
|
|
11
11
|
runtime?: Pick<LocalRuntimeStatus, "ok" | "installed" | "message">;
|
|
12
12
|
participantReadiness?: {
|
|
13
13
|
ok: boolean;
|
|
@@ -20,8 +20,9 @@ export interface LabSummary {
|
|
|
20
20
|
model: string;
|
|
21
21
|
maxCostUsd: number | null;
|
|
22
22
|
};
|
|
23
|
-
schema: typeof
|
|
24
|
-
|
|
23
|
+
schema: typeof STUDY_SUMMARY_SCHEMA;
|
|
24
|
+
/** The study's `id`. */
|
|
25
|
+
studyId: string;
|
|
25
26
|
title?: string;
|
|
26
27
|
description?: string;
|
|
27
28
|
/** "clone drawdb-io/drawdb", "app-url http://127.0.0.1:3000/", "this-repo". */
|
|
@@ -31,14 +32,14 @@ export interface LabSummary {
|
|
|
31
32
|
/** The model that will actually run, override or default. */
|
|
32
33
|
model?: string;
|
|
33
34
|
/**
|
|
34
|
-
* The reasoning effort that will actually run, override or default
|
|
35
|
+
* The reasoning effort that will actually run, override or default, and `"per-lane"` when the
|
|
35
36
|
* roster declares more than one, because a single value would be a lie about half the participants.
|
|
36
37
|
*
|
|
37
|
-
* Shown because it was a silent constant: unreachable from a
|
|
38
|
-
* default. A study variable you cannot see is one nobody chose
|
|
38
|
+
* Shown because it was a silent constant: unreachable from a study, so every run took the provider
|
|
39
|
+
* default. A study variable you cannot see is one nobody chose.
|
|
39
40
|
*/
|
|
40
41
|
reasoningEffort?: string;
|
|
41
|
-
caps:
|
|
42
|
+
caps: StudyCaps;
|
|
42
43
|
/**
|
|
43
44
|
* Whether the configured route's required keys resolve. Dry runs require none. This checks key
|
|
44
45
|
* presence, not local CLI authentication or provider validity. Undefined when not checked.
|
|
@@ -46,12 +47,12 @@ export interface LabSummary {
|
|
|
46
47
|
keysReady?: boolean;
|
|
47
48
|
missingKeys?: string[];
|
|
48
49
|
/**
|
|
49
|
-
* The planner's refusal message when a live key check finds the
|
|
50
|
-
* `humanish
|
|
50
|
+
* The planner's refusal message when a live key check finds the study will not plan, as
|
|
51
|
+
* `humanish run` would report it. `keysReady` is then unset: a refused plan lists no keys.
|
|
51
52
|
*/
|
|
52
53
|
planRefusal?: string;
|
|
53
54
|
}
|
|
54
|
-
export interface
|
|
55
|
+
export interface ReadStudySummaryOptions {
|
|
55
56
|
/** Skip the key probe (it touches vendor stores); the screen then shows no keys line. */
|
|
56
57
|
checkKeys?: boolean;
|
|
57
58
|
env?: NodeJS.ProcessEnv;
|
|
@@ -59,7 +60,7 @@ export interface ReadLabSummaryOptions {
|
|
|
59
60
|
keyDeps?: KeyResolutionDeps;
|
|
60
61
|
}
|
|
61
62
|
/**
|
|
62
|
-
* Describe one
|
|
63
|
-
* the
|
|
63
|
+
* Describe one study. Returns null when the manifest cannot be resolved: the caller already knows
|
|
64
|
+
* the study exists from the listing, so this failing means the file changed underneath them.
|
|
64
65
|
*/
|
|
65
|
-
export declare function
|
|
66
|
+
export declare function readStudySummary(cwd: string, study: string, options?: ReadStudySummaryOptions): Promise<StudySummary | null>;
|
|
@@ -1,28 +1,28 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { isLocalBrowserStudy } from "../substrates/local/runtime-config.js";
|
|
2
2
|
import { localRuntimeStatus } from "../substrates/local/runtime.js";
|
|
3
|
-
// What a
|
|
3
|
+
// What a study is, for the surface that has to describe it before you spend money.
|
|
4
4
|
//
|
|
5
|
-
// The run index and run detail answer questions about runs. This answers a question about the
|
|
6
|
-
// itself
|
|
5
|
+
// The run index and run detail answer questions about runs. This answers a question about the study
|
|
6
|
+
// itself (what it drives, who is in it, which model, and what it is allowed to spend), which is
|
|
7
7
|
// what a stakeholder reads on the screen where they decide whether to press Start.
|
|
8
8
|
//
|
|
9
9
|
// Resolved analysis defaults are shown independently of declared participant caps. A cap that is not
|
|
10
10
|
// declared is not "unlimited" and not "$0"; it is a line the screen does not draw.
|
|
11
|
-
import {
|
|
11
|
+
import { resolveStudyDryRun } from "./plan.js";
|
|
12
12
|
import { routeOf } from "./plan.js";
|
|
13
13
|
import { localCodexParticipantCheck, planCliRun } from "./doctor.js";
|
|
14
14
|
import { requiredKeys, requiredSubjectEnv } from "./requirements.js";
|
|
15
15
|
import { automaticAnalysisBudget } from "../analysis/automatic-config.js";
|
|
16
16
|
import { DEFAULT_OPENAI_CU_MODEL, DEFAULT_OPENAI_CU_REASONING_EFFORT, } from "../actors/computer-use/openai-provider.js";
|
|
17
|
-
import {
|
|
17
|
+
import { inspectStudyManifest } from "./discover.js";
|
|
18
18
|
import { isComputerUseComposition } from "./routing.js";
|
|
19
19
|
import { probeKeySources } from "../keys/key-resolution.js";
|
|
20
20
|
import { receivingRequiredKey } from "../comms/setup.js";
|
|
21
21
|
import { rosterOf } from "./parse/actors.js";
|
|
22
|
-
export const
|
|
22
|
+
export const STUDY_SUMMARY_SCHEMA = "humanish.study-summary.v1";
|
|
23
23
|
/**
|
|
24
24
|
* The effort every participant will run at, or `"per-lane"` when they differ. A participant that declares nothing
|
|
25
|
-
* inherits the actor's, and an actor that declares nothing gets the provider default
|
|
25
|
+
* inherits the actor's, and an actor that declares nothing gets the provider default, which is
|
|
26
26
|
* reported as the resolved value, exactly as `model` reports its default rather than hiding it.
|
|
27
27
|
*/
|
|
28
28
|
function reasoningEffortOf(config) {
|
|
@@ -35,7 +35,7 @@ function reasoningEffortOf(config) {
|
|
|
35
35
|
return "per-lane";
|
|
36
36
|
return resolved.size === 1 ? [...resolved][0] : fallback;
|
|
37
37
|
}
|
|
38
|
-
/** One short phrase for what the
|
|
38
|
+
/** One short phrase for what the study drives. */
|
|
39
39
|
function subjectOf(config) {
|
|
40
40
|
const subject = config.subject;
|
|
41
41
|
if (subject?.source === undefined)
|
|
@@ -47,7 +47,7 @@ function subjectOf(config) {
|
|
|
47
47
|
return `${subject.source} ${subject.appUrl}`;
|
|
48
48
|
return subject.source;
|
|
49
49
|
}
|
|
50
|
-
/** How many participants, and who
|
|
50
|
+
/** How many participants, and who; collapsed when they are all the same persona. */
|
|
51
51
|
function participantsOf(config) {
|
|
52
52
|
const actors = config.actors;
|
|
53
53
|
const actor = actors?.[0];
|
|
@@ -61,7 +61,7 @@ function participantsOf(config) {
|
|
|
61
61
|
const unique = [...new Set(personas)];
|
|
62
62
|
if (unique.length === 0)
|
|
63
63
|
return `${count} participant${count === 1 ? "" : "s"}`;
|
|
64
|
-
// Several participants of
|
|
64
|
+
// Several participants of one persona reads as "3 × skeptical-power-user"; genuinely different people
|
|
65
65
|
// are named, because which personas are in a study is the study's design.
|
|
66
66
|
return unique.length === 1 ? `${count} × ${unique[0]}` : unique.join(" · ");
|
|
67
67
|
}
|
|
@@ -76,11 +76,11 @@ function capsOf(config) {
|
|
|
76
76
|
};
|
|
77
77
|
}
|
|
78
78
|
/**
|
|
79
|
-
* Describe one
|
|
80
|
-
* the
|
|
79
|
+
* Describe one study. Returns null when the manifest cannot be resolved: the caller already knows
|
|
80
|
+
* the study exists from the listing, so this failing means the file changed underneath them.
|
|
81
81
|
*/
|
|
82
|
-
export async function
|
|
83
|
-
const inspected = await
|
|
82
|
+
export async function readStudySummary(cwd, study, options = {}) {
|
|
83
|
+
const inspected = await inspectStudyManifest(cwd, study).catch(() => null);
|
|
84
84
|
if (inspected === null || !inspected.ok || inspected.config === undefined)
|
|
85
85
|
return null;
|
|
86
86
|
const config = inspected.config;
|
|
@@ -88,8 +88,8 @@ export async function readLabSummary(cwd, lab, options = {}) {
|
|
|
88
88
|
const route = routeOf(inspected.config);
|
|
89
89
|
let keysReady;
|
|
90
90
|
let missingKeys;
|
|
91
|
-
const dryRun =
|
|
92
|
-
// A live key check reads the plan's requirements. A
|
|
91
|
+
const dryRun = resolveStudyDryRun(inspected.config, undefined, true) === true;
|
|
92
|
+
// A live key check reads the plan's requirements. A study the planner refuses has none to check,
|
|
93
93
|
// so the summary reports the refusal in place of its keys.
|
|
94
94
|
const planned = options.checkKeys === true && !dryRun ? await planCliRun(inspected.config, cwd) : undefined;
|
|
95
95
|
const planRefusal = planned?.ok === false ? planned.refusal.message : undefined;
|
|
@@ -129,7 +129,7 @@ export async function readLabSummary(cwd, lab, options = {}) {
|
|
|
129
129
|
const analysis = automaticAnalysisBudget(inspected.config.review?.analysis, route);
|
|
130
130
|
const subject = subjectOf(config);
|
|
131
131
|
const participants = participantsOf(config);
|
|
132
|
-
const runtime = options.checkKeys === true &&
|
|
132
|
+
const runtime = options.checkKeys === true && isLocalBrowserStudy(inspected.config)
|
|
133
133
|
? await localRuntimeStatus({
|
|
134
134
|
...(options.env ? { env: options.env } : {}),
|
|
135
135
|
media: inspected.config.execution?.desktop?.media !== undefined ||
|
|
@@ -137,7 +137,7 @@ export async function readLabSummary(cwd, lab, options = {}) {
|
|
|
137
137
|
})
|
|
138
138
|
: undefined;
|
|
139
139
|
const participantReadiness = options.checkKeys === true &&
|
|
140
|
-
|
|
140
|
+
isLocalBrowserStudy(inspected.config) &&
|
|
141
141
|
inspected.config.actors[0]?.type === "local-agent"
|
|
142
142
|
? await localCodexParticipantCheck({ env: options.env ?? process.env })
|
|
143
143
|
: undefined;
|
|
@@ -154,13 +154,13 @@ export async function readLabSummary(cwd, lab, options = {}) {
|
|
|
154
154
|
},
|
|
155
155
|
}
|
|
156
156
|
: {}),
|
|
157
|
-
schema:
|
|
157
|
+
schema: STUDY_SUMMARY_SCHEMA,
|
|
158
158
|
...(inspected.config.comms?.email?.kind === "real"
|
|
159
159
|
? {
|
|
160
160
|
communications: `Real email · ${inspected.config.comms.email.connection} · fresh inbox per participant · hosted processing · local review only`,
|
|
161
161
|
}
|
|
162
162
|
: {}),
|
|
163
|
-
|
|
163
|
+
studyId: String(config.id ?? study),
|
|
164
164
|
...(typeof config.title === "string" ? { title: config.title } : {}),
|
|
165
165
|
...(typeof config.description === "string" ? { description: config.description.trim() } : {}),
|
|
166
166
|
...(subject === undefined ? {} : { subject }),
|
|
@@ -4,20 +4,20 @@ declare const TASK_FUNNEL_SCHEMA: "humanish.task-funnel.v1";
|
|
|
4
4
|
* One task in a protocol. It has two halves that belong to two different people, and keeping them
|
|
5
5
|
* apart is the point of the type.
|
|
6
6
|
*
|
|
7
|
-
* `goal` belongs to the
|
|
7
|
+
* `goal` belongs to the participant. It is what they are asked to do, in their language, and it is
|
|
8
8
|
* the only half that reaches the prompt.
|
|
9
9
|
*
|
|
10
|
-
* `success` belongs to the
|
|
11
|
-
* never sees it
|
|
10
|
+
* `success` belongs to the researcher. It is how the task will be measured, and the participant
|
|
11
|
+
* never sees it. A moderator does not read the success criterion aloud, because telling someone
|
|
12
12
|
* how they will be judged changes what they do. A persona told "you succeed when the URL contains
|
|
13
13
|
* /dashboard" will go find that URL, which measures the instruction rather than the product.
|
|
14
14
|
*/
|
|
15
|
-
export interface
|
|
15
|
+
export interface StudyTask {
|
|
16
16
|
/** Stable id, used in evidence and in the funnel. Researcher-facing. */
|
|
17
17
|
id: string;
|
|
18
|
-
/**
|
|
18
|
+
/** Participant-facing. What they are asked to do. The only half that reaches the prompt. */
|
|
19
19
|
goal: string;
|
|
20
|
-
/**
|
|
20
|
+
/** Researcher-facing. Observation-shaped proof the task happened. Never rendered to the
|
|
21
21
|
* participant. Absent means the task is narrative-only: it is asked for but cannot be measured,
|
|
22
22
|
* and the funnel says so rather than quietly counting it as failed. */
|
|
23
23
|
success?: StopWhen;
|
|
@@ -38,12 +38,12 @@ export interface TaskFunnel {
|
|
|
38
38
|
total: number;
|
|
39
39
|
/** Tasks observed complete. */
|
|
40
40
|
completed: number;
|
|
41
|
-
/** Declared tasks with no `done` criterion
|
|
41
|
+
/** Declared tasks with no `done` criterion: shown to the participant, never observable. */
|
|
42
42
|
unobservable: number;
|
|
43
|
-
/** The first task that was
|
|
43
|
+
/** The first task that was not observed complete: where this participant stopped. */
|
|
44
44
|
stoppedAt?: string;
|
|
45
45
|
/** Per-task, in declaration order. `inputsObserved` is false when the task declares criteria
|
|
46
|
-
* whose inputs (url, text, appState)
|
|
46
|
+
* whose inputs (url, text, appState) never arrived in any observation this session, so the
|
|
47
47
|
* task was never actually measured. Absent when the task completed or is unobservable. */
|
|
48
48
|
tasks: Array<{
|
|
49
49
|
id: string;
|
|
@@ -53,22 +53,22 @@ export interface TaskFunnel {
|
|
|
53
53
|
inputsObserved?: boolean;
|
|
54
54
|
}>;
|
|
55
55
|
/** Observable tasks that were never measured, because the inputs their criteria need never
|
|
56
|
-
* arrived. Reported
|
|
57
|
-
* instrument, and reporting it as 0-completed blames the participant for it
|
|
56
|
+
* arrived. Reported separately from failures: a task nobody could observe is a gap in our
|
|
57
|
+
* instrument, and reporting it as 0-completed blames the participant for it. */
|
|
58
58
|
unmeasured: number;
|
|
59
59
|
}
|
|
60
60
|
/**
|
|
61
|
-
* Tracks task completion across a session. Stateful on purpose: a task completes
|
|
62
|
-
* observation that satisfies it, and stays complete even if the participant navigates away
|
|
61
|
+
* Tracks task completion across a session. Stateful on purpose: a task completes once, on the first
|
|
62
|
+
* observation that satisfies it, and stays complete even if the participant navigates away; you do
|
|
63
63
|
* not un-sign-up by going back to the home page.
|
|
64
64
|
*/
|
|
65
65
|
export declare class TaskTracker {
|
|
66
66
|
private readonly tasks;
|
|
67
67
|
private readonly completions;
|
|
68
68
|
/** Which observation fields were ever populated this session. A criterion whose field never
|
|
69
|
-
* arrived was never evaluated against anything, however many turns ran
|
|
69
|
+
* arrived was never evaluated against anything, however many turns ran. */
|
|
70
70
|
private readonly fieldsSeen;
|
|
71
|
-
constructor(tasks: readonly
|
|
71
|
+
constructor(tasks: readonly StudyTask[]);
|
|
72
72
|
/** Evaluate every still-incomplete task against one observation. Returns newly completed tasks. */
|
|
73
73
|
observe(observation: StopConditionObservation, turn: number): TaskCompletion[];
|
|
74
74
|
/** Which observation fields a task's criteria actually read. A task whose rules need `url` was
|
|
@@ -78,14 +78,14 @@ export declare class TaskTracker {
|
|
|
78
78
|
funnel(): TaskFunnel;
|
|
79
79
|
}
|
|
80
80
|
/**
|
|
81
|
-
* The task list as the
|
|
81
|
+
* The task list as the participant reads it: numbered, in order, in their own language.
|
|
82
82
|
*
|
|
83
83
|
* Reads `goal` and nothing else. The success criteria are the researcher's instrument and never
|
|
84
|
-
* appear here
|
|
84
|
+
* appear here: a participant who is told how they will be measured optimizes for the measurement,
|
|
85
85
|
* and the study stops being about the product. A test pins this, because it is the kind of leak a
|
|
86
86
|
* later convenience change makes without noticing.
|
|
87
87
|
*/
|
|
88
|
-
export declare function renderTaskPrompt(tasks: readonly
|
|
88
|
+
export declare function renderTaskPrompt(tasks: readonly StudyTask[]): string | undefined;
|
|
89
89
|
/** One line a stakeholder can read, with the denominator attached. */
|
|
90
90
|
export declare function formatTaskFunnel(funnel: TaskFunnel): string;
|
|
91
91
|
export {};
|
|
@@ -1,32 +1,32 @@
|
|
|
1
1
|
// Tasks: the researcher's protocol, expressed as config (docs/principles/three-roles.md).
|
|
2
2
|
//
|
|
3
|
-
// A
|
|
3
|
+
// A study could declare a prose `mission` and nothing else. That is a brief, not a protocol, and it
|
|
4
4
|
// left "where did people get stuck" answerable only from an actor's own narration, which is the one
|
|
5
5
|
// source a study should not have to take on faith.
|
|
6
6
|
//
|
|
7
|
-
// Real usability studies are built from discrete
|
|
7
|
+
// Real usability studies are built from discrete tasks, each with a written success criterion, and
|
|
8
8
|
// the result is a funnel: how far each participant got before they stopped. That funnel is the
|
|
9
9
|
// finding. A single pass/fail per participant throws it away.
|
|
10
10
|
//
|
|
11
|
-
// The criterion language is `stopWhen`, unchanged and already
|
|
11
|
+
// The criterion language is `stopWhen`, unchanged and already used by stop conditions: a task is
|
|
12
12
|
// done when an observation satisfies it. Reusing it means a task criterion is exactly as expressive
|
|
13
13
|
// as a stop condition, and an author who knows one knows the other.
|
|
14
14
|
//
|
|
15
|
-
// A task's completion is
|
|
15
|
+
// A task's completion is corroborated, not self-reported. The actor saying "I signed up" does not
|
|
16
16
|
// complete a task; the observed URL, page text, or app state does. That distinction is the whole
|
|
17
17
|
// reason to declare tasks at all.
|
|
18
18
|
import { evaluateStopWhen, } from "../actors/stop-conditions.js";
|
|
19
19
|
const TASK_FUNNEL_SCHEMA = "humanish.task-funnel.v1";
|
|
20
20
|
/**
|
|
21
|
-
* Tracks task completion across a session. Stateful on purpose: a task completes
|
|
22
|
-
* observation that satisfies it, and stays complete even if the participant navigates away
|
|
21
|
+
* Tracks task completion across a session. Stateful on purpose: a task completes once, on the first
|
|
22
|
+
* observation that satisfies it, and stays complete even if the participant navigates away; you do
|
|
23
23
|
* not un-sign-up by going back to the home page.
|
|
24
24
|
*/
|
|
25
25
|
export class TaskTracker {
|
|
26
26
|
tasks;
|
|
27
27
|
completions = new Map();
|
|
28
28
|
/** Which observation fields were ever populated this session. A criterion whose field never
|
|
29
|
-
* arrived was never evaluated against anything, however many turns ran
|
|
29
|
+
* arrived was never evaluated against anything, however many turns ran. */
|
|
30
30
|
fieldsSeen = new Set();
|
|
31
31
|
constructor(tasks) {
|
|
32
32
|
this.tasks = tasks;
|
|
@@ -77,7 +77,7 @@ export class TaskTracker {
|
|
|
77
77
|
const tasks = this.tasks.map((task) => {
|
|
78
78
|
const completion = this.completions.get(task.id);
|
|
79
79
|
const observable = task.success !== undefined;
|
|
80
|
-
// A task counts as measured when at least
|
|
80
|
+
// A task counts as measured when at least one field its criteria read was populated by some
|
|
81
81
|
// observation. `any` semantics: any satisfiable rule needed a field we actually saw.
|
|
82
82
|
const required = observable ? this.fieldsRequiredBy(task) : [];
|
|
83
83
|
const inputsObserved = required.length === 0 ? true : required.some((field) => this.fieldsSeen.has(field));
|
|
@@ -91,12 +91,12 @@ export class TaskTracker {
|
|
|
91
91
|
...(observable && completion === undefined ? { inputsObserved } : {}),
|
|
92
92
|
};
|
|
93
93
|
});
|
|
94
|
-
// Where they stopped is the first task not observed complete
|
|
94
|
+
// Where they stopped is the first task not observed complete, the thing a researcher reads
|
|
95
95
|
// first. An unobservable task cannot be "where they stopped", because nothing could have
|
|
96
96
|
// proven otherwise; skipping it avoids blaming a participant for a gap in the protocol.
|
|
97
97
|
// A task we never measured cannot be "where they stopped" either, for the same reason an
|
|
98
98
|
// unobservable one cannot: nothing could have proven otherwise, so naming it blames the
|
|
99
|
-
// participant for our gap
|
|
99
|
+
// participant for our gap.
|
|
100
100
|
const stoppedAt = tasks.find((task) => task.observable && !task.completed && task.inputsObserved !== false)?.id;
|
|
101
101
|
return {
|
|
102
102
|
schema: TASK_FUNNEL_SCHEMA,
|
|
@@ -110,10 +110,10 @@ export class TaskTracker {
|
|
|
110
110
|
}
|
|
111
111
|
}
|
|
112
112
|
/**
|
|
113
|
-
* The task list as the
|
|
113
|
+
* The task list as the participant reads it: numbered, in order, in their own language.
|
|
114
114
|
*
|
|
115
115
|
* Reads `goal` and nothing else. The success criteria are the researcher's instrument and never
|
|
116
|
-
* appear here
|
|
116
|
+
* appear here: a participant who is told how they will be measured optimizes for the measurement,
|
|
117
117
|
* and the study stops being about the product. A test pins this, because it is the kind of leak a
|
|
118
118
|
* later convenience change makes without noticing.
|
|
119
119
|
*/
|
|
@@ -131,9 +131,9 @@ export function formatTaskFunnel(funnel) {
|
|
|
131
131
|
const stopped = funnel.stoppedAt === undefined ? "" : `, stopped at "${funnel.stoppedAt}"`;
|
|
132
132
|
const unobservable = funnel.unobservable === 0 ? "" : `, ${funnel.unobservable} with no completion criterion`;
|
|
133
133
|
// Named separately from failures. "0/3 completed" alone reads as "no participant managed it",
|
|
134
|
-
// which is the wrong story when the criterion was never evaluated against anything
|
|
134
|
+
// which is the wrong story when the criterion was never evaluated against anything.
|
|
135
135
|
const unmeasured = funnel.unmeasured === 0
|
|
136
136
|
? ""
|
|
137
|
-
: `, ${funnel.unmeasured}
|
|
137
|
+
: `, ${funnel.unmeasured} never measured (the observations their criteria read never arrived)`;
|
|
138
138
|
return `${base}${stopped}${unobservable}${unmeasured}`;
|
|
139
139
|
}
|