humanish 0.107.0 → 0.108.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +22 -17
- package/CHANGELOG.md +14 -0
- package/CONTEXT.md +51 -44
- package/README.md +145 -172
- package/SECURITY.md +16 -26
- package/TELEMETRY.md +8 -6
- package/dist/actors/affordance.d.ts +4 -4
- package/dist/actors/affordance.js +12 -12
- package/dist/actors/agent-session.d.ts +2 -2
- package/dist/actors/agent-session.js +7 -7
- package/dist/actors/codex/app-server-actor-trace.d.ts +1 -1
- package/dist/actors/codex/app-server-actor-trace.js +1 -1
- package/dist/actors/codex/app-server-ui.js +1 -1
- package/dist/actors/codex/codex-admission.d.ts +91 -0
- package/dist/actors/codex/codex-admission.js +150 -0
- package/dist/actors/codex/restricted-executable.d.ts +4 -4
- package/dist/actors/codex/restricted-executable.js +8 -8
- package/dist/actors/codex/restricted-launch.d.ts +2 -1
- package/dist/actors/codex/restricted-launch.js +1 -1
- package/dist/actors/codex/restricted-participant.js +8 -6
- package/dist/actors/codex/restricted-policy.d.ts +2 -2
- package/dist/actors/codex/restricted-policy.js +2 -2
- package/dist/actors/codex/restricted-session.d.ts +2 -2
- package/dist/actors/codex/restricted-session.js +10 -5
- package/dist/actors/computer-use/actor.d.ts +16 -16
- package/dist/actors/computer-use/actor.js +2 -2
- package/dist/actors/computer-use/admission-limit.d.ts +1 -1
- package/dist/actors/computer-use/admission-limit.js +1 -1
- package/dist/actors/computer-use/loop/actions.d.ts +3 -3
- package/dist/actors/computer-use/loop/actions.js +10 -10
- package/dist/actors/computer-use/loop/backstop.d.ts +3 -3
- package/dist/actors/computer-use/loop/backstop.js +14 -12
- package/dist/actors/computer-use/loop/ending.d.ts +5 -5
- package/dist/actors/computer-use/loop/ending.js +7 -7
- package/dist/actors/computer-use/loop/observation.d.ts +2 -2
- package/dist/actors/computer-use/loop/observation.js +9 -9
- package/dist/actors/computer-use/loop/provider-call.d.ts +1 -1
- package/dist/actors/computer-use/loop/provider-call.js +1 -1
- package/dist/actors/computer-use/loop/session.js +3 -3
- package/dist/actors/computer-use/loop/spend.d.ts +1 -1
- package/dist/actors/computer-use/loop/spend.js +1 -1
- package/dist/actors/computer-use/loop/trace.js +9 -8
- package/dist/actors/computer-use/loop/types.d.ts +25 -23
- package/dist/actors/computer-use/loop/usage.js +2 -2
- package/dist/actors/computer-use/openai-context.d.ts +37 -0
- package/dist/actors/computer-use/openai-context.js +214 -0
- package/dist/actors/computer-use/openai-provider.d.ts +11 -8
- package/dist/actors/computer-use/openai-provider.js +45 -34
- package/dist/actors/computer-use/openai-requests.d.ts +36 -10
- package/dist/actors/computer-use/openai-requests.js +82 -15
- package/dist/actors/computer-use/openai-wire.d.ts +19 -7
- package/dist/actors/computer-use/openai-wire.js +36 -34
- package/dist/actors/contract.d.ts +78 -43
- package/dist/actors/contract.js +29 -23
- package/dist/actors/local-agent/claude-session.d.ts +1 -1
- package/dist/actors/local-agent/claude-session.js +10 -9
- package/dist/actors/local-agent/cli.d.ts +8 -6
- package/dist/actors/local-agent/cli.js +24 -28
- package/dist/actors/local-agent/readiness.d.ts +2 -2
- package/dist/actors/local-agent/readiness.js +4 -4
- package/dist/actors/reasoning-effort.js +6 -6
- package/dist/actors/registry.d.ts +10 -10
- package/dist/actors/registry.js +18 -17
- package/dist/actors/scripted-browser/actor.d.ts +8 -8
- package/dist/actors/scripted-browser/actor.js +16 -16
- package/dist/actors/scripted-browser/browser-command.js +1 -1
- package/dist/actors/scripted-browser/steps.js +1 -1
- package/dist/actors/scripted-browser/types.d.ts +1 -1
- package/dist/actors/stop-conditions.d.ts +1 -1
- package/dist/analysis/codex-config.d.ts +6 -2
- package/dist/analysis/codex-config.js +22 -14
- package/dist/analysis/service.js +5 -5
- package/dist/cli/commands/analyze.js +19 -16
- package/dist/cli/commands/codex.js +9 -9
- package/dist/cli/commands/comms.js +54 -33
- package/dist/cli/commands/feedback.js +20 -21
- package/dist/cli/commands/lab-format.d.ts +5 -4
- package/dist/cli/commands/lab-format.js +35 -23
- package/dist/cli/commands/lab-route-computer-use.js +6 -7
- package/dist/cli/commands/lab-route-preview.js +4 -3
- package/dist/cli/commands/lab-route-shared-world.js +3 -3
- package/dist/cli/commands/lab-run.js +10 -11
- package/dist/cli/commands/lab-scorer.d.ts +2 -2
- package/dist/cli/commands/lab-scorer.js +1 -1
- package/dist/cli/commands/migrate.d.ts +4 -0
- package/dist/cli/commands/migrate.js +65 -0
- package/dist/cli/commands/observe.js +147 -108
- package/dist/cli/commands/run-command.d.ts +10 -0
- package/dist/cli/commands/run-command.js +85 -0
- package/dist/cli/commands/runs.js +137 -138
- package/dist/cli/commands/setup.d.ts +2 -2
- package/dist/cli/commands/setup.js +26 -21
- package/dist/cli/commands/study.d.ts +3 -0
- package/dist/cli/commands/study.js +182 -0
- package/dist/cli/commands/tui.d.ts +4 -4
- package/dist/cli/commands/tui.js +21 -21
- package/dist/cli/commands/watch.js +60 -75
- package/dist/cli/deprecations.d.ts +18 -0
- package/dist/cli/deprecations.js +48 -0
- package/dist/cli/doctor.d.ts +15 -5
- package/dist/cli/doctor.js +103 -48
- package/dist/cli/first-run-path.d.ts +12 -10
- package/dist/cli/first-run-path.js +30 -28
- package/dist/cli/io.d.ts +41 -6
- package/dist/cli/io.js +97 -22
- package/dist/cli/observer-follow.d.ts +2 -2
- package/dist/cli/observer-follow.js +5 -6
- package/dist/cli/orientation.d.ts +6 -3
- package/dist/cli/orientation.js +43 -39
- package/dist/cli/product-sentence.d.ts +6 -0
- package/dist/cli/product-sentence.js +6 -0
- package/dist/cli/program.d.ts +6 -1
- package/dist/cli/program.js +109 -40
- package/dist/cli/telemetry.d.ts +15 -15
- package/dist/cli/telemetry.js +48 -40
- package/dist/cli.js +1 -1
- package/dist/comms/capture-surface.d.ts +13 -12
- package/dist/comms/capture-surface.js +23 -22
- package/dist/comms/catch-host.d.ts +7 -7
- package/dist/comms/catch-host.js +14 -14
- package/dist/comms/email-catch.d.ts +3 -3
- package/dist/comms/email-catch.js +8 -8
- package/dist/comms/external-evidence.js +2 -2
- package/dist/comms/extract.js +2 -2
- package/dist/comms/fake-inbox.d.ts +2 -2
- package/dist/comms/fake-inbox.js +2 -2
- package/dist/comms/receiving-render.d.ts +1 -1
- package/dist/comms/receiving-render.js +2 -2
- package/dist/comms/sandbox-catch-script.d.ts +6 -6
- package/dist/comms/sandbox-catch-script.js +6 -6
- package/dist/comms/sandbox-catch.d.ts +36 -37
- package/dist/comms/sandbox-catch.js +34 -35
- package/dist/comms/setup.js +42 -8
- package/dist/comms/thread-evidence.d.ts +1 -1
- package/dist/comms/thread-evidence.js +6 -6
- package/dist/comms/types.d.ts +6 -6
- package/dist/comms/types.js +3 -3
- package/dist/evidence/frame-signature.d.ts +1 -1
- package/dist/evidence/frame-signature.js +4 -4
- package/dist/evidence/redaction.d.ts +3 -3
- package/dist/evidence/redaction.js +4 -4
- package/dist/feedback/draft.js +36 -78
- package/dist/feedback/export-bundle.js +1 -1
- package/dist/feedback/export.d.ts +5 -1
- package/dist/feedback/export.js +7 -8
- package/dist/feedback/feedback.d.ts +2 -2
- package/dist/feedback/feedback.js +21 -3
- package/dist/guest/chromium-text.js +2 -2
- package/dist/guest/desktop-native.d.ts +1 -1
- package/dist/guest/runtime.js +1 -1
- package/dist/index.d.ts +12 -10
- package/dist/index.js +8 -6
- package/dist/keys/key-resolution.d.ts +5 -5
- package/dist/keys/key-resolution.js +23 -23
- package/dist/lab/adapter-extension.d.ts +7 -7
- package/dist/lab/adapter-extension.js +13 -36
- package/dist/lab/adapter-scorer-loader.d.ts +7 -7
- package/dist/lab/adapter-scorer-loader.js +32 -32
- package/dist/lab/composition-rules.js +42 -42
- package/dist/lab/config.js +65 -25
- package/dist/lab/deprecated-fields.d.ts +10 -0
- package/dist/lab/deprecated-fields.js +24 -0
- package/dist/lab/device-presets.d.ts +8 -8
- package/dist/lab/device-presets.js +11 -11
- package/dist/lab/discover.d.ts +13 -9
- package/dist/lab/discover.js +117 -57
- package/dist/lab/doctor.d.ts +10 -3
- package/dist/lab/doctor.js +47 -9
- package/dist/lab/init-templates.js +147 -147
- package/dist/lab/init.d.ts +1 -1
- package/dist/lab/init.js +41 -9
- package/dist/lab/keys.d.ts +2 -0
- package/dist/lab/keys.js +42 -8
- package/dist/lab/lab-deps.d.ts +7 -12
- package/dist/lab/parse/actors.js +9 -9
- package/dist/lab/parse/comms.js +5 -5
- package/dist/lab/parse/execution.js +13 -13
- package/dist/lab/parse/study-v3.d.ts +23 -0
- package/dist/lab/parse/study-v3.js +259 -0
- package/dist/lab/parse/subject-state.d.ts +7 -7
- package/dist/lab/parse/subject-state.js +16 -16
- package/dist/lab/parse/subject.js +28 -28
- package/dist/lab/parse/values.d.ts +1 -1
- package/dist/lab/parse/values.js +3 -3
- package/dist/lab/persona-resolve.d.ts +4 -4
- package/dist/lab/persona-resolve.js +4 -4
- package/dist/lab/plan-participants.d.ts +2 -2
- package/dist/lab/plan-types.d.ts +4 -4
- package/dist/lab/plan.d.ts +1 -7
- package/dist/lab/plan.js +10 -41
- package/dist/lab/preflight-probes.js +11 -11
- package/dist/lab/preflight-result.d.ts +2 -1
- package/dist/lab/preflight-result.js +5 -4
- package/dist/lab/preflight.d.ts +7 -5
- package/dist/lab/preflight.js +5 -5
- package/dist/lab/routing.d.ts +31 -25
- package/dist/lab/routing.js +60 -29
- package/dist/lab/run-lab-events.d.ts +0 -3
- package/dist/lab/run-lab-events.js +0 -6
- package/dist/lab/run-lab-homes.d.ts +22 -10
- package/dist/lab/run-lab-homes.js +5 -1
- package/dist/lab/run-lab-options.d.ts +1 -1
- package/dist/lab/run-lab-options.js +16 -13
- package/dist/lab/summary.d.ts +7 -6
- package/dist/lab/summary.js +10 -10
- package/dist/lab/tasks.d.ts +15 -15
- package/dist/lab/tasks.js +14 -14
- package/dist/lab/types.d.ts +101 -95
- package/dist/lab/types.js +2 -0
- package/dist/lab/validation.d.ts +13 -13
- package/dist/lab/validation.js +44 -42
- package/dist/lab/warnings.d.ts +10 -0
- package/dist/lab/warnings.js +61 -33
- package/dist/library-aliases.d.ts +56 -0
- package/dist/library-aliases.js +22 -0
- package/dist/observer/artifact.d.ts +1 -1
- package/dist/observer/artifact.js +6 -6
- package/dist/observer/data.d.ts +6 -6
- package/dist/observer/data.js +5 -4
- package/dist/observer/exposure.d.ts +1 -1
- package/dist/observer/exposure.js +11 -11
- package/dist/observer/http.js +1 -1
- package/dist/observer/library.d.ts +1 -1
- package/dist/observer/library.js +4 -2
- package/dist/observer/listen.js +5 -4
- package/dist/observer/render.d.ts +1 -1
- package/dist/observer/render.js +10 -5
- package/dist/observer/run-routes.d.ts +3 -3
- package/dist/observer/run-routes.js +1 -1
- package/dist/observer/serve.js +7 -3
- package/dist/observer/tunnel.js +1 -1
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle-parts.d.ts +9 -9
- package/dist/routes/computer-use/bundle-parts.js +15 -15
- package/dist/routes/computer-use/bundle.js +3 -3
- package/dist/routes/computer-use/e2b-desktop/comms.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/comms.js +14 -14
- package/dist/routes/computer-use/e2b-desktop/desktop.js +4 -4
- package/dist/routes/computer-use/e2b-desktop/fidelity.d.ts +4 -3
- package/dist/routes/computer-use/e2b-desktop/fidelity.js +11 -10
- package/dist/routes/computer-use/e2b-desktop/prepare.js +7 -7
- package/dist/routes/computer-use/e2b-desktop/start.js +9 -8
- package/dist/routes/computer-use/e2b-desktop/teardown.js +5 -4
- package/dist/routes/computer-use/external-comms.d.ts +1 -1
- package/dist/routes/computer-use/external-comms.js +1 -1
- package/dist/routes/computer-use/fanout-bundle.js +18 -18
- package/dist/routes/computer-use/fanout-records.js +10 -11
- package/dist/routes/computer-use/live-flush.d.ts +1 -1
- package/dist/routes/computer-use/live-flush.js +6 -6
- package/dist/routes/computer-use/live-phase.d.ts +4 -2
- package/dist/routes/computer-use/live-phase.js +6 -6
- package/dist/routes/computer-use/local-vm.js +1 -1
- package/dist/routes/computer-use/participant-desktop.d.ts +4 -4
- package/dist/routes/computer-use/participant-desktop.js +4 -4
- package/dist/routes/computer-use/participant-execution.d.ts +4 -4
- package/dist/routes/computer-use/participant-execution.js +9 -9
- package/dist/routes/computer-use/participant-model.d.ts +6 -0
- package/dist/routes/computer-use/participant-model.js +31 -13
- package/dist/routes/computer-use/participant-prompt.d.ts +11 -11
- package/dist/routes/computer-use/participant-prompt.js +6 -6
- package/dist/routes/computer-use/participant-runs.d.ts +7 -7
- package/dist/routes/computer-use/participant-runs.js +13 -13
- package/dist/routes/computer-use/plan.js +22 -22
- package/dist/routes/computer-use/preflight.js +21 -21
- package/dist/routes/computer-use/rerun-selection.js +1 -1
- package/dist/routes/computer-use/result.js +11 -11
- package/dist/routes/computer-use/route.d.ts +2 -1
- package/dist/routes/computer-use/route.js +15 -14
- package/dist/routes/computer-use/self-report.d.ts +7 -7
- package/dist/routes/computer-use/self-report.js +31 -31
- package/dist/routes/computer-use/setup.d.ts +1 -1
- package/dist/routes/computer-use/setup.js +17 -18
- package/dist/routes/computer-use/single-bundle.d.ts +7 -7
- package/dist/routes/computer-use/single-bundle.js +28 -30
- package/dist/routes/computer-use/subject-projection.d.ts +1 -1
- package/dist/routes/computer-use/subject-projection.js +6 -6
- package/dist/routes/computer-use/types.d.ts +40 -42
- package/dist/routes/computer-use/types.js +10 -10
- package/dist/routes/preview.d.ts +8 -2
- package/dist/routes/preview.js +8 -4
- package/dist/routes/scripted/bundle.d.ts +2 -2
- package/dist/routes/scripted/bundle.js +13 -12
- package/dist/routes/scripted/plan.d.ts +1 -1
- package/dist/routes/scripted/plan.js +9 -9
- package/dist/routes/scripted/result.js +5 -5
- package/dist/routes/scripted/route.js +5 -6
- package/dist/routes/scripted/scenario.js +2 -2
- package/dist/routes/scripted/setup.js +7 -7
- package/dist/routes/scripted/subject.js +1 -1
- package/dist/routes/scripted/surface-records.js +3 -3
- package/dist/routes/scripted/types.d.ts +3 -5
- package/dist/routes/scripted/types.js +1 -1
- package/dist/routes/shared-world/bundle-records.js +9 -8
- package/dist/routes/shared-world/bundle.d.ts +5 -5
- package/dist/routes/shared-world/bundle.js +40 -38
- package/dist/routes/shared-world/checkpoints.d.ts +1 -1
- package/dist/routes/shared-world/checkpoints.js +1 -1
- package/dist/routes/shared-world/comms.d.ts +5 -5
- package/dist/routes/shared-world/comms.js +14 -14
- package/dist/routes/shared-world/external-public.d.ts +4 -3
- package/dist/routes/shared-world/external-public.js +16 -12
- package/dist/routes/shared-world/handoff.d.ts +7 -6
- package/dist/routes/shared-world/handoff.js +65 -57
- package/dist/routes/shared-world/lobby-code.d.ts +2 -2
- package/dist/routes/shared-world/lobby-code.js +9 -9
- package/dist/routes/shared-world/participant-specs.d.ts +7 -6
- package/dist/routes/shared-world/participant-specs.js +31 -24
- package/dist/routes/shared-world/plan.js +7 -7
- package/dist/routes/shared-world/provenance.d.ts +2 -2
- package/dist/routes/shared-world/provenance.js +2 -2
- package/dist/routes/shared-world/provisioned.js +14 -13
- package/dist/routes/shared-world/result.js +20 -21
- package/dist/routes/shared-world/route.js +23 -22
- package/dist/routes/shared-world/setup.d.ts +3 -3
- package/dist/routes/shared-world/setup.js +4 -4
- package/dist/routes/shared-world/types.d.ts +30 -28
- package/dist/routes/shared-world/types.js +4 -4
- package/dist/routes/terminal/adapter.d.ts +3 -3
- package/dist/routes/terminal/adapter.js +5 -5
- package/dist/routes/terminal/bundle.d.ts +3 -3
- package/dist/routes/terminal/bundle.js +22 -21
- package/dist/routes/terminal/credentials.d.ts +4 -4
- package/dist/routes/terminal/credentials.js +7 -7
- package/dist/routes/terminal/dry-run.js +4 -4
- package/dist/routes/terminal/encoding.js +2 -2
- package/dist/routes/terminal/ledger.d.ts +3 -3
- package/dist/routes/terminal/ledger.js +14 -14
- package/dist/routes/terminal/live-finish.js +3 -3
- package/dist/routes/terminal/live-sandbox.js +12 -13
- package/dist/routes/terminal/persona.js +1 -1
- package/dist/routes/terminal/plan.js +11 -11
- package/dist/routes/terminal/recorder.js +2 -2
- package/dist/routes/terminal/result.js +8 -8
- package/dist/routes/terminal/route.js +6 -7
- package/dist/routes/terminal/sandbox.d.ts +1 -1
- package/dist/routes/terminal/sandbox.js +1 -1
- package/dist/routes/terminal/session.js +8 -7
- package/dist/routes/terminal/token-usage.d.ts +1 -1
- package/dist/routes/terminal/token-usage.js +7 -8
- package/dist/routes/terminal/trace.d.ts +2 -2
- package/dist/routes/terminal/trace.js +2 -2
- package/dist/routes/terminal/types.d.ts +26 -28
- package/dist/routes/terminal/types.js +3 -3
- package/dist/run/bundle-shape.js +3 -3
- package/dist/run/bundle.d.ts +46 -46
- package/dist/run/bundle.js +2 -1
- package/dist/run/cost-summary.js +3 -3
- package/dist/run/costs.js +7 -1
- package/dist/run/detail.d.ts +6 -6
- package/dist/run/detail.js +4 -4
- package/dist/run/dry-run-inputs.js +1 -1
- package/dist/run/dry-run.js +18 -18
- package/dist/run/judge.d.ts +19 -13
- package/dist/run/judge.js +18 -8
- package/dist/run/paths.d.ts +1 -1
- package/dist/run/paths.js +1 -1
- package/dist/run/pricing.d.ts +11 -6
- package/dist/run/pricing.js +20 -13
- package/dist/run/projection.d.ts +14 -14
- package/dist/run/projection.js +15 -15
- package/dist/run/reclaim.js +2 -2
- package/dist/run/results.d.ts +5 -5
- package/dist/run/run-index.d.ts +8 -6
- package/dist/run/run-index.js +15 -16
- package/dist/run/run-not-found.d.ts +5 -0
- package/dist/run/run-not-found.js +15 -0
- package/dist/run/run.d.ts +5 -4
- package/dist/run/run.js +1 -1
- package/dist/run/sandbox-receipts.d.ts +2 -2
- package/dist/run/sandbox-receipts.js +3 -3
- package/dist/run/shared-world-evidence.d.ts +36 -35
- package/dist/run/shared-world-evidence.js +4 -4
- package/dist/run/shared-world-shape.d.ts +1 -1
- package/dist/run/shared-world-shape.js +6 -6
- package/dist/run/stats.d.ts +12 -6
- package/dist/run/stats.js +116 -58
- package/dist/run/status.d.ts +8 -29
- package/dist/run/status.js +5 -27
- package/dist/run/stored-runs.js +2 -1
- package/dist/run/stream-shape.js +1 -1
- package/dist/run/streams.d.ts +6 -6
- package/dist/run/study-provenance.d.ts +47 -0
- package/dist/run/study-provenance.js +63 -0
- package/dist/run/study-result.d.ts +16 -0
- package/dist/run/study-result.js +9 -0
- package/dist/run/text.d.ts +2 -0
- package/dist/run/text.js +4 -0
- package/dist/run-lab.d.ts +7 -7
- package/dist/run-lab.js +1 -3
- package/dist/study/convert.d.ts +33 -0
- package/dist/study/convert.js +432 -0
- package/dist/study/field-paths.d.ts +12 -0
- package/dist/study/field-paths.js +126 -0
- package/dist/study/files.d.ts +26 -0
- package/dist/study/files.js +43 -0
- package/dist/study/migrate.d.ts +48 -0
- package/dist/study/migrate.js +408 -0
- package/dist/study/owned-files.d.ts +38 -0
- package/dist/study/owned-files.js +182 -0
- package/dist/subject/clone.d.ts +1 -1
- package/dist/subject/desktop-cli.d.ts +1 -1
- package/dist/subject/desktop-cli.js +2 -2
- package/dist/subject/local-tree-archive.js +1 -1
- package/dist/subject/local-tree.d.ts +1 -1
- package/dist/subject/node-bootstrap.js +1 -1
- package/dist/subject/runtime.js +8 -9
- package/dist/subject/serve.d.ts +1 -1
- package/dist/subject/serve.js +3 -3
- package/dist/subject/state.d.ts +1 -1
- package/dist/subject/steps.d.ts +5 -5
- package/dist/subject/steps.js +4 -4
- package/dist/substrates/command-failure.d.ts +3 -3
- package/dist/substrates/command-failure.js +3 -3
- package/dist/substrates/detached.d.ts +6 -6
- package/dist/substrates/detached.js +7 -7
- package/dist/substrates/e2b/cdp-probe.d.ts +9 -9
- package/dist/substrates/e2b/cdp-probe.js +14 -17
- package/dist/substrates/e2b/desktop-browser.d.ts +2 -2
- package/dist/substrates/e2b/desktop-browser.js +8 -8
- package/dist/substrates/e2b/desktop-cdp.d.ts +9 -9
- package/dist/substrates/e2b/desktop-cdp.js +8 -8
- package/dist/substrates/e2b/desktop-executor.d.ts +1 -1
- package/dist/substrates/e2b/desktop-executor.js +1 -1
- package/dist/substrates/e2b/desktop-geometry.d.ts +3 -3
- package/dist/substrates/e2b/desktop-geometry.js +7 -7
- package/dist/substrates/e2b/desktop-media.d.ts +1 -1
- package/dist/substrates/e2b/desktop-media.js +1 -1
- package/dist/substrates/e2b/desktop-screenshot-cleanup.d.ts +3 -3
- package/dist/substrates/e2b/desktop-screenshot-cleanup.js +3 -3
- package/dist/substrates/e2b/sandbox.d.ts +1 -1
- package/dist/substrates/e2b/sandbox.js +1 -1
- package/dist/substrates/e2b/sdk.d.ts +11 -11
- package/dist/substrates/e2b/sdk.js +6 -6
- package/dist/tui/actions.d.ts +1 -1
- package/dist/tui/actions.js +13 -13
- package/dist/tui/contract.d.ts +16 -16
- package/dist/tui/contract.js +10 -10
- package/dist/tui/launch.d.ts +4 -4
- package/dist/tui/launch.js +18 -19
- package/dist/tui/project.js +5 -5
- package/dist/tui-app.js +82 -82
- package/dist/verify/actor.d.ts +8 -8
- package/dist/verify/actor.js +36 -36
- package/dist/verify/artifacts.d.ts +2 -2
- package/dist/verify/artifacts.js +3 -3
- package/dist/verify/context-growth.d.ts +9 -0
- package/dist/verify/context-growth.js +62 -0
- package/dist/verify/costs.d.ts +6 -6
- package/dist/verify/costs.js +8 -8
- package/dist/verify/shared-world-concurrent.d.ts +2 -2
- package/dist/verify/shared-world-concurrent.js +79 -76
- package/dist/verify/shared-world.d.ts +2 -2
- package/dist/verify/shared-world.js +16 -16
- package/dist/verify/subject.d.ts +4 -4
- package/dist/verify/subject.js +18 -18
- package/dist/verify/verify.js +92 -43
- package/docs/README.md +25 -32
- package/docs/architecture/actor-contract.md +56 -87
- package/docs/architecture/desktop-sessions.md +7 -7
- package/docs/architecture/external-public-shared-world.md +61 -59
- package/docs/architecture/github-feedback-loop.md +1 -4
- package/docs/architecture/guest-desktop.md +2 -2
- package/docs/architecture/observer.md +34 -34
- package/docs/architecture/restricted-codex-analysis.md +34 -28
- package/docs/architecture/serve.md +34 -37
- package/docs/architecture/state-driven-executor.md +55 -58
- package/docs/architecture/task-protocol-support.md +5 -5
- package/docs/architecture/terminal-product-route.md +67 -107
- package/docs/contracts/README.md +24 -0
- package/docs/contracts/adapter-admission.md +3 -3
- package/docs/contracts/adapter-fixtures.md +2 -5
- package/docs/contracts/core.md +1 -3
- package/docs/contracts/errors.md +62 -0
- package/docs/contracts/feedback.md +8 -9
- package/docs/contracts/policy.md +7 -12
- package/docs/contracts/run-bundle.md +45 -44
- package/docs/contracts/schemas.md +421 -362
- package/docs/contracts/study-analysis.md +3 -3
- package/docs/contracts/study-costs.md +11 -7
- package/docs/decisions/0001-run-bundle-is-the-source-of-truth.md +1 -1
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +2 -2
- package/docs/decisions/0003-contained-paths-and-cleanup-authority.md +3 -3
- package/docs/principles/actor-fidelity.md +4 -4
- package/docs/principles/engineering.md +3 -3
- package/docs/principles/invariants-and-defaults.md +38 -38
- package/docs/principles/self-driving-harness.md +1 -5
- package/docs/principles/three-roles.md +1 -1
- package/docs/product/automatic-analysis.md +7 -7
- package/examples/participant/README.md +5 -5
- package/examples/participant/run.mjs +11 -10
- package/examples/scorer/README.md +3 -3
- package/examples/scorer/run.mjs +11 -12
- package/package.json +16 -17
- package/skills/humanish/SKILL.md +60 -58
- package/skills/humanish/agents/openai.yaml +1 -1
- package/AGENTS.md +0 -99
- package/CONTRIBUTING.md +0 -148
- package/dist/actors/codex/qualified-versions.d.ts +0 -72
- package/dist/actors/codex/qualified-versions.js +0 -132
- package/dist/cli/commands/lab.d.ts +0 -3
- package/dist/cli/commands/lab.js +0 -199
- package/dist/cli/renamed-options.d.ts +0 -11
- package/dist/cli/renamed-options.js +0 -18
- package/dist/observer/static.d.ts +0 -39
- package/dist/observer/static.js +0 -369
- package/docs/architecture/comms-inbox.md +0 -122
- package/docs/architecture/desktop-recording.md +0 -80
- package/docs/architecture/local-browser-runtime.md +0 -178
- package/docs/architecture/observer-review.md +0 -144
- package/docs/architecture/participant-media.md +0 -90
- package/docs/architecture/project-layout.md +0 -62
- package/docs/architecture/real-email-receiving.md +0 -136
- package/docs/assets/humanish-drawdb-hero.png +0 -0
- package/docs/assets/humanish-observer-hero.png +0 -0
- package/docs/goals/current.md +0 -207
- package/docs/product/lobby-trivia-3player-external-public.md +0 -77
- package/docs/product/open-source-install-experience.md +0 -240
- package/docs/ramp/README.md +0 -295
- package/docs/release/open-source-readiness.md +0 -222
- package/docs/release/public-readiness-standard.md +0 -210
- package/docs/roadmap/world-class-open-source-v0.md +0 -311
|
@@ -3,17 +3,14 @@
|
|
|
3
3
|
// onEvent turned into the emitter. The package's runLab refuses the removed bags
|
|
4
4
|
// (removedOptionRefusal).
|
|
5
5
|
import path from "node:path";
|
|
6
|
-
import { CUA_ACTOR_LAB_SCHEMA } from "../routes/computer-use/types.js";
|
|
7
|
-
import { SCRIPTED_BROWSER_LAB_SCHEMA } from "../routes/scripted/types.js";
|
|
8
|
-
import { CONCURRENT_SHARED_WORLD_LAB_SCHEMA } from "../routes/shared-world/types.js";
|
|
9
|
-
import { TERMINAL_PRODUCT_LAB_SCHEMA } from "../routes/terminal/types.js";
|
|
10
6
|
import { isLocalBrowserLab } from "../substrates/local/runtime-config.js";
|
|
11
7
|
import { resolveLabDryRun } from "./plan.js";
|
|
12
8
|
import { knownSecretValues, labEventEmitter } from "./run-lab-events.js";
|
|
13
9
|
import { rosterOf } from "./parse/actors.js";
|
|
10
|
+
import { studyResultIdentity } from "../run/study-result.js";
|
|
14
11
|
const unsupported = (option, route, reason) => ({
|
|
15
12
|
ok: false,
|
|
16
|
-
code: "
|
|
13
|
+
code: "HUMANISH_STUDY_OPTION_UNSUPPORTED",
|
|
17
14
|
message: `RunLabOptions.${option} is not supported on the ${route} route: ${reason}`,
|
|
18
15
|
});
|
|
19
16
|
// Fields RunLabOptions no longer has, and where each one's job went.
|
|
@@ -33,7 +30,7 @@ export function removedOptionRefusal(options) {
|
|
|
33
30
|
if (options.rerun !== undefined && Reflect.get(options.rerun, "laneIds") !== undefined)
|
|
34
31
|
return {
|
|
35
32
|
ok: false,
|
|
36
|
-
code: "
|
|
33
|
+
code: "HUMANISH_STUDY_OPTION_UNSUPPORTED",
|
|
37
34
|
message: 'RunLabOptions.rerun.laneIds was removed. Use rerun.participantIds. See docs/contracts/schemas.md, "Library options".',
|
|
38
35
|
};
|
|
39
36
|
const field = Object.keys(REMOVED_OPTIONS).find((key) => Reflect.get(options, key) !== undefined);
|
|
@@ -41,7 +38,7 @@ export function removedOptionRefusal(options) {
|
|
|
41
38
|
return undefined;
|
|
42
39
|
return {
|
|
43
40
|
ok: false,
|
|
44
|
-
code: "
|
|
41
|
+
code: "HUMANISH_STUDY_OPTION_UNSUPPORTED",
|
|
45
42
|
message: `RunLabOptions.${field} was removed. ${REMOVED_OPTIONS[field]} See docs/contracts/schemas.md, "Library options".`,
|
|
46
43
|
};
|
|
47
44
|
}
|
|
@@ -110,18 +107,24 @@ export function optionRefusalOutcome(config, route, options, refusal) {
|
|
|
110
107
|
const actor = config.actors[0]?.type ?? "";
|
|
111
108
|
const dryRun = resolveLabDryRun(config, options.dryRun, true) ?? true;
|
|
112
109
|
const runId = options.runId ?? "not-created";
|
|
113
|
-
const common = { ok: false, cwd,
|
|
110
|
+
const common = { ok: false, cwd, actor, dryRun, runId, warnings: [], error };
|
|
114
111
|
switch (route) {
|
|
115
112
|
case "preview":
|
|
116
113
|
return {
|
|
117
114
|
route: "preview",
|
|
118
|
-
result: {
|
|
115
|
+
result: {
|
|
116
|
+
...studyResultIdentity("preview", config.id),
|
|
117
|
+
ok: false,
|
|
118
|
+
cwd,
|
|
119
|
+
warnings: [],
|
|
120
|
+
error,
|
|
121
|
+
},
|
|
119
122
|
};
|
|
120
123
|
case "computer-use":
|
|
121
124
|
return {
|
|
122
125
|
route: "computer-use",
|
|
123
126
|
result: {
|
|
124
|
-
|
|
127
|
+
...studyResultIdentity("computer-use", config.id),
|
|
125
128
|
...common,
|
|
126
129
|
ok: false,
|
|
127
130
|
appUrl: config.subject.appUrl ?? config.subject.serve?.url ?? "",
|
|
@@ -132,7 +135,7 @@ export function optionRefusalOutcome(config, route, options, refusal) {
|
|
|
132
135
|
return {
|
|
133
136
|
route: "scripted",
|
|
134
137
|
result: {
|
|
135
|
-
|
|
138
|
+
...studyResultIdentity("scripted", config.id),
|
|
136
139
|
...common,
|
|
137
140
|
ok: false,
|
|
138
141
|
appUrl: config.subject.appUrl ?? "",
|
|
@@ -143,7 +146,7 @@ export function optionRefusalOutcome(config, route, options, refusal) {
|
|
|
143
146
|
return {
|
|
144
147
|
route: "terminal",
|
|
145
148
|
result: {
|
|
146
|
-
|
|
149
|
+
...studyResultIdentity("terminal", config.id),
|
|
147
150
|
...common,
|
|
148
151
|
ok: false,
|
|
149
152
|
product: config.subject.product?.name ?? "",
|
|
@@ -154,7 +157,7 @@ export function optionRefusalOutcome(config, route, options, refusal) {
|
|
|
154
157
|
return {
|
|
155
158
|
route: "shared-world",
|
|
156
159
|
result: {
|
|
157
|
-
|
|
160
|
+
...studyResultIdentity("shared-world", config.id),
|
|
158
161
|
...common,
|
|
159
162
|
ok: false,
|
|
160
163
|
topology: "shared-world",
|
package/dist/lab/summary.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type LocalRuntimeStatus } from "../substrates/local/runtime.js";
|
|
2
2
|
import { type KeyResolutionDeps } from "../keys/key-resolution.js";
|
|
3
|
-
export declare const
|
|
3
|
+
export declare const STUDY_SUMMARY_SCHEMA = "humanish.study-summary.v1";
|
|
4
4
|
export interface LabCaps {
|
|
5
5
|
/** Per-participant blast-radius budget. */
|
|
6
6
|
laneUsd?: number;
|
|
@@ -20,8 +20,9 @@ export interface LabSummary {
|
|
|
20
20
|
model: string;
|
|
21
21
|
maxCostUsd: number | null;
|
|
22
22
|
};
|
|
23
|
-
schema: typeof
|
|
24
|
-
|
|
23
|
+
schema: typeof STUDY_SUMMARY_SCHEMA;
|
|
24
|
+
/** The study's `id`. */
|
|
25
|
+
studyId: string;
|
|
25
26
|
title?: string;
|
|
26
27
|
description?: string;
|
|
27
28
|
/** "clone drawdb-io/drawdb", "app-url http://127.0.0.1:3000/", "this-repo". */
|
|
@@ -31,11 +32,11 @@ export interface LabSummary {
|
|
|
31
32
|
/** The model that will actually run, override or default. */
|
|
32
33
|
model?: string;
|
|
33
34
|
/**
|
|
34
|
-
* The reasoning effort that will actually run, override or default
|
|
35
|
+
* The reasoning effort that will actually run, override or default, and `"per-lane"` when the
|
|
35
36
|
* roster declares more than one, because a single value would be a lie about half the participants.
|
|
36
37
|
*
|
|
37
38
|
* Shown because it was a silent constant: unreachable from a lab, so every run took the provider
|
|
38
|
-
* default. A study variable you cannot see is one nobody chose
|
|
39
|
+
* default. A study variable you cannot see is one nobody chose.
|
|
39
40
|
*/
|
|
40
41
|
reasoningEffort?: string;
|
|
41
42
|
caps: LabCaps;
|
|
@@ -59,7 +60,7 @@ export interface ReadLabSummaryOptions {
|
|
|
59
60
|
keyDeps?: KeyResolutionDeps;
|
|
60
61
|
}
|
|
61
62
|
/**
|
|
62
|
-
* Describe one lab. Returns null when the manifest cannot be resolved
|
|
63
|
+
* Describe one lab. Returns null when the manifest cannot be resolved: the caller already knows
|
|
63
64
|
* the lab exists from the listing, so this failing means the file changed underneath them.
|
|
64
65
|
*/
|
|
65
66
|
export declare function readLabSummary(cwd: string, lab: string, options?: ReadLabSummaryOptions): Promise<LabSummary | null>;
|
package/dist/lab/summary.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { isLocalBrowserLab } from "../substrates/local/runtime-config.js";
|
|
2
2
|
import { localRuntimeStatus } from "../substrates/local/runtime.js";
|
|
3
|
-
// What a lab
|
|
3
|
+
// What a lab is, for the surface that has to describe it before you spend money.
|
|
4
4
|
//
|
|
5
|
-
// The run index and run detail answer questions about runs. This answers a question about the
|
|
6
|
-
// itself
|
|
5
|
+
// The run index and run detail answer questions about runs. This answers a question about the lab
|
|
6
|
+
// itself (what it drives, who is in it, which model, and what it is allowed to spend), which is
|
|
7
7
|
// what a stakeholder reads on the screen where they decide whether to press Start.
|
|
8
8
|
//
|
|
9
9
|
// Resolved analysis defaults are shown independently of declared participant caps. A cap that is not
|
|
@@ -19,10 +19,10 @@ import { isComputerUseComposition } from "./routing.js";
|
|
|
19
19
|
import { probeKeySources } from "../keys/key-resolution.js";
|
|
20
20
|
import { receivingRequiredKey } from "../comms/setup.js";
|
|
21
21
|
import { rosterOf } from "./parse/actors.js";
|
|
22
|
-
export const
|
|
22
|
+
export const STUDY_SUMMARY_SCHEMA = "humanish.study-summary.v1";
|
|
23
23
|
/**
|
|
24
24
|
* The effort every participant will run at, or `"per-lane"` when they differ. A participant that declares nothing
|
|
25
|
-
* inherits the actor's, and an actor that declares nothing gets the provider default
|
|
25
|
+
* inherits the actor's, and an actor that declares nothing gets the provider default, which is
|
|
26
26
|
* reported as the resolved value, exactly as `model` reports its default rather than hiding it.
|
|
27
27
|
*/
|
|
28
28
|
function reasoningEffortOf(config) {
|
|
@@ -47,7 +47,7 @@ function subjectOf(config) {
|
|
|
47
47
|
return `${subject.source} ${subject.appUrl}`;
|
|
48
48
|
return subject.source;
|
|
49
49
|
}
|
|
50
|
-
/** How many participants, and who
|
|
50
|
+
/** How many participants, and who; collapsed when they are all the same persona. */
|
|
51
51
|
function participantsOf(config) {
|
|
52
52
|
const actors = config.actors;
|
|
53
53
|
const actor = actors?.[0];
|
|
@@ -61,7 +61,7 @@ function participantsOf(config) {
|
|
|
61
61
|
const unique = [...new Set(personas)];
|
|
62
62
|
if (unique.length === 0)
|
|
63
63
|
return `${count} participant${count === 1 ? "" : "s"}`;
|
|
64
|
-
// Several participants of
|
|
64
|
+
// Several participants of one persona reads as "3 × skeptical-power-user"; genuinely different people
|
|
65
65
|
// are named, because which personas are in a study is the study's design.
|
|
66
66
|
return unique.length === 1 ? `${count} × ${unique[0]}` : unique.join(" · ");
|
|
67
67
|
}
|
|
@@ -76,7 +76,7 @@ function capsOf(config) {
|
|
|
76
76
|
};
|
|
77
77
|
}
|
|
78
78
|
/**
|
|
79
|
-
* Describe one lab. Returns null when the manifest cannot be resolved
|
|
79
|
+
* Describe one lab. Returns null when the manifest cannot be resolved: the caller already knows
|
|
80
80
|
* the lab exists from the listing, so this failing means the file changed underneath them.
|
|
81
81
|
*/
|
|
82
82
|
export async function readLabSummary(cwd, lab, options = {}) {
|
|
@@ -154,13 +154,13 @@ export async function readLabSummary(cwd, lab, options = {}) {
|
|
|
154
154
|
},
|
|
155
155
|
}
|
|
156
156
|
: {}),
|
|
157
|
-
schema:
|
|
157
|
+
schema: STUDY_SUMMARY_SCHEMA,
|
|
158
158
|
...(inspected.config.comms?.email?.kind === "real"
|
|
159
159
|
? {
|
|
160
160
|
communications: `Real email · ${inspected.config.comms.email.connection} · fresh inbox per participant · hosted processing · local review only`,
|
|
161
161
|
}
|
|
162
162
|
: {}),
|
|
163
|
-
|
|
163
|
+
studyId: String(config.id ?? lab),
|
|
164
164
|
...(typeof config.title === "string" ? { title: config.title } : {}),
|
|
165
165
|
...(typeof config.description === "string" ? { description: config.description.trim() } : {}),
|
|
166
166
|
...(subject === undefined ? {} : { subject }),
|
package/dist/lab/tasks.d.ts
CHANGED
|
@@ -4,20 +4,20 @@ declare const TASK_FUNNEL_SCHEMA: "humanish.task-funnel.v1";
|
|
|
4
4
|
* One task in a protocol. It has two halves that belong to two different people, and keeping them
|
|
5
5
|
* apart is the point of the type.
|
|
6
6
|
*
|
|
7
|
-
* `goal` belongs to the
|
|
7
|
+
* `goal` belongs to the participant. It is what they are asked to do, in their language, and it is
|
|
8
8
|
* the only half that reaches the prompt.
|
|
9
9
|
*
|
|
10
|
-
* `success` belongs to the
|
|
11
|
-
* never sees it
|
|
10
|
+
* `success` belongs to the researcher. It is how the task will be measured, and the participant
|
|
11
|
+
* never sees it. A moderator does not read the success criterion aloud, because telling someone
|
|
12
12
|
* how they will be judged changes what they do. A persona told "you succeed when the URL contains
|
|
13
13
|
* /dashboard" will go find that URL, which measures the instruction rather than the product.
|
|
14
14
|
*/
|
|
15
15
|
export interface LabTask {
|
|
16
16
|
/** Stable id, used in evidence and in the funnel. Researcher-facing. */
|
|
17
17
|
id: string;
|
|
18
|
-
/**
|
|
18
|
+
/** Participant-facing. What they are asked to do. The only half that reaches the prompt. */
|
|
19
19
|
goal: string;
|
|
20
|
-
/**
|
|
20
|
+
/** Researcher-facing. Observation-shaped proof the task happened. Never rendered to the
|
|
21
21
|
* participant. Absent means the task is narrative-only: it is asked for but cannot be measured,
|
|
22
22
|
* and the funnel says so rather than quietly counting it as failed. */
|
|
23
23
|
success?: StopWhen;
|
|
@@ -38,12 +38,12 @@ export interface TaskFunnel {
|
|
|
38
38
|
total: number;
|
|
39
39
|
/** Tasks observed complete. */
|
|
40
40
|
completed: number;
|
|
41
|
-
/** Declared tasks with no `done` criterion
|
|
41
|
+
/** Declared tasks with no `done` criterion: shown to the participant, never observable. */
|
|
42
42
|
unobservable: number;
|
|
43
|
-
/** The first task that was
|
|
43
|
+
/** The first task that was not observed complete: where this participant stopped. */
|
|
44
44
|
stoppedAt?: string;
|
|
45
45
|
/** Per-task, in declaration order. `inputsObserved` is false when the task declares criteria
|
|
46
|
-
* whose inputs (url, text, appState)
|
|
46
|
+
* whose inputs (url, text, appState) never arrived in any observation this session, so the
|
|
47
47
|
* task was never actually measured. Absent when the task completed or is unobservable. */
|
|
48
48
|
tasks: Array<{
|
|
49
49
|
id: string;
|
|
@@ -53,20 +53,20 @@ export interface TaskFunnel {
|
|
|
53
53
|
inputsObserved?: boolean;
|
|
54
54
|
}>;
|
|
55
55
|
/** Observable tasks that were never measured, because the inputs their criteria need never
|
|
56
|
-
* arrived. Reported
|
|
57
|
-
* instrument, and reporting it as 0-completed blames the participant for it
|
|
56
|
+
* arrived. Reported separately from failures: a task nobody could observe is a gap in our
|
|
57
|
+
* instrument, and reporting it as 0-completed blames the participant for it. */
|
|
58
58
|
unmeasured: number;
|
|
59
59
|
}
|
|
60
60
|
/**
|
|
61
|
-
* Tracks task completion across a session. Stateful on purpose: a task completes
|
|
62
|
-
* observation that satisfies it, and stays complete even if the participant navigates away
|
|
61
|
+
* Tracks task completion across a session. Stateful on purpose: a task completes once, on the first
|
|
62
|
+
* observation that satisfies it, and stays complete even if the participant navigates away; you do
|
|
63
63
|
* not un-sign-up by going back to the home page.
|
|
64
64
|
*/
|
|
65
65
|
export declare class TaskTracker {
|
|
66
66
|
private readonly tasks;
|
|
67
67
|
private readonly completions;
|
|
68
68
|
/** Which observation fields were ever populated this session. A criterion whose field never
|
|
69
|
-
* arrived was never evaluated against anything, however many turns ran
|
|
69
|
+
* arrived was never evaluated against anything, however many turns ran. */
|
|
70
70
|
private readonly fieldsSeen;
|
|
71
71
|
constructor(tasks: readonly LabTask[]);
|
|
72
72
|
/** Evaluate every still-incomplete task against one observation. Returns newly completed tasks. */
|
|
@@ -78,10 +78,10 @@ export declare class TaskTracker {
|
|
|
78
78
|
funnel(): TaskFunnel;
|
|
79
79
|
}
|
|
80
80
|
/**
|
|
81
|
-
* The task list as the
|
|
81
|
+
* The task list as the participant reads it: numbered, in order, in their own language.
|
|
82
82
|
*
|
|
83
83
|
* Reads `goal` and nothing else. The success criteria are the researcher's instrument and never
|
|
84
|
-
* appear here
|
|
84
|
+
* appear here: a participant who is told how they will be measured optimizes for the measurement,
|
|
85
85
|
* and the study stops being about the product. A test pins this, because it is the kind of leak a
|
|
86
86
|
* later convenience change makes without noticing.
|
|
87
87
|
*/
|
package/dist/lab/tasks.js
CHANGED
|
@@ -1,32 +1,32 @@
|
|
|
1
1
|
// Tasks: the researcher's protocol, expressed as config (docs/principles/three-roles.md).
|
|
2
2
|
//
|
|
3
|
-
// A lab could declare a prose `mission` and nothing else. That is a brief, not a protocol
|
|
3
|
+
// A lab could declare a prose `mission` and nothing else. That is a brief, not a protocol, and it
|
|
4
4
|
// left "where did people get stuck" answerable only from an actor's own narration, which is the one
|
|
5
5
|
// source a study should not have to take on faith.
|
|
6
6
|
//
|
|
7
|
-
// Real usability studies are built from discrete
|
|
7
|
+
// Real usability studies are built from discrete tasks, each with a written success criterion, and
|
|
8
8
|
// the result is a funnel: how far each participant got before they stopped. That funnel is the
|
|
9
9
|
// finding. A single pass/fail per participant throws it away.
|
|
10
10
|
//
|
|
11
|
-
// The criterion language is `stopWhen`, unchanged and already
|
|
11
|
+
// The criterion language is `stopWhen`, unchanged and already used by stop conditions: a task is
|
|
12
12
|
// done when an observation satisfies it. Reusing it means a task criterion is exactly as expressive
|
|
13
13
|
// as a stop condition, and an author who knows one knows the other.
|
|
14
14
|
//
|
|
15
|
-
// A task's completion is
|
|
15
|
+
// A task's completion is corroborated, not self-reported. The actor saying "I signed up" does not
|
|
16
16
|
// complete a task; the observed URL, page text, or app state does. That distinction is the whole
|
|
17
17
|
// reason to declare tasks at all.
|
|
18
18
|
import { evaluateStopWhen, } from "../actors/stop-conditions.js";
|
|
19
19
|
const TASK_FUNNEL_SCHEMA = "humanish.task-funnel.v1";
|
|
20
20
|
/**
|
|
21
|
-
* Tracks task completion across a session. Stateful on purpose: a task completes
|
|
22
|
-
* observation that satisfies it, and stays complete even if the participant navigates away
|
|
21
|
+
* Tracks task completion across a session. Stateful on purpose: a task completes once, on the first
|
|
22
|
+
* observation that satisfies it, and stays complete even if the participant navigates away; you do
|
|
23
23
|
* not un-sign-up by going back to the home page.
|
|
24
24
|
*/
|
|
25
25
|
export class TaskTracker {
|
|
26
26
|
tasks;
|
|
27
27
|
completions = new Map();
|
|
28
28
|
/** Which observation fields were ever populated this session. A criterion whose field never
|
|
29
|
-
* arrived was never evaluated against anything, however many turns ran
|
|
29
|
+
* arrived was never evaluated against anything, however many turns ran. */
|
|
30
30
|
fieldsSeen = new Set();
|
|
31
31
|
constructor(tasks) {
|
|
32
32
|
this.tasks = tasks;
|
|
@@ -77,7 +77,7 @@ export class TaskTracker {
|
|
|
77
77
|
const tasks = this.tasks.map((task) => {
|
|
78
78
|
const completion = this.completions.get(task.id);
|
|
79
79
|
const observable = task.success !== undefined;
|
|
80
|
-
// A task counts as measured when at least
|
|
80
|
+
// A task counts as measured when at least one field its criteria read was populated by some
|
|
81
81
|
// observation. `any` semantics: any satisfiable rule needed a field we actually saw.
|
|
82
82
|
const required = observable ? this.fieldsRequiredBy(task) : [];
|
|
83
83
|
const inputsObserved = required.length === 0 ? true : required.some((field) => this.fieldsSeen.has(field));
|
|
@@ -91,12 +91,12 @@ export class TaskTracker {
|
|
|
91
91
|
...(observable && completion === undefined ? { inputsObserved } : {}),
|
|
92
92
|
};
|
|
93
93
|
});
|
|
94
|
-
// Where they stopped is the first task not observed complete
|
|
94
|
+
// Where they stopped is the first task not observed complete, the thing a researcher reads
|
|
95
95
|
// first. An unobservable task cannot be "where they stopped", because nothing could have
|
|
96
96
|
// proven otherwise; skipping it avoids blaming a participant for a gap in the protocol.
|
|
97
97
|
// A task we never measured cannot be "where they stopped" either, for the same reason an
|
|
98
98
|
// unobservable one cannot: nothing could have proven otherwise, so naming it blames the
|
|
99
|
-
// participant for our gap
|
|
99
|
+
// participant for our gap.
|
|
100
100
|
const stoppedAt = tasks.find((task) => task.observable && !task.completed && task.inputsObserved !== false)?.id;
|
|
101
101
|
return {
|
|
102
102
|
schema: TASK_FUNNEL_SCHEMA,
|
|
@@ -110,10 +110,10 @@ export class TaskTracker {
|
|
|
110
110
|
}
|
|
111
111
|
}
|
|
112
112
|
/**
|
|
113
|
-
* The task list as the
|
|
113
|
+
* The task list as the participant reads it: numbered, in order, in their own language.
|
|
114
114
|
*
|
|
115
115
|
* Reads `goal` and nothing else. The success criteria are the researcher's instrument and never
|
|
116
|
-
* appear here
|
|
116
|
+
* appear here: a participant who is told how they will be measured optimizes for the measurement,
|
|
117
117
|
* and the study stops being about the product. A test pins this, because it is the kind of leak a
|
|
118
118
|
* later convenience change makes without noticing.
|
|
119
119
|
*/
|
|
@@ -131,9 +131,9 @@ export function formatTaskFunnel(funnel) {
|
|
|
131
131
|
const stopped = funnel.stoppedAt === undefined ? "" : `, stopped at "${funnel.stoppedAt}"`;
|
|
132
132
|
const unobservable = funnel.unobservable === 0 ? "" : `, ${funnel.unobservable} with no completion criterion`;
|
|
133
133
|
// Named separately from failures. "0/3 completed" alone reads as "no participant managed it",
|
|
134
|
-
// which is the wrong story when the criterion was never evaluated against anything
|
|
134
|
+
// which is the wrong story when the criterion was never evaluated against anything.
|
|
135
135
|
const unmeasured = funnel.unmeasured === 0
|
|
136
136
|
? ""
|
|
137
|
-
: `, ${funnel.unmeasured}
|
|
137
|
+
: `, ${funnel.unmeasured} never measured (the observations their criteria read never arrived)`;
|
|
138
138
|
return `${base}${stopped}${unobservable}${unmeasured}`;
|
|
139
139
|
}
|