humanish 0.108.0 → 0.109.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +58 -59
- package/CHANGELOG.md +13 -0
- package/CONTEXT.md +44 -44
- package/README.md +3 -3
- package/TELEMETRY.md +2 -2
- package/dist/actors/codex/app-server-trace.js +2 -1
- package/dist/actors/codex/restricted-participant.js +12 -12
- package/dist/actors/codex/restricted-session.d.ts +2 -2
- package/dist/actors/computer-use/actor.d.ts +6 -6
- package/dist/actors/computer-use/actor.js +2 -2
- package/dist/actors/computer-use/admission-limit.d.ts +2 -2
- package/dist/actors/computer-use/admission-limit.js +3 -3
- package/dist/actors/computer-use/executor-error.d.ts +2 -2
- package/dist/actors/computer-use/executor-error.js +3 -3
- package/dist/actors/computer-use/loop/actions.js +5 -5
- package/dist/actors/computer-use/loop/debrief.js +2 -2
- package/dist/actors/computer-use/loop/ending.d.ts +1 -1
- package/dist/actors/computer-use/loop/ending.js +12 -11
- package/dist/actors/computer-use/loop/observation.js +6 -5
- package/dist/actors/computer-use/loop/provider-call.js +11 -11
- package/dist/actors/computer-use/loop/session.d.ts +4 -4
- package/dist/actors/computer-use/loop/session.js +2 -2
- package/dist/actors/computer-use/loop/trace.js +3 -2
- package/dist/actors/computer-use/loop/types.d.ts +24 -24
- package/dist/actors/computer-use/loop/usage.d.ts +2 -2
- package/dist/actors/computer-use/loop.d.ts +6 -1
- package/dist/actors/computer-use/loop.js +26 -4
- package/dist/actors/computer-use/openai-provider.d.ts +2 -2
- package/dist/actors/computer-use/openai-provider.js +8 -8
- package/dist/actors/computer-use/provider-error.d.ts +4 -4
- package/dist/actors/computer-use/provider-error.js +6 -6
- package/dist/actors/contract.d.ts +4 -4
- package/dist/actors/contract.js +1 -1
- package/dist/actors/local-agent/claude-session.js +3 -3
- package/dist/actors/local-agent/cli.d.ts +1 -1
- package/dist/actors/local-agent/cli.js +1 -1
- package/dist/actors/reasoning-effort.js +3 -3
- package/dist/actors/registry.d.ts +2 -2
- package/dist/actors/registry.js +3 -3
- package/dist/actors/scripted-browser/actor.d.ts +2 -2
- package/dist/actors/scripted-browser/actor.js +1 -1
- package/dist/analysis/automatic-completion.d.ts +7 -7
- package/dist/analysis/automatic-completion.js +1 -1
- package/dist/analysis/automatic-config.d.ts +6 -6
- package/dist/analysis/automatic-config.js +1 -1
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +1 -1
- package/dist/analysis/types.d.ts +6 -0
- package/dist/browser-control/client.js +16 -16
- package/dist/browser-control/dispatcher.js +4 -4
- package/dist/browser-control/protocol.js +14 -14
- package/dist/browser-control/recording-transfer.js +4 -4
- package/dist/browser-control/transport.js +7 -7
- package/dist/cli/commands/analysis-signals.d.ts +2 -2
- package/dist/cli/commands/comms.js +14 -22
- package/dist/cli/commands/observe.d.ts +0 -1
- package/dist/cli/commands/observe.js +0 -15
- package/dist/cli/commands/run-command.d.ts +2 -2
- package/dist/cli/commands/run-command.js +3 -3
- package/dist/cli/commands/runs.js +3 -5
- package/dist/cli/commands/setup.js +5 -7
- package/dist/cli/commands/study-format.d.ts +9 -0
- package/dist/cli/commands/{lab-format.js → study-format.js} +5 -5
- package/dist/cli/commands/{lab-route-computer-use.d.ts → study-route-computer-use.d.ts} +6 -6
- package/dist/cli/commands/{lab-route-computer-use.js → study-route-computer-use.js} +12 -12
- package/dist/cli/commands/{lab-route-open.d.ts → study-route-open.d.ts} +4 -4
- package/dist/cli/commands/{lab-route-open.js → study-route-open.js} +2 -2
- package/dist/cli/commands/{lab-route-preview.d.ts → study-route-preview.d.ts} +7 -7
- package/dist/cli/commands/{lab-route-preview.js → study-route-preview.js} +3 -3
- package/dist/cli/commands/study-route-run.d.ts +19 -0
- package/dist/cli/commands/{lab-route-run.js → study-route-run.js} +6 -6
- package/dist/cli/commands/study-route-scripted.d.ts +17 -0
- package/dist/cli/commands/{lab-route-scripted.js → study-route-scripted.js} +4 -4
- package/dist/cli/commands/{lab-route-shared-world.d.ts → study-route-shared-world.d.ts} +5 -5
- package/dist/cli/commands/{lab-route-shared-world.js → study-route-shared-world.js} +7 -7
- package/dist/cli/commands/study-route-terminal.d.ts +17 -0
- package/dist/cli/commands/{lab-route-terminal.js → study-route-terminal.js} +4 -4
- package/dist/cli/commands/study-run.d.ts +9 -0
- package/dist/cli/commands/{lab-run.js → study-run.js} +34 -34
- package/dist/cli/commands/{lab-scorer.d.ts → study-scorer.d.ts} +5 -5
- package/dist/cli/commands/{lab-scorer.js → study-scorer.js} +1 -1
- package/dist/cli/commands/study.js +13 -49
- package/dist/cli/commands/tui.js +12 -12
- package/dist/cli/commands/watch.js +30 -65
- package/dist/cli/deprecations.d.ts +1 -13
- package/dist/cli/deprecations.js +0 -30
- package/dist/cli/doctor.d.ts +3 -3
- package/dist/cli/doctor.js +16 -16
- package/dist/cli/first-run-path.d.ts +4 -4
- package/dist/cli/first-run-path.js +7 -7
- package/dist/cli/io.d.ts +3 -3
- package/dist/cli/io.js +2 -2
- package/dist/cli/observer-follow.d.ts +2 -2
- package/dist/cli/observer-follow.js +2 -2
- package/dist/cli/orientation.d.ts +4 -4
- package/dist/cli/orientation.js +58 -20
- package/dist/cli/program.js +3 -4
- package/dist/cli/telemetry.d.ts +1 -1
- package/dist/cli/telemetry.js +14 -20
- package/dist/comms/catch-host.d.ts +1 -1
- package/dist/comms/catch-host.js +3 -3
- package/dist/comms/connections.d.ts +1 -1
- package/dist/comms/connections.js +2 -2
- package/dist/comms/external-evidence.d.ts +5 -5
- package/dist/comms/external-evidence.js +4 -3
- package/dist/comms/fake-inbox.d.ts +2 -2
- package/dist/comms/fake-inbox.js +2 -2
- package/dist/comms/receiving-runtime.d.ts +4 -4
- package/dist/comms/sandbox-catch.d.ts +2 -2
- package/dist/comms/sandbox-catch.js +2 -2
- package/dist/comms/setup.d.ts +2 -2
- package/dist/comms/setup.js +29 -32
- package/dist/feedback/draft.js +2 -2
- package/dist/feedback/export.js +3 -2
- package/dist/feedback/proof.d.ts +1 -1
- package/dist/feedback/proof.js +2 -2
- package/dist/guest/bootstrap.js +6 -6
- package/dist/guest/browser-tools.js +8 -8
- package/dist/guest/chromium-text.js +17 -17
- package/dist/guest/desktop-executor.d.ts +2 -2
- package/dist/guest/desktop-executor.js +20 -20
- package/dist/guest/desktop-keys.js +4 -4
- package/dist/guest/desktop-media.d.ts +2 -2
- package/dist/guest/desktop-media.js +16 -16
- package/dist/guest/desktop-native.js +10 -10
- package/dist/guest/runtime-desktop.js +2 -2
- package/dist/guest/runtime-processes.d.ts +2 -2
- package/dist/guest/runtime-processes.js +2 -2
- package/dist/guest/runtime.js +5 -5
- package/dist/index.d.ts +13 -15
- package/dist/index.js +6 -8
- package/dist/observer/artifact.d.ts +1 -1
- package/dist/observer/artifact.js +1 -1
- package/dist/observer/data.d.ts +1 -1
- package/dist/observer/library.d.ts +2 -0
- package/dist/observer/library.js +4 -1
- package/dist/observer/render.js +7 -1
- package/dist/observer/run-routes.d.ts +1 -0
- package/dist/observer/run-routes.js +37 -12
- package/dist/observer/serve.d.ts +1 -1
- package/dist/observer-app.html +1 -1
- package/dist/routes/computer-use/bundle-parts.js +3 -2
- package/dist/routes/computer-use/bundle.d.ts +2 -2
- package/dist/routes/computer-use/bundle.js +7 -6
- package/dist/routes/computer-use/e2b-desktop/comms.d.ts +5 -5
- package/dist/routes/computer-use/e2b-desktop/comms.js +5 -4
- package/dist/routes/computer-use/e2b-desktop/fidelity.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +1 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.js +3 -3
- package/dist/routes/computer-use/e2b-desktop/start.js +3 -4
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +2 -2
- package/dist/routes/computer-use/e2b-desktop/teardown.js +2 -3
- package/dist/routes/computer-use/external-comms.d.ts +3 -3
- package/dist/routes/computer-use/fanout-bundle.js +11 -10
- package/dist/routes/computer-use/fanout-records.js +2 -2
- package/dist/routes/computer-use/live-flush.d.ts +3 -3
- package/dist/routes/computer-use/live-phase.d.ts +2 -3
- package/dist/routes/computer-use/live-phase.js +3 -3
- package/dist/routes/computer-use/local-tree-pack.d.ts +5 -5
- package/dist/routes/computer-use/local-tree-pack.js +1 -1
- package/dist/routes/computer-use/local-vm.d.ts +8 -8
- package/dist/routes/computer-use/local-vm.js +2 -2
- package/dist/routes/computer-use/participant-desktop.d.ts +3 -3
- package/dist/routes/computer-use/participant-desktop.js +1 -1
- package/dist/routes/computer-use/participant-execution.js +1 -1
- package/dist/routes/computer-use/participant-model.js +3 -3
- package/dist/routes/computer-use/participant-prompt.d.ts +5 -5
- package/dist/routes/computer-use/participant-prompt.js +3 -3
- package/dist/routes/computer-use/participant-runs.d.ts +10 -10
- package/dist/routes/computer-use/participant-runs.js +14 -13
- package/dist/routes/computer-use/plan.d.ts +12 -12
- package/dist/routes/computer-use/plan.js +24 -24
- package/dist/routes/computer-use/preflight.d.ts +5 -5
- package/dist/routes/computer-use/preflight.js +5 -5
- package/dist/routes/computer-use/rerun-selection.d.ts +1 -1
- package/dist/routes/computer-use/rerun-selection.js +3 -2
- package/dist/routes/computer-use/result.d.ts +4 -4
- package/dist/routes/computer-use/result.js +13 -12
- package/dist/routes/computer-use/route.d.ts +12 -12
- package/dist/routes/computer-use/route.js +19 -19
- package/dist/routes/computer-use/self-report.js +1 -1
- package/dist/routes/computer-use/setup.d.ts +22 -22
- package/dist/routes/computer-use/setup.js +15 -15
- package/dist/routes/computer-use/single-bundle.d.ts +4 -4
- package/dist/routes/computer-use/single-bundle.js +14 -13
- package/dist/routes/computer-use/subject-projection.d.ts +3 -3
- package/dist/routes/computer-use/types.d.ts +34 -34
- package/dist/routes/computer-use/types.js +3 -3
- package/dist/routes/preview.d.ts +6 -6
- package/dist/routes/preview.js +5 -5
- package/dist/routes/scripted/bundle.d.ts +6 -6
- package/dist/routes/scripted/bundle.js +10 -9
- package/dist/routes/scripted/plan.d.ts +7 -7
- package/dist/routes/scripted/plan.js +6 -6
- package/dist/routes/scripted/result.d.ts +3 -3
- package/dist/routes/scripted/result.js +9 -9
- package/dist/routes/scripted/route.d.ts +9 -9
- package/dist/routes/scripted/route.js +11 -11
- package/dist/routes/scripted/scenario.d.ts +1 -1
- package/dist/routes/scripted/scenario.js +3 -3
- package/dist/routes/scripted/setup.d.ts +6 -6
- package/dist/routes/scripted/setup.js +4 -4
- package/dist/routes/scripted/subject.d.ts +5 -5
- package/dist/routes/scripted/subject.js +3 -3
- package/dist/routes/scripted/surface-records.d.ts +1 -1
- package/dist/routes/scripted/surface-records.js +2 -2
- package/dist/routes/scripted/surface-sessions.d.ts +3 -3
- package/dist/routes/scripted/types.d.ts +17 -17
- package/dist/routes/scripted/types.js +1 -1
- package/dist/routes/shared-world/bundle-records.js +3 -3
- package/dist/routes/shared-world/bundle.js +15 -14
- package/dist/routes/shared-world/checkpoints.d.ts +3 -3
- package/dist/routes/shared-world/checkpoints.js +1 -1
- package/dist/routes/shared-world/comms.d.ts +9 -9
- package/dist/routes/shared-world/comms.js +6 -5
- package/dist/routes/shared-world/participant-specs.d.ts +3 -3
- package/dist/routes/shared-world/participant-specs.js +7 -7
- package/dist/routes/shared-world/plan.d.ts +9 -9
- package/dist/routes/shared-world/plan.js +9 -9
- package/dist/routes/shared-world/provisioned.d.ts +8 -8
- package/dist/routes/shared-world/provisioned.js +4 -4
- package/dist/routes/shared-world/result.d.ts +6 -6
- package/dist/routes/shared-world/result.js +11 -10
- package/dist/routes/shared-world/route.d.ts +11 -11
- package/dist/routes/shared-world/route.js +19 -19
- package/dist/routes/shared-world/setup.d.ts +13 -13
- package/dist/routes/shared-world/setup.js +19 -19
- package/dist/routes/shared-world/types.d.ts +27 -27
- package/dist/routes/terminal/adapter.d.ts +2 -2
- package/dist/routes/terminal/adapter.js +4 -4
- package/dist/routes/terminal/bundle.d.ts +11 -11
- package/dist/routes/terminal/bundle.js +12 -12
- package/dist/routes/terminal/credentials.d.ts +4 -4
- package/dist/routes/terminal/credentials.js +5 -5
- package/dist/routes/terminal/dry-run.d.ts +4 -4
- package/dist/routes/terminal/dry-run.js +9 -9
- package/dist/routes/terminal/ledger.d.ts +2 -2
- package/dist/routes/terminal/lifetime.d.ts +1 -1
- package/dist/routes/terminal/lifetime.js +1 -1
- package/dist/routes/terminal/live-finish.d.ts +2 -2
- package/dist/routes/terminal/live-finish.js +7 -7
- package/dist/routes/terminal/live-sandbox.d.ts +2 -2
- package/dist/routes/terminal/live-sandbox.js +3 -2
- package/dist/routes/terminal/participant-text.js +2 -1
- package/dist/routes/terminal/persona.d.ts +2 -2
- package/dist/routes/terminal/persona.js +2 -2
- package/dist/routes/terminal/plan.d.ts +7 -7
- package/dist/routes/terminal/plan.js +5 -5
- package/dist/routes/terminal/result.d.ts +5 -5
- package/dist/routes/terminal/result.js +7 -7
- package/dist/routes/terminal/route.d.ts +9 -9
- package/dist/routes/terminal/route.js +14 -14
- package/dist/routes/terminal/session.d.ts +5 -5
- package/dist/routes/terminal/session.js +7 -7
- package/dist/routes/terminal/trace.d.ts +2 -2
- package/dist/routes/terminal/types.d.ts +24 -24
- package/dist/routes/terminal/types.js +2 -2
- package/dist/run/bundle.d.ts +7 -7
- package/dist/run/bundle.js +2 -2
- package/dist/run/costs.d.ts +23 -0
- package/dist/run/costs.js +72 -43
- package/dist/run/dry-run-inputs.d.ts +1 -1
- package/dist/run/dry-run-inputs.js +1 -1
- package/dist/run/dry-run.js +2 -2
- package/dist/run/outcomes.d.ts +1 -1
- package/dist/run/participant.d.ts +1 -1
- package/dist/run/preflight-receipts.js +1 -1
- package/dist/run/pricing.js +2 -2
- package/dist/run/projection.d.ts +32 -32
- package/dist/run/projection.js +44 -44
- package/dist/run/reclaim.d.ts +1 -1
- package/dist/run/reclaim.js +1 -1
- package/dist/run/results.d.ts +2 -2
- package/dist/run/run-cost.d.ts +53 -0
- package/dist/run/run-cost.js +105 -0
- package/dist/run/run-index-analysis.d.ts +15 -0
- package/dist/run/run-index-analysis.js +77 -0
- package/dist/run/run-index.d.ts +11 -1
- package/dist/run/run-index.js +63 -49
- package/dist/run/run.d.ts +3 -3
- package/dist/run/run.js +2 -2
- package/dist/run/shared-world-evidence.d.ts +1 -1
- package/dist/run/shared-world-shape.js +1 -1
- package/dist/run/stats.d.ts +3 -3
- package/dist/run/stats.js +14 -14
- package/dist/run/status.d.ts +10 -2
- package/dist/run/status.js +6 -3
- package/dist/run/streams.d.ts +2 -2
- package/dist/run/study-provenance.d.ts +4 -2
- package/dist/run/study-provenance.js +7 -4
- package/dist/run/study-result.d.ts +0 -5
- package/dist/run/study-result.js +1 -1
- package/dist/run-study.d.ts +112 -0
- package/dist/{run-lab.js → run-study.js} +52 -45
- package/dist/{lab → study}/adapter-extension.d.ts +8 -20
- package/dist/{lab → study}/adapter-extension.js +2 -14
- package/dist/{lab → study}/adapter-scorer-loader.d.ts +10 -10
- package/dist/{lab → study}/adapter-scorer-loader.js +3 -3
- package/dist/study/composition-rules.d.ts +7 -0
- package/dist/{lab → study}/composition-rules.js +12 -12
- package/dist/study/config.d.ts +15 -0
- package/dist/{lab → study}/config.js +37 -20
- package/dist/study/convert.d.ts +1 -1
- package/dist/study/convert.js +15 -15
- package/dist/{lab → study}/device-presets.d.ts +2 -2
- package/dist/{lab → study}/discover.d.ts +29 -23
- package/dist/{lab → study}/discover.js +97 -118
- package/dist/{lab → study}/doctor.d.ts +11 -11
- package/dist/{lab → study}/doctor.js +18 -18
- package/dist/study/files.d.ts +2 -2
- package/dist/study/files.js +6 -5
- package/dist/{lab → study}/init-templates.d.ts +2 -2
- package/dist/{lab → study}/init-templates.js +16 -11
- package/dist/{lab → study}/init.d.ts +1 -1
- package/dist/{lab → study}/init.js +7 -4
- package/dist/{lab → study}/keys.d.ts +2 -2
- package/dist/{lab → study}/keys.js +10 -10
- package/dist/study/migrate.d.ts +1 -1
- package/dist/{lab → study}/parse/actors.d.ts +5 -5
- package/dist/{lab → study}/parse/actors.js +5 -5
- package/dist/study/parse/comms.d.ts +12 -0
- package/dist/study/parse/execution.d.ts +19 -0
- package/dist/{lab → study}/parse/study-v3.d.ts +3 -5
- package/dist/{lab → study}/parse/study-v3.js +3 -3
- package/dist/{lab → study}/parse/subject-state.d.ts +8 -8
- package/dist/{lab → study}/parse/subject-state.js +4 -4
- package/dist/{lab → study}/parse/subject.d.ts +3 -3
- package/dist/{lab → study}/parse/subject.js +8 -8
- package/dist/{lab → study}/parse/values.d.ts +2 -2
- package/dist/{lab → study}/persona-resolve.d.ts +4 -4
- package/dist/{lab → study}/persona-resolve.js +2 -2
- package/dist/{lab → study}/plan-base.d.ts +7 -7
- package/dist/{lab → study}/plan-base.js +1 -1
- package/dist/{lab → study}/plan-participants.d.ts +10 -10
- package/dist/{lab → study}/plan-participants.js +4 -4
- package/dist/{lab → study}/plan-types.d.ts +28 -28
- package/dist/{lab → study}/plan-types.js +2 -2
- package/dist/study/plan.d.ts +13 -0
- package/dist/{lab → study}/plan.js +20 -20
- package/dist/study/preflight-probes.d.ts +3 -0
- package/dist/{lab → study}/preflight-probes.js +9 -8
- package/dist/study/preflight-result.d.ts +8 -0
- package/dist/{lab → study}/preflight-result.js +3 -3
- package/dist/{lab → study}/preflight.d.ts +31 -31
- package/dist/{lab → study}/preflight.js +8 -8
- package/dist/{lab → study}/requirements.d.ts +3 -3
- package/dist/{lab → study}/requirements.js +1 -1
- package/dist/study/route-inputs.d.ts +28 -0
- package/dist/{lab → study}/route-inputs.js +1 -1
- package/dist/{lab → study}/routing.d.ts +12 -12
- package/dist/{lab/run-lab-events.d.ts → study/run-study-events.d.ts} +10 -10
- package/dist/{lab/run-lab-events.js → study/run-study-events.js} +3 -3
- package/dist/{lab/run-lab-homes.d.ts → study/run-study-homes.d.ts} +8 -17
- package/dist/study/run-study-homes.js +3 -0
- package/dist/study/run-study-options.d.ts +28 -0
- package/dist/{lab/run-lab-options.js → study/run-study-options.js} +13 -12
- package/dist/{lab/lab-deps.d.ts → study/study-deps.d.ts} +2 -2
- package/dist/{lab/lab-deps.js → study/study-deps.js} +2 -2
- package/dist/{lab → study}/summary.d.ts +10 -10
- package/dist/{lab → study}/summary.js +15 -15
- package/dist/{lab → study}/tasks.d.ts +3 -3
- package/dist/{lab → study}/tasks.js +1 -1
- package/dist/{lab → study}/types.d.ts +104 -104
- package/dist/{lab → study}/types.js +1 -1
- package/dist/{lab → study}/validation.d.ts +19 -19
- package/dist/{lab → study}/validation.js +13 -13
- package/dist/{lab → study}/warnings.d.ts +4 -4
- package/dist/{lab → study}/warnings.js +1 -1
- package/dist/subject/clone.d.ts +6 -6
- package/dist/subject/clone.js +2 -2
- package/dist/subject/local-tree-archive.d.ts +2 -2
- package/dist/subject/local-tree-archive.js +3 -3
- package/dist/subject/local-tree.d.ts +3 -3
- package/dist/subject/runtime.js +2 -2
- package/dist/subject/serve.d.ts +4 -4
- package/dist/subject/state.d.ts +3 -3
- package/dist/subject/state.js +1 -1
- package/dist/subject/steps.d.ts +1 -1
- package/dist/subject/steps.js +1 -1
- package/dist/substrates/desktop-session.js +2 -2
- package/dist/substrates/detached.d.ts +1 -1
- package/dist/substrates/e2b/desktop-browser.d.ts +4 -4
- package/dist/substrates/e2b/desktop-browser.js +1 -1
- package/dist/substrates/e2b/desktop-cdp.d.ts +1 -1
- package/dist/substrates/e2b/desktop-cdp.js +1 -1
- package/dist/substrates/e2b/desktop-executor.d.ts +1 -1
- package/dist/substrates/e2b/desktop-executor.js +12 -12
- package/dist/substrates/e2b/desktop-geometry.d.ts +1 -1
- package/dist/substrates/e2b/desktop-geometry.js +1 -1
- package/dist/substrates/e2b/desktop-media.d.ts +3 -3
- package/dist/substrates/e2b/lifetime.js +1 -1
- package/dist/substrates/e2b/sandbox.d.ts +3 -3
- package/dist/substrates/e2b/sandbox.js +1 -1
- package/dist/substrates/e2b/sdk.d.ts +1 -1
- package/dist/substrates/e2b/sdk.js +1 -1
- package/dist/substrates/local/runtime-config.d.ts +4 -4
- package/dist/substrates/local/runtime-config.js +4 -4
- package/dist/tui/contract.d.ts +8 -8
- package/dist/tui/contract.js +2 -2
- package/dist/tui/launch.d.ts +3 -3
- package/dist/tui/launch.js +22 -16
- package/dist/tui-app.js +139 -139
- package/dist/verify/subject.d.ts +1 -1
- package/dist/verify/subject.js +1 -1
- package/dist/verify/verify.js +2 -1
- package/docs/architecture/actor-contract.md +5 -5
- package/docs/architecture/browser-control.md +1 -1
- package/docs/architecture/external-public-shared-world.md +8 -7
- package/docs/architecture/github-feedback-loop.md +1 -1
- package/docs/architecture/task-protocol-support.md +1 -1
- package/docs/architecture/terminal-product-route.md +18 -18
- package/docs/contracts/adapter-fixtures.md +2 -2
- package/docs/contracts/errors.md +8 -8
- package/docs/contracts/run-bundle.md +1 -4
- package/docs/contracts/schemas.md +61 -42
- package/docs/contracts/study-analysis.md +4 -1
- package/docs/contracts/study-costs.md +17 -2
- package/examples/README.md +1 -1
- package/examples/scorer/scorer.mjs +3 -3
- package/package.json +1 -1
- package/skills/humanish/SKILL.md +8 -6
- package/skills/humanish/agents/openai.yaml +1 -1
- package/dist/cli/commands/lab-format.d.ts +0 -9
- package/dist/cli/commands/lab-route-run.d.ts +0 -19
- package/dist/cli/commands/lab-route-scripted.d.ts +0 -17
- package/dist/cli/commands/lab-route-terminal.d.ts +0 -17
- package/dist/cli/commands/lab-run.d.ts +0 -9
- package/dist/lab/composition-rules.d.ts +0 -7
- package/dist/lab/config.d.ts +0 -7
- package/dist/lab/deprecated-fields.d.ts +0 -10
- package/dist/lab/deprecated-fields.js +0 -24
- package/dist/lab/parse/comms.d.ts +0 -12
- package/dist/lab/parse/execution.d.ts +0 -19
- package/dist/lab/plan.d.ts +0 -13
- package/dist/lab/preflight-probes.d.ts +0 -3
- package/dist/lab/preflight-result.d.ts +0 -8
- package/dist/lab/route-inputs.d.ts +0 -28
- package/dist/lab/run-lab-homes.js +0 -7
- package/dist/lab/run-lab-options.d.ts +0 -28
- package/dist/library-aliases.d.ts +0 -56
- package/dist/library-aliases.js +0 -22
- package/dist/run-lab.d.ts +0 -112
- /package/dist/{lab → study}/device-presets.js +0 -0
- /package/dist/{lab → study}/init-paths.d.ts +0 -0
- /package/dist/{lab → study}/init-paths.js +0 -0
- /package/dist/{lab → study}/init-plan.d.ts +0 -0
- /package/dist/{lab → study}/init-plan.js +0 -0
- /package/dist/{lab → study}/parse/comms.js +0 -0
- /package/dist/{lab → study}/parse/execution.js +0 -0
- /package/dist/{lab → study}/parse/values.js +0 -0
- /package/dist/{lab → study}/participant-assignment.d.ts +0 -0
- /package/dist/{lab → study}/participant-assignment.js +0 -0
- /package/dist/{lab → study}/persona.d.ts +0 -0
- /package/dist/{lab → study}/persona.js +0 -0
- /package/dist/{lab → study}/routing.js +0 -0
package/dist/run/results.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ import type { RunStudyProvenance } from "./study-provenance.js";
|
|
|
4
4
|
export declare const CLEANUP_SCHEMA = "humanish.cleanup-result.v1";
|
|
5
5
|
export interface RunOptions {
|
|
6
6
|
/** Which manifest produced this run. */
|
|
7
|
-
|
|
7
|
+
study?: RunStudyProvenance;
|
|
8
8
|
cwd: string;
|
|
9
9
|
dryRun?: boolean;
|
|
10
10
|
runId?: string;
|
|
@@ -30,7 +30,7 @@ export interface RunResult {
|
|
|
30
30
|
observer?: ObserverResult;
|
|
31
31
|
warnings: string[];
|
|
32
32
|
error?: {
|
|
33
|
-
code: "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_ANALYSIS_UNSUPPORTED" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_STUDY_OPTION_UNSUPPORTED" | "HUMANISH_STUDY_COMMS_UNSUPPORTED" | "HUMANISH_LIVE_RUN_UNIMPLEMENTED" | "HUMANISH_INVALID_CWD" | "HUMANISH_RUN_ID_IN_USE" | "HUMANISH_INVALID_PARTICIPANT_COUNT" | "HUMANISH_INVALID_PORT" | "HUMANISH_UNSUPPORTED_RERUN_FLAGS" | "HUMANISH_WATCH_OPTION_CONFLICT" | "HUMANISH_WATCH_SAFE_NOT_APPLICABLE" | "HUMANISH_RUN_OPTION_CONFLICT" | "HUMANISH_OBSERVE_OPTION_CONFLICT" | "HUMANISH_STUDY_SCORER_BAD_REF" | "HUMANISH_STUDY_SCORER_NOT_FOUND" | "HUMANISH_STUDY_SCORER_LOAD_FAILED" | "HUMANISH_STUDY_SCORER_NO_HOOKS" | "HUMANISH_STUDY_SCORER_UNSUPPORTED_BACKEND";
|
|
33
|
+
code: "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_ANALYSIS_UNSUPPORTED" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_STUDY_OPTION_UNSUPPORTED" | "HUMANISH_STUDY_V2_UNSUPPORTED" | "HUMANISH_STUDY_COMMS_UNSUPPORTED" | "HUMANISH_LIVE_RUN_UNIMPLEMENTED" | "HUMANISH_INVALID_CWD" | "HUMANISH_RUN_ID_IN_USE" | "HUMANISH_INVALID_PARTICIPANT_COUNT" | "HUMANISH_INVALID_PORT" | "HUMANISH_UNSUPPORTED_RERUN_FLAGS" | "HUMANISH_WATCH_OPTION_CONFLICT" | "HUMANISH_WATCH_SAFE_NOT_APPLICABLE" | "HUMANISH_RUN_OPTION_CONFLICT" | "HUMANISH_OBSERVE_OPTION_CONFLICT" | "HUMANISH_STUDY_SCORER_BAD_REF" | "HUMANISH_STUDY_SCORER_NOT_FOUND" | "HUMANISH_STUDY_SCORER_LOAD_FAILED" | "HUMANISH_STUDY_SCORER_NO_HOOKS" | "HUMANISH_STUDY_SCORER_UNSUPPORTED_BACKEND";
|
|
34
34
|
message: string;
|
|
35
35
|
};
|
|
36
36
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { RunCostSummary } from "./bundle.js";
|
|
2
|
+
import type { AnalysisAccounting } from "./costs.js";
|
|
3
|
+
import type { RunIndexEntry } from "./run-index.js";
|
|
4
|
+
/** The fields of run.json's `cost` that the run cost reads. Completeness is unknown when absent. */
|
|
5
|
+
export type RunCostSubtotal = Pick<RunCostSummary, "estimatedTotalUsd"> & Partial<Pick<RunCostSummary, "fullyEstimated" | "ratesAsOf" | "placeholder">>;
|
|
6
|
+
/** A run's analysis spend, every attempt counted once, as `readAnalysisAccounting` reads it. */
|
|
7
|
+
export interface RunAnalysisCost {
|
|
8
|
+
/** Requests the run's analyses sent, or may have sent. */
|
|
9
|
+
requests: number;
|
|
10
|
+
/** Their known estimated dollars; null when none has a price. */
|
|
11
|
+
estimatedUsd: number | null;
|
|
12
|
+
/** True when every request has a complete price and the history was read whole. */
|
|
13
|
+
complete: boolean;
|
|
14
|
+
/** Who billed the requests that recorded usage. */
|
|
15
|
+
providers: Array<"openai" | "codex">;
|
|
16
|
+
}
|
|
17
|
+
/** Complete, a lower bound, or unknown (`null`) when only the subtotal's dollars are at hand. */
|
|
18
|
+
type Completeness = boolean | null;
|
|
19
|
+
export interface RunCost {
|
|
20
|
+
/** Participants and desktops; null when the run has no cost summary, as in a dry run. */
|
|
21
|
+
run: {
|
|
22
|
+
usd: number | null;
|
|
23
|
+
complete: Completeness;
|
|
24
|
+
ratesAsOf: string | null;
|
|
25
|
+
placeholder: boolean;
|
|
26
|
+
} | null;
|
|
27
|
+
/** The run's analysis requests; null when it sent none. */
|
|
28
|
+
analysis: RunAnalysisCost | null;
|
|
29
|
+
/** The known dollars of both; null when neither has a figure. */
|
|
30
|
+
total: {
|
|
31
|
+
usd: number;
|
|
32
|
+
complete: Completeness;
|
|
33
|
+
} | null;
|
|
34
|
+
}
|
|
35
|
+
/** The accounting's analysis spend, or null when the run sent no analysis request. */
|
|
36
|
+
export declare function analysisCostOf(accounting: Pick<AnalysisAccounting, "dispatched" | "unresolved" | "unpriced" | "estimatedUsd" | "providers" | "warnings">): RunAnalysisCost | null;
|
|
37
|
+
export declare function runCost(subtotal: RunCostSubtotal | null | undefined, analysis?: RunAnalysisCost | null): RunCost;
|
|
38
|
+
/**
|
|
39
|
+
* A run index entry's cost: its participants-and-desktops figure, complete or a lower bound when
|
|
40
|
+
* the status record says which, plus its analyses.
|
|
41
|
+
*/
|
|
42
|
+
export declare function indexedRunCost(entry: Pick<RunIndexEntry, "estimatedCostUsd" | "estimatedCostComplete" | "analysisCost">): RunCost;
|
|
43
|
+
/**
|
|
44
|
+
* The cost as separate parts: participants and desktops, then the analysis and the total when the
|
|
45
|
+
* run sent an analysis request. Empty when the run has no cost summary.
|
|
46
|
+
*/
|
|
47
|
+
export declare function runCostParts(cost: RunCost): string[];
|
|
48
|
+
/**
|
|
49
|
+
* The cost in one short label for a list row: the total, and how many analyses it includes.
|
|
50
|
+
* Undefined when the run has no cost summary.
|
|
51
|
+
*/
|
|
52
|
+
export declare function runCostLabel(cost: RunCost): string | undefined;
|
|
53
|
+
export {};
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/** The accounting's analysis spend, or null when the run sent no analysis request. */
|
|
2
|
+
export function analysisCostOf(accounting) {
|
|
3
|
+
const requests = accounting.dispatched + accounting.unresolved;
|
|
4
|
+
if (requests === 0)
|
|
5
|
+
return null;
|
|
6
|
+
const historyWhole = accounting.warnings.every((warning) => warning === "ANALYSIS_HISTORY_NOT_RECORDED");
|
|
7
|
+
return {
|
|
8
|
+
requests,
|
|
9
|
+
estimatedUsd: accounting.estimatedUsd,
|
|
10
|
+
complete: accounting.unpriced === 0 && historyWhole,
|
|
11
|
+
providers: [...accounting.providers],
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
export function runCost(subtotal, analysis) {
|
|
15
|
+
if (subtotal === null || subtotal === undefined)
|
|
16
|
+
return { run: null, analysis: null, total: null };
|
|
17
|
+
const run = {
|
|
18
|
+
usd: typeof subtotal.estimatedTotalUsd === "number" ? subtotal.estimatedTotalUsd : null,
|
|
19
|
+
complete: subtotal.fullyEstimated === undefined ? null : subtotal.fullyEstimated,
|
|
20
|
+
ratesAsOf: subtotal.ratesAsOf ?? null,
|
|
21
|
+
placeholder: subtotal.placeholder === true,
|
|
22
|
+
};
|
|
23
|
+
const spent = analysis ?? null;
|
|
24
|
+
const known = [run.usd, spent?.estimatedUsd].filter((usd) => typeof usd === "number");
|
|
25
|
+
if (known.length === 0)
|
|
26
|
+
return { run, analysis: spent, total: null };
|
|
27
|
+
const lowerBound = run.usd === null || run.complete === false || spent?.complete === false;
|
|
28
|
+
return {
|
|
29
|
+
run,
|
|
30
|
+
analysis: spent,
|
|
31
|
+
total: {
|
|
32
|
+
usd: round6(known.reduce((sum, usd) => sum + usd, 0)),
|
|
33
|
+
complete: lowerBound ? false : run.complete,
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* A run index entry's cost: its participants-and-desktops figure, complete or a lower bound when
|
|
39
|
+
* the status record says which, plus its analyses.
|
|
40
|
+
*/
|
|
41
|
+
export function indexedRunCost(entry) {
|
|
42
|
+
return runCost({
|
|
43
|
+
estimatedTotalUsd: entry.estimatedCostUsd ?? null,
|
|
44
|
+
...(entry.estimatedCostComplete === undefined
|
|
45
|
+
? {}
|
|
46
|
+
: { fullyEstimated: entry.estimatedCostComplete }),
|
|
47
|
+
}, entry.analysisCost);
|
|
48
|
+
}
|
|
49
|
+
/** Six decimals, as the run cost summary rounds, so sums of small estimates do not drift. */
|
|
50
|
+
function round6(value) {
|
|
51
|
+
return Math.round(value * 1e6) / 1e6;
|
|
52
|
+
}
|
|
53
|
+
/** An estimate, saying so when some of the spend has no price and is not in it. */
|
|
54
|
+
function estimateText(usd, complete) {
|
|
55
|
+
return `est. ~$${usd.toFixed(2)}${complete === false ? " plus unpriced usage" : ""}`;
|
|
56
|
+
}
|
|
57
|
+
function analysesNoun(requests) {
|
|
58
|
+
return requests === 1 ? "analysis" : `${requests} analyses`;
|
|
59
|
+
}
|
|
60
|
+
function billedBy(providers) {
|
|
61
|
+
const names = providers.map((provider) => provider === "openai" ? "OpenAI API key" : "Codex account");
|
|
62
|
+
return names.length === 0 ? "" : ` (${names.join(" and ")})`;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* The cost as separate parts: participants and desktops, then the analysis and the total when the
|
|
66
|
+
* run sent an analysis request. Empty when the run has no cost summary.
|
|
67
|
+
*/
|
|
68
|
+
export function runCostParts(cost) {
|
|
69
|
+
if (cost.run === null)
|
|
70
|
+
return [];
|
|
71
|
+
const rates = `rates as of ${cost.run.ratesAsOf}${cost.run.placeholder ? ", placeholder" : ""}`;
|
|
72
|
+
const parts = [
|
|
73
|
+
cost.run.usd === null
|
|
74
|
+
? "Participants + desktops: cost not estimated"
|
|
75
|
+
: `Participants + desktops: ${estimateText(cost.run.usd, cost.run.complete)} (${rates})`,
|
|
76
|
+
];
|
|
77
|
+
const analysis = cost.analysis;
|
|
78
|
+
if (analysis === null)
|
|
79
|
+
return parts;
|
|
80
|
+
const noun = analysesNoun(analysis.requests);
|
|
81
|
+
const label = noun.charAt(0).toUpperCase() + noun.slice(1);
|
|
82
|
+
if (analysis.estimatedUsd === null)
|
|
83
|
+
parts.push(analysis.providers.length === 1 && analysis.providers[0] === "codex"
|
|
84
|
+
? `${label}: Codex account, dollar cost unknown`
|
|
85
|
+
: `${label}: cost not estimated${billedBy(analysis.providers)}`);
|
|
86
|
+
else
|
|
87
|
+
parts.push(`${label}: ${estimateText(analysis.estimatedUsd, analysis.complete)}${billedBy(analysis.providers)}`);
|
|
88
|
+
if (cost.total !== null)
|
|
89
|
+
parts.push(`Total: ${estimateText(cost.total.usd, cost.total.complete)}`);
|
|
90
|
+
return parts;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* The cost in one short label for a list row: the total, and how many analyses it includes.
|
|
94
|
+
* Undefined when the run has no cost summary.
|
|
95
|
+
*/
|
|
96
|
+
export function runCostLabel(cost) {
|
|
97
|
+
if (cost.run === null)
|
|
98
|
+
return undefined;
|
|
99
|
+
const amount = cost.total === null
|
|
100
|
+
? "cost not estimated"
|
|
101
|
+
: `~$${cost.total.usd.toFixed(2)} est.${cost.total.complete === false ? " plus unpriced usage" : ""}`;
|
|
102
|
+
return cost.analysis === null
|
|
103
|
+
? amount
|
|
104
|
+
: `${amount}, with ${analysesNoun(cost.analysis.requests)}`;
|
|
105
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { type RunAnalysisCost } from "./run-cost.js";
|
|
2
|
+
/** Per-run analysis costs, keyed on their fingerprint. Nothing here is authoritative. */
|
|
3
|
+
export declare class AnalysisCostCache {
|
|
4
|
+
private readonly slots;
|
|
5
|
+
get(runId: string, fingerprint: string): {
|
|
6
|
+
cost: RunAnalysisCost | undefined;
|
|
7
|
+
} | undefined;
|
|
8
|
+
set(runId: string, fingerprint: string, cost: RunAnalysisCost | undefined): void;
|
|
9
|
+
retain(keep: Set<string>): void;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The run's analysis spend for its index entry, read with the same reader stats uses. Undefined
|
|
13
|
+
* when the run sent no analysis request or its run directory cannot be bound.
|
|
14
|
+
*/
|
|
15
|
+
export declare function readIndexedAnalysisCost(cwd: string, runId: string, cache: AnalysisCostCache | undefined): Promise<RunAnalysisCost | undefined>;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { readdir, stat } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { AUTOMATIC_ANALYSIS_DIRECTORY } from "../analysis/job.js";
|
|
4
|
+
import { ANALYSIS_DIRECTORY, ANALYSIS_EXECUTION_DIRECTORY } from "../analysis/store.js";
|
|
5
|
+
import { readAnalysisAccounting } from "./costs.js";
|
|
6
|
+
import { bindExistingRunArtifactPaths } from "./paths.js";
|
|
7
|
+
import { analysisCostOf } from "./run-cost.js";
|
|
8
|
+
// A run's analysis is written after its status record and bundle are final, so the index keys a
|
|
9
|
+
// run's analysis cost on a fingerprint of the analysis directories, not on the status record. A
|
|
10
|
+
// receipt lands inside an attempt directory that already exists, which changes that directory's
|
|
11
|
+
// mtime and not its parent's, so each attempt directory is part of the fingerprint.
|
|
12
|
+
/** Stat signatures of everything an analysis write changes; null when the run has no analysis. */
|
|
13
|
+
async function analysisFingerprint(runDir) {
|
|
14
|
+
const signatures = [];
|
|
15
|
+
for (const directory of [ANALYSIS_EXECUTION_DIRECTORY, ANALYSIS_DIRECTORY]) {
|
|
16
|
+
const root = path.join(runDir, directory);
|
|
17
|
+
const rootSignature = await signature(root);
|
|
18
|
+
if (rootSignature === null)
|
|
19
|
+
continue;
|
|
20
|
+
signatures.push(`${directory}:${rootSignature}`);
|
|
21
|
+
const children = await readdir(root).catch(() => []);
|
|
22
|
+
for (const child of children.sort((left, right) => left.localeCompare(right)))
|
|
23
|
+
signatures.push(`${directory}/${child}:${await signature(path.join(root, child))}`);
|
|
24
|
+
}
|
|
25
|
+
const job = await signature(path.join(runDir, AUTOMATIC_ANALYSIS_DIRECTORY, "job.json"));
|
|
26
|
+
if (job !== null)
|
|
27
|
+
signatures.push(`job:${job}`);
|
|
28
|
+
return signatures.length === 0 ? null : signatures.join("|");
|
|
29
|
+
}
|
|
30
|
+
async function signature(file) {
|
|
31
|
+
try {
|
|
32
|
+
const stats = await stat(file);
|
|
33
|
+
return `${Number(stats.ino)}:${stats.mtimeMs}:${stats.ctimeMs}:${stats.size}`;
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** Per-run analysis costs, keyed on their fingerprint. Nothing here is authoritative. */
|
|
40
|
+
export class AnalysisCostCache {
|
|
41
|
+
slots = new Map();
|
|
42
|
+
get(runId, fingerprint) {
|
|
43
|
+
const slot = this.slots.get(runId);
|
|
44
|
+
return slot?.fingerprint === fingerprint ? { cost: slot.cost } : undefined;
|
|
45
|
+
}
|
|
46
|
+
set(runId, fingerprint, cost) {
|
|
47
|
+
this.slots.set(runId, { fingerprint, cost });
|
|
48
|
+
}
|
|
49
|
+
retain(keep) {
|
|
50
|
+
for (const runId of this.slots.keys())
|
|
51
|
+
if (!keep.has(runId))
|
|
52
|
+
this.slots.delete(runId);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* The run's analysis spend for its index entry, read with the same reader stats uses. Undefined
|
|
57
|
+
* when the run sent no analysis request or its run directory cannot be bound.
|
|
58
|
+
*/
|
|
59
|
+
export async function readIndexedAnalysisCost(cwd, runId, cache) {
|
|
60
|
+
const fingerprint = await analysisFingerprint(path.join(cwd, ".humanish", "runs", runId));
|
|
61
|
+
if (fingerprint === null)
|
|
62
|
+
return undefined;
|
|
63
|
+
const cached = cache?.get(runId, fingerprint);
|
|
64
|
+
if (cached !== undefined)
|
|
65
|
+
return cached.cost;
|
|
66
|
+
let prepared;
|
|
67
|
+
try {
|
|
68
|
+
prepared = await bindExistingRunArtifactPaths(cwd, runId);
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
// Not cached: a directory that cannot be bound now may bind on the next refresh.
|
|
72
|
+
return undefined;
|
|
73
|
+
}
|
|
74
|
+
const cost = analysisCostOf(await readAnalysisAccounting(prepared)) ?? undefined;
|
|
75
|
+
cache?.set(runId, fingerprint, cost);
|
|
76
|
+
return cost;
|
|
77
|
+
}
|
package/dist/run/run-index.d.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { AnalysisCostCache } from "./run-index-analysis.js";
|
|
2
|
+
import type { RunAnalysisCost } from "./run-cost.js";
|
|
1
3
|
import { type RunLiveness, type RunStatusRecord } from "./status.js";
|
|
2
4
|
import { type RunStudyProvenance } from "./study-provenance.js";
|
|
3
5
|
declare const RUN_INDEX_SCHEMA = "humanish.run-index.v1";
|
|
@@ -23,8 +25,13 @@ export interface RunIndexEntry {
|
|
|
23
25
|
reachedGoal: number;
|
|
24
26
|
reportedFriction?: number;
|
|
25
27
|
};
|
|
28
|
+
/** Participants and desktops only; analysis bills separately and is in `analysisCost`. */
|
|
26
29
|
estimatedCostUsd?: number | null;
|
|
27
|
-
/**
|
|
30
|
+
/** False when that figure is a lower bound; absent when unknown (a status record before 0.109). */
|
|
31
|
+
estimatedCostComplete?: boolean;
|
|
32
|
+
/** The run's analysis requests, every attempt counted once; absent when it sent none. */
|
|
33
|
+
analysisCost?: RunAnalysisCost;
|
|
34
|
+
/** Wall-clock span when both ends are known; used for per-study medians. */
|
|
28
35
|
durationMs?: number;
|
|
29
36
|
}
|
|
30
37
|
export interface RunIndexResult {
|
|
@@ -49,6 +56,8 @@ interface CacheKey {
|
|
|
49
56
|
*/
|
|
50
57
|
export declare class RunIndexCache {
|
|
51
58
|
private readonly slots;
|
|
59
|
+
/** Analysis costs, keyed on their own fingerprint: analysis lands after the status record. */
|
|
60
|
+
readonly analysis: AnalysisCostCache;
|
|
52
61
|
get(runId: string, key: CacheKey): RunIndexEntry | undefined;
|
|
53
62
|
set(runId: string, key: CacheKey, entry: RunIndexEntry): void;
|
|
54
63
|
/** Drop entries for runs that no longer exist, so a long-lived surface cannot leak. */
|
|
@@ -81,6 +90,7 @@ interface BundleFacts {
|
|
|
81
90
|
};
|
|
82
91
|
cost?: {
|
|
83
92
|
estimatedTotalUsd?: number | null;
|
|
93
|
+
fullyEstimated?: unknown;
|
|
84
94
|
};
|
|
85
95
|
}
|
|
86
96
|
/**
|
package/dist/run/run-index.js
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
// - One unreadable run directory degrades that run, never the listing.
|
|
16
16
|
import { readdir, readFile, stat } from "node:fs/promises";
|
|
17
17
|
import path from "node:path";
|
|
18
|
+
import { AnalysisCostCache, readIndexedAnalysisCost } from "./run-index-analysis.js";
|
|
18
19
|
import { RUN_STATUS_FILE, classifyRunStatus, isRunStatusRecord, } from "./status.js";
|
|
19
20
|
import { studyProvenanceOf } from "./study-provenance.js";
|
|
20
21
|
import { RUN_BUNDLE_FILE } from "./bundle.js";
|
|
@@ -26,6 +27,8 @@ const RUN_INDEX_SCHEMA = "humanish.run-index.v1";
|
|
|
26
27
|
*/
|
|
27
28
|
export class RunIndexCache {
|
|
28
29
|
slots = new Map();
|
|
30
|
+
/** Analysis costs, keyed on their own fingerprint: analysis lands after the status record. */
|
|
31
|
+
analysis = new AnalysisCostCache();
|
|
29
32
|
get(runId, key) {
|
|
30
33
|
const slot = this.slots.get(runId);
|
|
31
34
|
if (slot === undefined)
|
|
@@ -46,6 +49,7 @@ export class RunIndexCache {
|
|
|
46
49
|
if (!keep.has(runId))
|
|
47
50
|
this.slots.delete(runId);
|
|
48
51
|
}
|
|
52
|
+
this.analysis.retain(keep);
|
|
49
53
|
}
|
|
50
54
|
get size() {
|
|
51
55
|
return this.slots.size;
|
|
@@ -90,6 +94,9 @@ function entryFromStatus(record, nowMs) {
|
|
|
90
94
|
...(record.outcome?.estimatedCostUsd === undefined
|
|
91
95
|
? {}
|
|
92
96
|
: { estimatedCostUsd: record.outcome.estimatedCostUsd }),
|
|
97
|
+
...(typeof record.outcome?.estimatedCostComplete === "boolean"
|
|
98
|
+
? { estimatedCostComplete: record.outcome.estimatedCostComplete }
|
|
99
|
+
: {}),
|
|
93
100
|
...(Number.isFinite(started) && Number.isFinite(ended) ? { durationMs: ended - started } : {}),
|
|
94
101
|
};
|
|
95
102
|
}
|
|
@@ -136,6 +143,9 @@ function entryFromBundle(runId, bundle) {
|
|
|
136
143
|
...(bundle.cost?.estimatedTotalUsd === undefined
|
|
137
144
|
? {}
|
|
138
145
|
: { estimatedCostUsd: bundle.cost.estimatedTotalUsd }),
|
|
146
|
+
...(typeof bundle.cost?.fullyEstimated === "boolean"
|
|
147
|
+
? { estimatedCostComplete: bundle.cost.fullyEstimated }
|
|
148
|
+
: {}),
|
|
139
149
|
};
|
|
140
150
|
}
|
|
141
151
|
/**
|
|
@@ -159,58 +169,15 @@ export async function readRunIndex(cwdInput, options = {}) {
|
|
|
159
169
|
const runs = [];
|
|
160
170
|
const unreadable = [];
|
|
161
171
|
for (const runId of runIds) {
|
|
162
|
-
const
|
|
163
|
-
|
|
164
|
-
const bundleFile = path.join(runDir, RUN_BUNDLE_FILE);
|
|
165
|
-
// Cheapest source first: the status record. Its stat is the cache key, so a live run whose
|
|
166
|
-
// record ticks every 5s re-reads 586 bytes and nothing else.
|
|
167
|
-
const statusKey = await statKey(statusFile);
|
|
168
|
-
if (statusKey !== null) {
|
|
169
|
-
const cached = cache?.get(runId, statusKey);
|
|
170
|
-
if (cached !== undefined) {
|
|
171
|
-
// A running record's liveness is time-dependent, so it is recomputed even on a cache hit:
|
|
172
|
-
// an unchanged record can still have gone stale. An ended record (finished or
|
|
173
|
-
// interrupted, both with completedAt) keeps the liveness it was cached with.
|
|
174
|
-
runs.push(cached.derivedFrom === "status" &&
|
|
175
|
-
cached.updatedAt !== undefined &&
|
|
176
|
-
cached.completedAt === undefined
|
|
177
|
-
? {
|
|
178
|
-
...cached,
|
|
179
|
-
liveness: classifyRunStatus({ state: "running", updatedAt: cached.updatedAt }, nowMs),
|
|
180
|
-
}
|
|
181
|
-
: cached);
|
|
182
|
-
continue;
|
|
183
|
-
}
|
|
184
|
-
const raw = await readJson(statusFile);
|
|
185
|
-
if (usableStatusRecord(raw, runId)) {
|
|
186
|
-
const entry = entryFromStatus(raw, nowMs);
|
|
187
|
-
cache?.set(runId, statusKey, entry);
|
|
188
|
-
runs.push(entry);
|
|
189
|
-
continue;
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
// No usable status record: fall back to the bundle. This is every run written before the
|
|
193
|
-
// contract existed, and it is why an old project still lists correctly.
|
|
194
|
-
const bundleKey = await statKey(bundleFile);
|
|
195
|
-
if (bundleKey !== null) {
|
|
196
|
-
const cached = cache?.get(runId, bundleKey);
|
|
197
|
-
if (cached !== undefined) {
|
|
198
|
-
runs.push(cached);
|
|
199
|
-
continue;
|
|
200
|
-
}
|
|
201
|
-
const raw = await readJson(bundleFile);
|
|
202
|
-
if (raw !== null && typeof raw === "object") {
|
|
203
|
-
const entry = entryFromBundle(runId, raw);
|
|
204
|
-
cache?.set(runId, bundleKey, entry);
|
|
205
|
-
runs.push(entry);
|
|
206
|
-
continue;
|
|
207
|
-
}
|
|
172
|
+
const entry = await readEntry(runId, path.join(runsRoot, runId), cache, nowMs);
|
|
173
|
+
if (entry === "unreadable") {
|
|
208
174
|
unreadable.push(runId);
|
|
209
175
|
continue;
|
|
210
176
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
177
|
+
const analysisCost = entry.derivedFrom === "directory"
|
|
178
|
+
? undefined
|
|
179
|
+
: await readIndexedAnalysisCost(cwd, runId, cache?.analysis);
|
|
180
|
+
runs.push(analysisCost === undefined ? entry : { ...entry, analysisCost });
|
|
214
181
|
}
|
|
215
182
|
cache?.retain(runIds);
|
|
216
183
|
runs.sort((left, right) => sortKey(right) - sortKey(left));
|
|
@@ -221,6 +188,53 @@ export async function readRunIndex(cwdInput, options = {}) {
|
|
|
221
188
|
unreadable,
|
|
222
189
|
};
|
|
223
190
|
}
|
|
191
|
+
/** One run's entry, cheapest source first, or "unreadable" when its bundle cannot be parsed. */
|
|
192
|
+
async function readEntry(runId, runDir, cache, nowMs) {
|
|
193
|
+
const statusFile = path.join(runDir, RUN_STATUS_FILE);
|
|
194
|
+
const bundleFile = path.join(runDir, RUN_BUNDLE_FILE);
|
|
195
|
+
// Cheapest source first: the status record. Its stat is the cache key, so a live run whose
|
|
196
|
+
// record ticks every 5s re-reads 586 bytes and nothing else.
|
|
197
|
+
const statusKey = await statKey(statusFile);
|
|
198
|
+
if (statusKey !== null) {
|
|
199
|
+
const cached = cache?.get(runId, statusKey);
|
|
200
|
+
if (cached !== undefined) {
|
|
201
|
+
// A running record's liveness is time-dependent, so it is recomputed even on a cache hit:
|
|
202
|
+
// an unchanged record can still have gone stale. An ended record (finished or
|
|
203
|
+
// interrupted, both with completedAt) keeps the liveness it was cached with.
|
|
204
|
+
return cached.derivedFrom === "status" &&
|
|
205
|
+
cached.updatedAt !== undefined &&
|
|
206
|
+
cached.completedAt === undefined
|
|
207
|
+
? {
|
|
208
|
+
...cached,
|
|
209
|
+
liveness: classifyRunStatus({ state: "running", updatedAt: cached.updatedAt }, nowMs),
|
|
210
|
+
}
|
|
211
|
+
: cached;
|
|
212
|
+
}
|
|
213
|
+
const raw = await readJson(statusFile);
|
|
214
|
+
if (usableStatusRecord(raw, runId)) {
|
|
215
|
+
const entry = entryFromStatus(raw, nowMs);
|
|
216
|
+
cache?.set(runId, statusKey, entry);
|
|
217
|
+
return entry;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
// No usable status record: fall back to the bundle. This is every run written before the
|
|
221
|
+
// contract existed, and it is why an old project still lists correctly.
|
|
222
|
+
const bundleKey = await statKey(bundleFile);
|
|
223
|
+
if (bundleKey !== null) {
|
|
224
|
+
const cached = cache?.get(runId, bundleKey);
|
|
225
|
+
if (cached !== undefined)
|
|
226
|
+
return cached;
|
|
227
|
+
const raw = await readJson(bundleFile);
|
|
228
|
+
if (raw === null || typeof raw !== "object")
|
|
229
|
+
return "unreadable";
|
|
230
|
+
const entry = entryFromBundle(runId, raw);
|
|
231
|
+
cache?.set(runId, bundleKey, entry);
|
|
232
|
+
return entry;
|
|
233
|
+
}
|
|
234
|
+
// Neither record nor bundle: receipts without an outcome. That is precisely an interrupted
|
|
235
|
+
// run (the shape a dropped connection leaves), and saying so is more useful than hiding it.
|
|
236
|
+
return { runId, derivedFrom: "directory", liveness: "interrupted" };
|
|
237
|
+
}
|
|
224
238
|
/** Newest-first ordering: the most recent thing known about a run, else its id's own timestamp. */
|
|
225
239
|
function sortKey(entry) {
|
|
226
240
|
for (const candidate of [entry.completedAt, entry.updatedAt, entry.startedAt]) {
|
package/dist/run/run.d.ts
CHANGED
|
@@ -11,10 +11,10 @@ interface StartRunOptions {
|
|
|
11
11
|
runId?: string | undefined;
|
|
12
12
|
mintRunId: () => string;
|
|
13
13
|
mode: "dry-run" | "live";
|
|
14
|
-
|
|
14
|
+
study?: RunStudyProvenance | undefined;
|
|
15
15
|
/** review.md for the published bundle. */
|
|
16
16
|
renderReview: (bundle: RunBundle) => string;
|
|
17
|
-
/** Used by `FinishedRun.renderObserver`; `render` is the `
|
|
17
|
+
/** Used by `FinishedRun.renderObserver`; `render` is the `StudyDeps.renderObserver` seam. */
|
|
18
18
|
observer?: {
|
|
19
19
|
open: boolean;
|
|
20
20
|
render?: typeof renderObserver | undefined;
|
|
@@ -27,7 +27,7 @@ interface Run {
|
|
|
27
27
|
readonly createdAt: string;
|
|
28
28
|
readonly mode: "dry-run" | "live";
|
|
29
29
|
/** The manifest the run came from; bundleHead copies it into the bundle. */
|
|
30
|
-
readonly
|
|
30
|
+
readonly study?: RunStudyProvenance;
|
|
31
31
|
/** Routes write their evidence files through these and hand them to participants. */
|
|
32
32
|
readonly paths: PreparedRunArtifactPaths;
|
|
33
33
|
/**
|
package/dist/run/run.js
CHANGED
|
@@ -88,7 +88,7 @@ export async function runScope(fn) {
|
|
|
88
88
|
const created = await createRunArtifactPaths(options.cwd, runId);
|
|
89
89
|
if (!created.ok)
|
|
90
90
|
return created;
|
|
91
|
-
// beginRunStatus reads only the mode and the
|
|
91
|
+
// beginRunStatus reads only the mode and the study provenance from the run's options.
|
|
92
92
|
status = beginRunStatus(created.paths, { ...options, runId });
|
|
93
93
|
// Registered before the first write lands: a signal while it is still settling reaches
|
|
94
94
|
// the handle, whose interrupt queues behind that write.
|
|
@@ -156,7 +156,7 @@ export async function runScope(fn) {
|
|
|
156
156
|
runId,
|
|
157
157
|
createdAt,
|
|
158
158
|
mode: options.mode,
|
|
159
|
-
...(options.
|
|
159
|
+
...(options.study === undefined ? {} : { study: options.study }),
|
|
160
160
|
paths,
|
|
161
161
|
writeSnapshot(bundle) {
|
|
162
162
|
if (closed)
|
|
@@ -15,7 +15,7 @@ export interface SharedWorldPlane {
|
|
|
15
15
|
* (the first-class provisioned-subject target every actor drove). A digest replaces the
|
|
16
16
|
* raw URL: a getHost URL embeds the (live) sandbox id and matches the publish-safety e2b-URL
|
|
17
17
|
* redaction, so, like the stream URL and sandbox ids, it never lands raw in a published
|
|
18
|
-
* bundle (the raw tokenless URL is surfaced only on the ephemeral
|
|
18
|
+
* bundle (the raw tokenless URL is surfaced only on the ephemeral study result). The orchestrator
|
|
19
19
|
* confirms the URL is tokenless (no authKey, so no secret reaches the bundle) before digesting.
|
|
20
20
|
* verify proves every actor drove this host by digest equality. Absent on the sequential route.
|
|
21
21
|
*/
|
|
@@ -4,7 +4,7 @@ import { isNonNegativeSafeInteger, isRecord } from "./type-guards.js";
|
|
|
4
4
|
// The promptDigest convention: sha256 hex, first 16 chars. A "seeded" record without a real
|
|
5
5
|
// digest cannot pin "same recipe" across bundles, so verify treats it as a hollow claim.
|
|
6
6
|
export const COMMAND_DIGEST_PATTERN = /^[0-9a-f]{16}$/;
|
|
7
|
-
// Env var name shape (mirrors ENV_NAME_PATTERN in src/
|
|
7
|
+
// Env var name shape (mirrors ENV_NAME_PATTERN in src/study/parse/values.ts). externalEnvNames
|
|
8
8
|
// must hold names only: a value sneaking into the list trips this check (a free secret tripwire).
|
|
9
9
|
export const SUBJECT_ENV_NAME_PATTERN = /^[A-Z][A-Z0-9_]*$/;
|
|
10
10
|
// Sequential: the three disclosures a sequential shared-world bundle must pin (verify fails closed
|
package/dist/run/stats.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ interface StatsParticipants {
|
|
|
5
5
|
reachedGoal: number;
|
|
6
6
|
reportedFriction: number;
|
|
7
7
|
}
|
|
8
|
-
interface
|
|
8
|
+
interface StatsStudyRow {
|
|
9
9
|
/** The study's `id`. */
|
|
10
10
|
study: string;
|
|
11
11
|
runs: number;
|
|
@@ -55,7 +55,7 @@ export interface StatsResult {
|
|
|
55
55
|
verdicts: Record<string, number>;
|
|
56
56
|
costs: CostTotals;
|
|
57
57
|
};
|
|
58
|
-
studies:
|
|
58
|
+
studies: StatsStudyRow[];
|
|
59
59
|
days: StatsDayRow[];
|
|
60
60
|
/** Same selected runs; individual accounting gaps are inspectable without reading private evidence. */
|
|
61
61
|
costsByRun: CostRow[];
|
|
@@ -67,7 +67,7 @@ export interface StatsOptions {
|
|
|
67
67
|
/** ISO date or datetime; runs that started before it are excluded. */
|
|
68
68
|
since?: string;
|
|
69
69
|
/** Lab id; runs from other labs are excluded. */
|
|
70
|
-
|
|
70
|
+
study?: string;
|
|
71
71
|
nowMs?: number;
|
|
72
72
|
}
|
|
73
73
|
export interface StatsFailure {
|
package/dist/run/stats.js
CHANGED
|
@@ -40,12 +40,12 @@ function runEstimateUsd(entry) {
|
|
|
40
40
|
function entryTime(entry) {
|
|
41
41
|
return entry.startedAt ?? entry.completedAt ?? entry.updatedAt;
|
|
42
42
|
}
|
|
43
|
-
function
|
|
44
|
-
const
|
|
45
|
-
let row =
|
|
43
|
+
function addToStudyRow(studies, entry, runUsd, costs) {
|
|
44
|
+
const studyId = entry.study?.id ?? "(no study)";
|
|
45
|
+
let row = studies.get(studyId);
|
|
46
46
|
if (row === undefined) {
|
|
47
47
|
row = {
|
|
48
|
-
study:
|
|
48
|
+
study: studyId,
|
|
49
49
|
runs: 0,
|
|
50
50
|
live: 0,
|
|
51
51
|
dryRun: 0,
|
|
@@ -59,7 +59,7 @@ function addToLabRow(labs, entry, runUsd, costs) {
|
|
|
59
59
|
costs: emptyCostTotals(),
|
|
60
60
|
runCosts: [],
|
|
61
61
|
};
|
|
62
|
-
|
|
62
|
+
studies.set(studyId, row);
|
|
63
63
|
}
|
|
64
64
|
row.runs += 1;
|
|
65
65
|
if (entry.mode === "live")
|
|
@@ -123,7 +123,7 @@ export async function computeStats(cwdInput, options = {}) {
|
|
|
123
123
|
}
|
|
124
124
|
const index = await readRunIndex(cwd, options.nowMs === undefined ? {} : { nowMs: options.nowMs });
|
|
125
125
|
// Corrupt source metadata must not hide separately retained paid analysis receipts.
|
|
126
|
-
// These directories cannot be attributed to a
|
|
126
|
+
// These directories cannot be attributed to a study or date, so scoped filters exclude them.
|
|
127
127
|
const indexedIds = new Set(index.runs.map((entry) => entry.runId));
|
|
128
128
|
const entries = [
|
|
129
129
|
...index.runs,
|
|
@@ -136,7 +136,7 @@ export async function computeStats(cwdInput, options = {}) {
|
|
|
136
136
|
})),
|
|
137
137
|
];
|
|
138
138
|
const selected = entries.filter((entry) => {
|
|
139
|
-
if (options.
|
|
139
|
+
if (options.study !== undefined && entry.study?.id !== options.study)
|
|
140
140
|
return false;
|
|
141
141
|
if (sinceMs !== undefined) {
|
|
142
142
|
const at = entryTime(entry);
|
|
@@ -159,7 +159,7 @@ export async function computeStats(cwdInput, options = {}) {
|
|
|
159
159
|
verdicts: {},
|
|
160
160
|
costs: emptyCostTotals(),
|
|
161
161
|
};
|
|
162
|
-
const
|
|
162
|
+
const studies = new Map();
|
|
163
163
|
const days = new Map();
|
|
164
164
|
const costsByRun = [];
|
|
165
165
|
for (const entry of selected) {
|
|
@@ -181,10 +181,10 @@ export async function computeStats(cwdInput, options = {}) {
|
|
|
181
181
|
addParticipants(totals.participants, entry);
|
|
182
182
|
if (entry.verdict !== undefined)
|
|
183
183
|
totals.verdicts[entry.verdict] = (totals.verdicts[entry.verdict] ?? 0) + 1;
|
|
184
|
-
|
|
184
|
+
addToStudyRow(studies, entry, runUsd, accounting.costs);
|
|
185
185
|
addToDayRow(days, entry, runUsd, accounting.costs);
|
|
186
186
|
}
|
|
187
|
-
const
|
|
187
|
+
const studyRows = [...studies.values()]
|
|
188
188
|
.map(({ durations, runCosts, ...row }) => ({
|
|
189
189
|
...row,
|
|
190
190
|
...(row.judged === 0 ? {} : { passRate: round6(row.passed / row.judged) }),
|
|
@@ -199,9 +199,9 @@ export async function computeStats(cwdInput, options = {}) {
|
|
|
199
199
|
ok: true,
|
|
200
200
|
cwd,
|
|
201
201
|
...(options.since === undefined ? {} : { since: options.since }),
|
|
202
|
-
...(options.
|
|
202
|
+
...(options.study === undefined ? {} : { study: options.study }),
|
|
203
203
|
totals: { ...totals, estimatedSpendUsd: round6(totals.estimatedSpendUsd) },
|
|
204
|
-
studies:
|
|
204
|
+
studies: studyRows,
|
|
205
205
|
days: [...days.values()]
|
|
206
206
|
.map((row) => ({ ...row, estimatedSpendUsd: round6(row.estimatedSpendUsd) }))
|
|
207
207
|
.sort((a, b) => a.day.localeCompare(b.day)),
|
|
@@ -282,7 +282,7 @@ export function formatStatsHuman(result) {
|
|
|
282
282
|
return { error: result.error };
|
|
283
283
|
const t = result.totals;
|
|
284
284
|
const scope = [
|
|
285
|
-
result.study === undefined ? undefined : `
|
|
285
|
+
result.study === undefined ? undefined : `study ${result.study}`,
|
|
286
286
|
result.since === undefined ? undefined : `since ${result.since}`,
|
|
287
287
|
].filter((part) => part !== undefined);
|
|
288
288
|
const lines = [
|
|
@@ -295,7 +295,7 @@ export function formatStatsHuman(result) {
|
|
|
295
295
|
if (reportable) {
|
|
296
296
|
lines.push(...totalsLines(t));
|
|
297
297
|
if (result.studies.length > 0)
|
|
298
|
-
lines.push("", "per
|
|
298
|
+
lines.push("", "per study:", ...result.studies.map((row) => {
|
|
299
299
|
const rate = row.passRate === undefined ? "no verdicts" : `${row.passed} of ${row.judged} passed`;
|
|
300
300
|
const duration = row.medianDurationMs === undefined
|
|
301
301
|
? "no timed live runs"
|