humanish 0.110.0 → 0.110.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +15 -12
- package/CHANGELOG.md +26 -0
- package/CONTEXT.md +8 -6
- package/README.md +103 -53
- package/dist/actors/computer-use/loop/ending.js +4 -4
- package/dist/actors/local-agent/claude-participant.d.ts +74 -0
- package/dist/actors/local-agent/claude-participant.js +243 -0
- package/dist/actors/local-agent/claude-session.d.ts +16 -3
- package/dist/actors/local-agent/claude-session.js +64 -23
- package/dist/actors/local-agent/cli.d.ts +17 -0
- package/dist/actors/local-agent/cli.js +111 -24
- package/dist/actors/local-agent/readiness.d.ts +2 -2
- package/dist/actors/local-agent/readiness.js +20 -12
- package/dist/cli/commands/analyze.js +21 -5
- package/dist/cli/commands/observe.js +15 -2
- package/dist/cli/commands/run-signals.d.ts +2 -0
- package/dist/cli/commands/run-signals.js +106 -45
- package/dist/cli/commands/runs.d.ts +5 -0
- package/dist/cli/commands/runs.js +113 -74
- package/dist/cli/commands/setup.js +23 -14
- package/dist/cli/commands/study-route-computer-use.js +9 -3
- package/dist/cli/commands/study-route-open.js +1 -0
- package/dist/cli/commands/study-route-preview.js +1 -0
- package/dist/cli/commands/study-route-scripted.js +2 -0
- package/dist/cli/commands/study-route-shared-world.js +6 -2
- package/dist/cli/commands/study-route-terminal.js +2 -0
- package/dist/cli/commands/watch.js +14 -4
- package/dist/cli/doctor.d.ts +4 -1
- package/dist/cli/doctor.js +51 -10
- package/dist/cli/findings.d.ts +103 -0
- package/dist/cli/findings.js +317 -0
- package/dist/cli/first-run-path.d.ts +32 -11
- package/dist/cli/first-run-path.js +113 -69
- package/dist/cli/invocation.d.ts +7 -0
- package/dist/cli/invocation.js +32 -0
- package/dist/cli/io.d.ts +3 -1
- package/dist/cli/io.js +8 -15
- package/dist/cli/observer-follow.d.ts +20 -0
- package/dist/cli/observer-follow.js +36 -1
- package/dist/cli/orientation.js +16 -14
- package/dist/cli/product-sentence.d.ts +3 -3
- package/dist/cli/product-sentence.js +3 -3
- package/dist/cli/version.d.ts +1 -0
- package/dist/cli/version.js +11 -0
- package/dist/evidence/encoded-text.d.ts +9 -6
- package/dist/evidence/encoded-text.js +84 -19
- package/dist/evidence/redaction.d.ts +2 -0
- package/dist/evidence/redaction.js +123 -21
- package/dist/feedback/export-bundle.js +2 -2
- package/dist/keys/env-file.d.ts +4 -0
- package/dist/keys/env-file.js +17 -0
- package/dist/keys/key-resolution.js +6 -5
- package/dist/observer/data.d.ts +14 -1
- package/dist/observer/data.js +19 -9
- package/dist/observer/exposure.js +1 -1
- package/dist/observer/library.d.ts +4 -0
- package/dist/observer/library.js +21 -17
- package/dist/observer/render.js +10 -3
- package/dist/observer/run-routes.d.ts +2 -0
- package/dist/observer/run-routes.js +51 -35
- package/dist/observer/serve.js +2 -1
- package/dist/observer-app.html +3 -3
- package/dist/routes/computer-use/bundle.js +3 -1
- package/dist/routes/computer-use/fanout-records.js +1 -1
- package/dist/routes/computer-use/participant-execution.js +3 -1
- package/dist/routes/computer-use/participant-model.js +5 -2
- package/dist/routes/computer-use/plan.js +9 -4
- package/dist/routes/computer-use/preflight.js +1 -1
- package/dist/routes/computer-use/result.js +17 -28
- package/dist/routes/computer-use/single-bundle.js +7 -1
- package/dist/routes/scripted/bundle.js +4 -2
- package/dist/routes/scripted/plan.js +2 -2
- package/dist/routes/scripted/result.d.ts +3 -5
- package/dist/routes/scripted/result.js +16 -22
- package/dist/routes/scripted/scenario.js +4 -4
- package/dist/routes/shared-world/bundle-records.js +6 -1
- package/dist/routes/shared-world/bundle.js +2 -0
- package/dist/routes/shared-world/plan.js +6 -2
- package/dist/routes/shared-world/result.js +22 -22
- package/dist/routes/terminal/bundle.js +6 -3
- package/dist/routes/terminal/credentials.d.ts +5 -5
- package/dist/routes/terminal/credentials.js +14 -6
- package/dist/routes/terminal/dry-run.js +8 -12
- package/dist/routes/terminal/ledger.d.ts +3 -3
- package/dist/routes/terminal/ledger.js +6 -6
- package/dist/routes/terminal/live-finish.js +15 -13
- package/dist/routes/terminal/plan.js +12 -4
- package/dist/routes/terminal/result.d.ts +6 -7
- package/dist/routes/terminal/result.js +6 -22
- package/dist/routes/terminal/route.js +5 -4
- package/dist/routes/terminal/runtime-auth.d.ts +6 -0
- package/dist/routes/terminal/runtime-auth.js +5 -0
- package/dist/routes/terminal/session.js +1 -1
- package/dist/run/bundle-shape.d.ts +3 -1
- package/dist/run/bundle-shape.js +25 -1
- package/dist/run/bundle.d.ts +24 -0
- package/dist/run/display.d.ts +59 -0
- package/dist/run/display.js +106 -0
- package/dist/run/dry-run.js +13 -8
- package/dist/run/judge.d.ts +3 -2
- package/dist/run/judge.js +12 -1
- package/dist/run/local-only-files.d.ts +13 -0
- package/dist/run/local-only-files.js +35 -0
- package/dist/run/pricing.js +1 -1
- package/dist/run/projection.d.ts +0 -2
- package/dist/run/projection.js +0 -11
- package/dist/run/reclaim-outcomes.d.ts +89 -0
- package/dist/run/reclaim-outcomes.js +213 -0
- package/dist/run/reclaim.d.ts +23 -17
- package/dist/run/reclaim.js +213 -114
- package/dist/run/results.d.ts +10 -20
- package/dist/run/results.js +2 -2
- package/dist/run/run-index.d.ts +11 -1
- package/dist/run/run-index.js +11 -0
- package/dist/run/run-not-found.js +3 -2
- package/dist/run/run.d.ts +34 -18
- package/dist/run/run.js +166 -67
- package/dist/run/sandbox-creates.d.ts +59 -0
- package/dist/run/sandbox-creates.js +140 -0
- package/dist/run/sandbox-receipts.d.ts +20 -5
- package/dist/run/sandbox-receipts.js +56 -7
- package/dist/run/stats.d.ts +5 -0
- package/dist/run/stats.js +8 -4
- package/dist/run/status.d.ts +6 -9
- package/dist/run/status.js +7 -23
- package/dist/run/stored-runs.d.ts +5 -14
- package/dist/run/stored-runs.js +31 -146
- package/dist/run/streams.d.ts +3 -2
- package/dist/study/agents-section.d.ts +21 -0
- package/dist/study/agents-section.js +86 -0
- package/dist/study/doctor.d.ts +1 -1
- package/dist/study/doctor.js +8 -4
- package/dist/study/init-templates.js +0 -95
- package/dist/study/init.js +45 -37
- package/dist/study/keys.d.ts +12 -0
- package/dist/study/keys.js +32 -0
- package/dist/study/local-browser-host.d.ts +11 -0
- package/dist/study/local-browser-host.js +42 -0
- package/dist/study/parse/actors.js +9 -1
- package/dist/study/parse/subject.js +12 -0
- package/dist/study/parse/url-credentials.d.ts +9 -0
- package/dist/study/parse/url-credentials.js +50 -0
- package/dist/study/persona.d.ts +4 -0
- package/dist/study/persona.js +4 -2
- package/dist/study/preflight-probes.js +2 -1
- package/dist/study/types.d.ts +8 -6
- package/dist/study/url-credentials.d.ts +6 -0
- package/dist/study/url-credentials.js +34 -0
- package/dist/substrates/e2b/peer-install.d.ts +51 -0
- package/dist/substrates/e2b/peer-install.js +161 -0
- package/dist/substrates/e2b/sandbox.d.ts +34 -6
- package/dist/substrates/e2b/sandbox.js +141 -30
- package/dist/substrates/e2b/sdk.d.ts +33 -0
- package/dist/substrates/e2b/sdk.js +21 -21
- package/dist/substrates/local/firecracker-desktop.js +3 -2
- package/dist/substrates/local/runtime-host.js +5 -4
- package/dist/substrates/local/runtime.js +5 -4
- package/dist/substrates/shell.d.ts +2 -0
- package/dist/substrates/shell.js +4 -0
- package/dist/tui/contract.js +5 -4
- package/dist/tui-app.js +133 -133
- package/dist/verify/liveness.d.ts +19 -3
- package/dist/verify/liveness.js +47 -25
- package/dist/verify/verify.d.ts +3 -0
- package/dist/verify/verify.js +8 -3
- package/docs/architecture/actor-contract.md +2 -1
- package/docs/architecture/observer.md +22 -7
- package/docs/architecture/task-protocol-support.md +13 -13
- package/docs/architecture/terminal-product-route.md +15 -13
- package/docs/contracts/run-bundle.md +95 -10
- package/docs/contracts/schemas.md +44 -26
- package/docs/contracts/study-analysis.md +45 -0
- package/docs/contracts/study-costs.md +8 -0
- package/docs/decisions/0001-run-bundle-is-the-source-of-truth.md +3 -0
- package/docs/decisions/0003-contained-paths-and-cleanup-authority.md +9 -3
- package/docs/principles/invariants-and-defaults.md +11 -8
- package/package.json +6 -3
- package/skills/humanish/SKILL.md +38 -20
package/ARCHITECTURE.md
CHANGED
|
@@ -63,11 +63,14 @@ not.
|
|
|
63
63
|
`judgeComputerUseRun` (`src/routes/computer-use/bundle.ts`) and builds the bundle with
|
|
64
64
|
`buildCuaRunBundle` (`src/routes/computer-use/bundle.ts`), by the rules in `src/run/judge.ts`. A
|
|
65
65
|
declared scorer's failures fold in through `foldScorerFailures` (`src/run/judge.ts`), so a scorer
|
|
66
|
-
can fail a run and never pass one. `Run.finish` publishes `run.json
|
|
67
|
-
outcome
|
|
66
|
+
can fail a run and never pass one. `Run.finish` publishes `run.json` with the run's `ok` and
|
|
67
|
+
execution outcome in its `outcome`, then the `status.json` outcome copied from it, the review
|
|
68
|
+
and Observer data, and `.humanish/runs/latest.json` last.
|
|
68
69
|
7. **Observer.** `renderObserver` (`src/observer/render.ts`) verifies the bundle with
|
|
69
70
|
`verifyRunPrepared` (`src/verify/verify.ts`) and writes `observer/index.html` from
|
|
70
|
-
`buildObserverData` (`src/observer/data.ts`).
|
|
71
|
+
`buildObserverData` (`src/observer/data.ts`). When it does not render, `FinishedRun.renderObserver`
|
|
72
|
+
(`src/run/run.ts`) adds the `evidence` failure to `run.json`'s outcome. Every surface that shows
|
|
73
|
+
whether a run passed reads that outcome through `runDisplay` (`src/run/display.ts`).
|
|
71
74
|
8. **Analysis.** `completeAutomaticAnalysis` (`src/analysis/automatic-completion.ts`) runs
|
|
72
75
|
`runAutomaticAnalysis` (`src/analysis/automatic.ts`) on the `FinishedRun` that `Run.finish`
|
|
73
76
|
issued. Dry runs and studies with `review.analysis: false` skip it.
|
|
@@ -104,7 +107,7 @@ not.
|
|
|
104
107
|
| `humanish/` | This repo's own studies, personas, scenarios, fixtures and coverage notes | `humanish/studies/first-run.yaml` |
|
|
105
108
|
| `examples/` | Library examples shipped in the npm package: a participant and a scorer | `examples/README.md` |
|
|
106
109
|
| `adapters/` | Adapter fixture sets that `tests/study/adapter-fixtures.test.ts` checks | `adapters/fixtures/README.md` |
|
|
107
|
-
| `bench/` | Benchmark apps with planted defects and
|
|
110
|
+
| `bench/` | Benchmark apps with planted defects, the scored runner behind `pnpm bench` and the September results | `bench/DEFECTS.md` |
|
|
108
111
|
| `fixtures/` | Synthetic apps and cases that tests and scripts copy | `fixtures/minimal-app/README.md` |
|
|
109
112
|
| `skills/` | The companion agent skill that `npx skills add` installs | `skills/humanish/SKILL.md` |
|
|
110
113
|
| `runtime/` | Desktop and browser image recipes | `runtime/browser-guest/README.md` |
|
|
@@ -126,14 +129,14 @@ Each row names the code that enforces a guarantee and the tests that pin it. The
|
|
|
126
129
|
numbered invariants are in [invariants and defaults](docs/principles/invariants-and-defaults.md#invariants);
|
|
127
130
|
comments name an invariant by its rule.
|
|
128
131
|
|
|
129
|
-
| Guarantee | Enforced by
|
|
130
|
-
| ------------------------------------------------------------------------------------------------------------------------ |
|
|
131
|
-
| The run bundle is the source of truth, and the Observer derives from it | `verifyRun` grades only `run.json` and its artifacts (`src/verify/verify.ts`). It reads `status.json` and the sandbox and reclaim receipts only for the `RUN_NOT_FINISHED` warning (`
|
|
132
|
-
| Unsupported execution is refused before side effects | `parseStudy` (`src/study/config.ts`), `taskProtocolValidationReason` (`src/study/validation.ts`), `planComputerUseStudy` (`src/routes/computer-use/plan.ts`)
|
|
133
|
-
| A sandbox id is recorded before any work runs in it | `acquireE2BDesktopSandbox` and `acquireE2BShellSandbox` (`src/substrates/e2b/sandbox.ts`) write the receipt before they return the handle, and every route that creates a sandbox calls them; `reclaimRunSandboxes` (`src/run/reclaim.ts`) switches on the receipt's provider
|
|
134
|
-
| Bundles carry no secrets | `redactText` and `scrubLiterals` (`src/evidence/redaction.ts`); `scanRunPublicSafetyArtifacts` (`src/verify/artifacts.ts`); `buildShareSafety` (`src/verify/verify.ts`) grades `share_ready`, `local_only` or `blocked`
|
|
135
|
-
| Goldens pin route output | `runDirSnapshot` (`tests/helpers/run-golden.ts`) snapshots a whole run folder
|
|
136
|
-
| A run is closed on every exit, only a published run is analyzed, and `latest.json` moves only after its run is published | `runScope` and `FinishedRun` (`src/run/run.ts`); `completeAutomaticAnalysis` requires an issued `FinishedRun` for the result's run; `Run.finish` writes `latest.json` last through `writePreparedRunLatestPointer` (`src/run/contained-output.ts`)
|
|
132
|
+
| Guarantee | Enforced by | Pinned by |
|
|
133
|
+
| ------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
134
|
+
| The run bundle is the source of truth, and the Observer derives from it | `verifyRun` grades only `run.json` and its artifacts (`src/verify/verify.ts`). It reads `status.json` and the sandbox and reclaim receipts only for the `RUN_NOT_FINISHED` warning (`runNotFinished` in `src/verify/liveness.ts`), which changes no check and no grade. `buildObserverData` takes the bundle (`src/observer/data.ts`) | `tests/observer/data-contract.test.ts`; `tests/verify/run-not-finished.test.ts` checks that the grades are the same when `status.json` contradicts the bundle or is gone; `docs/decisions/0001-run-bundle-is-the-source-of-truth.md` |
|
|
135
|
+
| Unsupported execution is refused before side effects | `parseStudy` (`src/study/config.ts`), `taskProtocolValidationReason` (`src/study/validation.ts`), `planComputerUseStudy` (`src/routes/computer-use/plan.ts`) | `tests/admission/library.test.ts` checks each library refusal for a run directory, a desktop module load, a caller executor or provider, and a subprocess; `tests/admission/cli.test.ts` checks each CLI refusal for a run directory, a started process and a scorer's host code; `tests/study/task-route-preflight.test.ts`; `pnpm cli:preflight:test` runs `scripts/task-route-preflight-proof.mjs` with `tests/fixtures/task-route-preflight/deny-side-effects.mjs` preloaded |
|
|
136
|
+
| A sandbox id is recorded before any work runs in it | `acquireE2BDesktopSandbox` and `acquireE2BShellSandbox` (`src/substrates/e2b/sandbox.ts`) tag the sandbox with its run's owner tags, register the create with `beginSandboxCreate` (`src/run/sandbox-creates.ts`) and write the receipt before they return the handle, and every route that creates a sandbox calls them; `reclaimRunSandboxes` (`src/run/reclaim.ts`) switches on the receipt's provider, kills by exact id and by the run's exact tags | `tests/substrates/e2b/sandbox.test.ts`, which also checks that `Sandbox.create` has one call site in `src/`, `tests/routes/terminal/acquisition-boundary.test.ts`, `tests/run/reclaim.test.ts`; `tests/routes/scripted/route.test.ts` checks create, receipt, then first work |
|
|
137
|
+
| Bundles carry no secrets | `redactText` and `scrubLiterals` (`src/evidence/redaction.ts`); `scanRunPublicSafetyArtifacts` (`src/verify/artifacts.ts`); `buildShareSafety` (`src/verify/verify.ts`) grades `share_ready`, `local_only` or `blocked` | `tests/evidence/redaction-hooks.test.ts`, `tests/run/transient-comms-secrets.test.ts`, `tests/verify/evidence-refs.test.ts` |
|
|
138
|
+
| Goldens pin route output | `runDirSnapshot` (`tests/helpers/run-golden.ts`) snapshots a whole run folder | `tests/golden/routes/`, `tests/golden/observer-data/`, `tests/golden/labs/`; `tests/study/plan-lab.test.ts` checks that every `StudyRoute` has a dry-run golden |
|
|
139
|
+
| A run is closed on every exit, only a published run is analyzed, and `latest.json` moves only after its run is published | `runScope` and `FinishedRun` (`src/run/run.ts`); `completeAutomaticAnalysis` requires an issued `FinishedRun` for the result's run; `Run.finish` writes `latest.json` last through `writePreparedRunLatestPointer` (`src/run/contained-output.ts`) | `tests/run/run-lifecycle.test.ts`, which also checks that `src/run/run.ts` is the pointer's one writer, `tests/analysis/automatic-analysis.test.ts` |
|
|
137
140
|
|
|
138
141
|
The receipt write is best-effort; the sandbox's server-side timeout ends a sandbox with no
|
|
139
142
|
receipt. A study check probe journals its receipt with `withPreflightSandbox`
|
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,32 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.110.2: Credential URLs refused, terminal key proxied, watch exits for agents (2026-10-05)
|
|
12
|
+
|
|
13
|
+
humanish 0.110.2 refuses a study URL that carries a user name, password or credential parameter,
|
|
14
|
+
and `humanish verify` catches all 41 secret formats its test lists, where 0.110.1 caught 13. A
|
|
15
|
+
terminal study that declares no `execution.runtimeAuth` keeps the OpenAI key in E2B's host-side
|
|
16
|
+
proxy rule (`openai-egress`) and gives Codex a placeholder. No Observer server hands out
|
|
17
|
+
`sandbox-receipts.ndjson` or `status.json`, and a Ctrl-C during desktop startup no longer writes
|
|
18
|
+
the raw sandbox id outside `sandbox-receipts.ndjson`. For a coding agent, `watch` and `observe`
|
|
19
|
+
print the Observer path and exit when nobody is at a terminal, a rerun of `init` rewrites only its
|
|
20
|
+
own `AGENTS.md` section, and suggested commands use one invocation style.
|
|
21
|
+
|
|
22
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.110.2)
|
|
23
|
+
|
|
24
|
+
## 0.110.1: Claude participants restricted, sandboxes reclaimed by tag (2026-10-04)
|
|
25
|
+
|
|
26
|
+
humanish 0.110.1 starts a Claude Code participant with Read as its only tool, in a scratch folder,
|
|
27
|
+
with a short environment allowlist and no saved transcript, and needs Claude Code 2.1.248 or newer.
|
|
28
|
+
A Ctrl-C in the first seconds of a live run no longer leaves a sandbox running: every sandbox
|
|
29
|
+
carries the run's tags, and the signal handler and `humanish reclaim` kill what E2B lists under
|
|
30
|
+
them. `reclaim` reports `clean`, `unconfirmed` or `unknown`, and `humanish cleanup` is now a
|
|
31
|
+
deprecated alias of `reclaim --check`. run.json records how a run ended, every surface shows a
|
|
32
|
+
failed run as failed, and `review`, `analyze show` and a live run's last lines print the analysis
|
|
33
|
+
findings as text.
|
|
34
|
+
|
|
35
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.110.1)
|
|
36
|
+
|
|
11
37
|
## 0.110.0: --dotenv, and sandbox ids only in receipts (2026-10-03)
|
|
12
38
|
|
|
13
39
|
humanish 0.110.0 renames `--env-file` to `--dotenv`, so a missing file reaches humanish's exit 2
|
package/CONTEXT.md
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
# Glossary
|
|
2
2
|
|
|
3
|
-
humanish is "Synthetic user research for apps, CLIs, and agent-facing product flows."
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
them equal.
|
|
3
|
+
humanish is "Synthetic user research for apps, CLIs, and agent-facing product flows." package.json,
|
|
4
|
+
`humanish --help` and bare `humanish` use that sentence (`PRODUCT_SENTENCE` in
|
|
5
|
+
`src/cli/product-sentence.ts`), line 3 of README.md and the humanish skill's description open with
|
|
6
|
+
it, and tests/cli/product-sentence.test.ts holds them equal. [How a study works](https://humanish.dev/docs/concepts)
|
|
7
|
+
is the user-facing version of the first table below.
|
|
7
8
|
|
|
8
9
|
Use one word per concept. Docs and new code use the terms in the first table. Some manifest,
|
|
9
10
|
bundle and output fields keep older spellings; the second table maps them.
|
|
@@ -18,10 +19,10 @@ bundle and output fields keep older spellings; the second table maps them.
|
|
|
18
19
|
| mission | The free-form brief a model-driven participant receives: `actor.mission`. `actor.tasks[]` adds discrete tasks: each has a `goal` the participant sees and an optional `success` check it does not. | `StudyActor` in `src/study/types.ts` |
|
|
19
20
|
| assignment | What one participant was asked to do: the mission, a `participants` entry's `instruction` or the homogeneous `participants.instruction` (recorded as `focus`), and the tasks. Each stream records it as `streams[].assignment`. | `participantAssignment` in `src/study/participant-assignment.ts`; `RunParticipantAssignment` in `src/run/streams.ts` |
|
|
20
21
|
| scenario | A scenario file, `humanish/scenarios/<id>.yaml` (`humanish.scenario.v1`): a goal and ordered steps with expectations. A scripted study names one in `scenario:`, and only the scripted route executes those steps. A v2 study file's `scenario` block held run settings (`mode`, `caps`, `ref`), which v3 moves to the top level. A model-driven participant follows its mission. | `StudyScenario` in `src/study/types.ts`; `src/routes/scripted/route.ts` resolves the scenario, and `executeBrowserPersonaStep` (`src/actors/scripted-browser/steps.ts`) runs each step |
|
|
21
|
-
| surface | One browser viewport that a scripted run replays its journey on: `desktop` or `mobile`. A scripted study runs the desktop surface by default, and `
|
|
22
|
+
| surface | One browser viewport that a scripted run replays its journey on: `desktop` or `mobile`. A scripted study runs the desktop surface by default, and `surfaces: [desktop, mobile]` adds the mobile one. Each surface writes `actor-<surface>.json` and `traces/<surface>.json`. | `BrowserSurface` and `browserSurfaces` in `src/actors/scripted-browser/types.ts`; the plan's `surfaces` in `src/routes/scripted/plan.ts` |
|
|
22
23
|
| journey | A scenario's `browser.steps` parsed into ordered steps with a goal and a start path. The scripted route replays it once on each surface, and each surface's session reports the first step that did not pass. | `BrowserPersonaJourney` in `src/actors/scripted-browser/types.ts`; `parseBrowserPersonaJourneyFromScenario` in `src/actors/scripted-browser/journey.ts` |
|
|
23
24
|
| subject | The product under study, declared by `subject.source`: `clone`, `app-url`, `local-tree`, `terminal-product`, `desktop-cli`, `local-app` or `this-repo`. `run.json` records its provenance under `subject`. | `StudySubject` in `src/study/types.ts`, parsed in `src/study/parse/subject.ts` |
|
|
24
|
-
| actor | The model or program that drives a participant, declared by `
|
|
25
|
+
| actor | The model or program that drives a participant, declared by `actor.type`, e.g. `openai-computer-use`, `local-agent`, `scripted-browser`, `codex-exec`. | `src/actors/`; `actorRegistry` in `src/actors/registry.ts` lists them, plus `codex-app-server`, which no study can name; `ActorTrace` in `src/actors/contract.ts` is its record |
|
|
25
26
|
| run kind | What an actor can do: `code`, `computer-use`, `scripted-browser` or `terminal`. An actor declares its run kinds in `capabilities.lanes`, and its trace records the one it ran as `lane`; neither field names a participant. Routing admits a study's actor only for a run kind it declares. `computer-use` and `terminal` name both a run kind and a route, but a route is where a study runs and a run kind is what an actor does. | `ActorRunKind` in `src/actors/contract.ts`; `actorResolvesToComputerUse` (`src/study/routing.ts`) and its siblings |
|
|
26
27
|
| adapter | Product-specific code kept out of core. A study names a scorer module in `review.scorer.ref`, or the CLI passes `--scorer`, and a library caller passes the same functions as `RunStudyOptions.scorer`: `score`, `deriveFeedback` and `deriveArtifacts`. Their output is stored under the adapter's namespace in `run.json` (`adapterScore`, `adapterArtifacts` and a feedback candidate's `adapter`). | `AdapterScorerModule` in `src/study/adapter-scorer-loader.ts`; `applyBrowserScorer` in `src/study/adapter-extension.ts`; fixture sets in `adapters/` |
|
|
27
28
|
| computer use | A model drives a participant by reading screenshots of a desktop and acting with mouse and keyboard. `openai-computer-use` and `local-agent` are the computer-use actors. CUA is the short form in code names such as `CuaExecutor` and in the older spellings listed under `cua` below. | `src/routes/computer-use/`, `src/actors/computer-use/` |
|
|
@@ -31,6 +32,7 @@ bundle and output fields keep older spellings; the second table maps them.
|
|
|
31
32
|
| substrate | Where a participant's desktop or shell runs: a hosted E2B sandbox, or a local Firecracker VM (inside Lima on macOS). | `src/substrates/e2b/`, `src/substrates/local/` |
|
|
32
33
|
| participant desktop | The desktop one computer-use or shared-world participant drives: a substrate plus the route's concerns, such as subject provisioning, comms and the participant's plan. It lives in the route, so `src/substrates/` holds only provider primitives ([ADR 0004](docs/decisions/0004-participant-desktops-stay-in-the-route.md)). | `ParticipantDesktop` in `src/routes/computer-use/participant-desktop.ts`; `createE2BParticipantDesktop` in `src/routes/computer-use/e2b-desktop/desktop.ts`; the local VM's in `src/routes/computer-use/local-vm.ts` |
|
|
33
34
|
| receipt | A line in a run's `sandbox-receipts.ndjson` that records a sandbox's id, provider, the participant or subject it belongs to (`laneId`), and its time limit as soon as the sandbox is created, before any work runs in it. `humanish reclaim` kills sandboxes by these ids after an interrupted run. | `SandboxReceipt` in `src/run/sandbox-receipts.ts`; `reclaimRunSandboxes` in `src/run/reclaim.ts` |
|
|
35
|
+
| owner tags | The E2B metadata every sandbox humanish creates carries: `tool: humanish`, the run's `runId`, and `runKey`, a digest of the run directory's inode and creation time. The run journals them in an owner line before its first create. `humanish reclaim` lists sandboxes that match all three, which finds one whose id never reached a receipt. | `sandboxOwnerTags` in `src/run/sandbox-creates.ts`; `findE2BSandboxesByTags` in `src/substrates/e2b/sandbox.ts` |
|
|
34
36
|
| topology | Whether participants share one instance of the subject. `route: shared-world` gives one for all; every other route gives each participant its own (`per-lane-worlds`). `run.json` records it as `sharedWorld.topology`. | `StudySubjectTopology` in `src/study/types.ts` |
|
|
35
37
|
| plane | The one instance of the subject that a shared-world study's participants use together. A `provisioned` plane is a `clone` or `local-tree` subject that humanish serves in a sandbox, with a declared subject state. An `external-public` plane is a deployment already running at `subject.appUrl`, whose owner the study names in `subject.publicTarget.owner`. Each plane has at least two participants. | `SharedWorldPlane` in `src/study/plan-types.ts`; `src/routes/shared-world/` |
|
|
36
38
|
| stream | One live or recorded view of a participant, such as a browser or a terminal, in `run.json` and the Observer. | `RunStream` in `src/run/streams.ts` |
|
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# humanish
|
|
2
2
|
|
|
3
|
-
Synthetic user research for apps, CLIs, and agent-facing product flows.
|
|
3
|
+
Synthetic user research for apps, CLIs, and agent-facing product flows. AI participants with a persona and a task use your app on real desktops, and each run leaves evidence you can verify: what they did, where they got stuck, and what it cost.
|
|
4
4
|
|
|
5
5
|
[](https://humanish.dev/demo)
|
|
6
6
|
|
|
@@ -8,6 +8,24 @@ Synthetic user research for apps, CLIs, and agent-facing product flows.
|
|
|
8
8
|
lobby of a multiplayer game on its live deployment, each on its own hosted desktop. The page
|
|
9
9
|
replays the real Observer; nothing runs from it.
|
|
10
10
|
|
|
11
|
+
## Learn how a study works
|
|
12
|
+
|
|
13
|
+
A study is a YAML file in your project. It names the app under study, the task, the
|
|
14
|
+
participants and a spend limit. Each participant has a persona: a file that sets who they are
|
|
15
|
+
and traits such as patience and keyboard use.
|
|
16
|
+
|
|
17
|
+
`humanish run <study>` gives each participant a desktop, either a hosted E2B desktop or a
|
|
18
|
+
disposable browser VM on your machine. On the computer-use route a model drives each
|
|
19
|
+
participant: every turn it reads a screenshot and answers with mouse and keyboard actions, until
|
|
20
|
+
the participant finishes, gets stuck or reaches a limit.
|
|
21
|
+
|
|
22
|
+
The run writes its evidence to gitignored `.humanish/runs/<run id>/`: screenshots, every
|
|
23
|
+
action, each participant's report and the estimated cost. The Observer replays it,
|
|
24
|
+
`humanish review` prints the findings of the analysis that follows a live run, and
|
|
25
|
+
`humanish verify` grades whether the evidence is safe to share.
|
|
26
|
+
[How a study works](https://humanish.dev/docs/concepts) walks through one annotated study file,
|
|
27
|
+
and the [study file reference](https://humanish.dev/docs/study-files) lists every field.
|
|
28
|
+
|
|
11
29
|
## Try it without keys
|
|
12
30
|
|
|
13
31
|
In a project directory, with Node.js 22.19 or newer:
|
|
@@ -16,21 +34,44 @@ In a project directory, with Node.js 22.19 or newer:
|
|
|
16
34
|
npm install --save-dev humanish
|
|
17
35
|
npx humanish init --yes
|
|
18
36
|
npx humanish run first-run
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
`init` writes starter studies and personas under `humanish/`, adds `.humanish/` to
|
|
40
|
+
`.gitignore`, and adds `humanish:*` scripts to `package.json` and a humanish section to
|
|
41
|
+
`AGENTS.md`, and lists each file it created or changed. It never overwrites an existing study
|
|
42
|
+
file; a rerun replaces only its own `AGENTS.md` section. `run first-run` is a dry run: four
|
|
43
|
+
synthetic participants, with no browser, model, key or spend. It prints:
|
|
44
|
+
|
|
45
|
+
```text
|
|
46
|
+
humanish run dry-run
|
|
47
|
+
run: dryrun-2026-10-04T07-11-55-932Z-d11afab6
|
|
48
|
+
participants: 4
|
|
49
|
+
bundle: .humanish/runs/dryrun-2026-10-04T07-11-55-932Z-d11afab6/run.json
|
|
50
|
+
review: .humanish/runs/dryrun-2026-10-04T07-11-55-932Z-d11afab6/review.md
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Replay the run in the Observer, then check its evidence:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
19
56
|
npx humanish observe --run latest --open
|
|
57
|
+
npx humanish verify
|
|
20
58
|
```
|
|
21
59
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
60
|
+
```text
|
|
61
|
+
verified dryrun-2026-10-04T07-11-55-932Z-d11afab6 · share_ready · 16 checks passed
|
|
62
|
+
```
|
|
25
63
|
|
|
26
|
-
|
|
64
|
+
A dry run shows the evidence format and tests no product behavior. To run participants against
|
|
65
|
+
your own app, follow the [own-app guide](https://humanish.dev/docs/your-app).
|
|
66
|
+
|
|
67
|
+
[Quickstart](https://humanish.dev/docs) · [How a study works](https://humanish.dev/docs/concepts) · [Study your app](https://humanish.dev/docs/your-app) · [What a study costs](https://humanish.dev/docs/what-a-study-costs) · [Trust boundaries](https://humanish.dev/docs/trust-boundaries) · [CLI reference](https://humanish.dev/docs/cli) · [Limits and evidence](https://humanish.dev/failure-modes)
|
|
27
68
|
|
|
28
69
|
## Run a live study
|
|
29
70
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
preview and local
|
|
71
|
+
A live study gives each participant a computer to use. By default that computer is a hosted
|
|
72
|
+
desktop from E2B, which humanish drives through the `@e2b/desktop` package. Install it in the
|
|
73
|
+
same project: `npm install --save-dev @e2b/desktop`. A one-shot `npx humanish@latest` cannot see
|
|
74
|
+
a copy installed elsewhere. The keyless preview and a study in a local browser do not need it.
|
|
34
75
|
|
|
35
76
|
Choose how the participant runs:
|
|
36
77
|
|
|
@@ -81,7 +122,7 @@ browser, or use Codex account quota. The local study needs a supported Codex CLI
|
|
|
81
122
|
version and ChatGPT login. Linux x64 also needs local rootful Docker, KVM and
|
|
82
123
|
TUN; M3-or-newer Apple Silicon Macs need native ARM64 Node and Lima 2.2+.
|
|
83
124
|
|
|
84
|
-
### Study drawDB
|
|
125
|
+
### Study drawDB with an OpenAI key
|
|
85
126
|
|
|
86
127
|
Set the desktop and model keys with hidden prompts, then send one synthetic participant into the
|
|
87
128
|
included drawDB study:
|
|
@@ -97,7 +138,7 @@ npx humanish observe --run latest --open
|
|
|
97
138
|
```
|
|
98
139
|
|
|
99
140
|
Existing `E2B_API_KEY` and `OPENAI_API_KEY` environment variables also work.
|
|
100
|
-
`doctor --study` checks the
|
|
141
|
+
`doctor --study` checks the study's local setup without launching a
|
|
101
142
|
desktop or making a model request. It reports key presence, not remote key
|
|
102
143
|
validity, model access, or quota. `study check` checks manifest metadata by default.
|
|
103
144
|
`try-live` clones and studies drawDB, not your project. Set the keys before `init`:
|
|
@@ -110,20 +151,15 @@ dollar cap. Automatic analysis after the run is a separate request;
|
|
|
110
151
|
[Read the results](#read-the-results) gives its limit and how to turn it off.
|
|
111
152
|
Hosted desktop time is additional. Participant caps are checked
|
|
112
153
|
between turns and are not provider billing ceilings. Allow a few minutes for
|
|
113
|
-
the app to build and the participant to work. See [
|
|
154
|
+
the app to build and the participant to work. See [what a study costs](https://humanish.dev/docs/what-a-study-costs).
|
|
114
155
|
|
|
115
156
|
## See what studies have found
|
|
116
157
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
goes into feedback drafts, export bundles and an `observe --all --safe` library, and the
|
|
123
|
-
end of the pipeline is a public-safe feedback draft you can turn into a real
|
|
124
|
-
issue. Committed study source lives under `humanish/`; run evidence lands under
|
|
125
|
-
gitignored `.humanish/`. [ARCHITECTURE.md](ARCHITECTURE.md) traces `humanish run <study>` through
|
|
126
|
-
the code.
|
|
158
|
+
A study's evidence includes per-task completion funnels and participant outcomes with the
|
|
159
|
+
denominator attached. The share-safety gate decides what goes into feedback drafts, export
|
|
160
|
+
bundles and an `observe --all --safe` library, and the end of the pipeline is a public-safe
|
|
161
|
+
feedback draft you can turn into a real issue. [ARCHITECTURE.md](ARCHITECTURE.md) traces
|
|
162
|
+
`humanish run <study>` through the code.
|
|
127
163
|
|
|
128
164
|
The researcher declares the study, the participant tries the product, and the
|
|
129
165
|
stakeholder reads what happened. [Three roles](docs/principles/three-roles.md)
|
|
@@ -141,7 +177,7 @@ maintainer did not write: TodoMVC 5 of 6 findings confirmed against the source,
|
|
|
141
177
|
drawDB 11 of 12, none invented
|
|
142
178
|
([TodoMVC](https://github.com/danielgwilson/humanish/blob/main/bench/RESULTS-TODOMVC-2026-09-01.md), [drawDB](https://github.com/danielgwilson/humanish/blob/main/bench/RESULTS-DRAWDB-2026-09-01.md)).
|
|
143
179
|
Cold install to a live study: 9 of 9 fresh directories reached the goal in 108 to 200 seconds;
|
|
144
|
-
the five
|
|
180
|
+
the five with an OpenAI key cost $0.16 to $0.35 each, and the four on Codex or Claude Code are
|
|
145
181
|
unpriced ([study record](https://github.com/danielgwilson/humanish/blob/main/docs/evidence/computer-use/cold-install-try-live-2026-09-01.md)).
|
|
146
182
|
Same mission, different personas: keyboard-first participants reported drawDB's
|
|
147
183
|
database modal 5 of 5 times and TodoMVC's mouse-only rename 6 of 6; mouse newcomers
|
|
@@ -160,6 +196,8 @@ npx humanish verify --run latest --json
|
|
|
160
196
|
npx humanish feedback issue --run latest --repo owner/repo --format markdown
|
|
161
197
|
```
|
|
162
198
|
|
|
199
|
+
`review` lists each analysis finding with its impact, confidence, recovery,
|
|
200
|
+
affected participants and cited captures; `--json` has them under `analysis`.
|
|
163
201
|
`feedback issue` prints a draft and requires `share_ready` evidence. A live run
|
|
164
202
|
with raw screenshots can be valid local evidence and still fail that sharing
|
|
165
203
|
gate. `humanish export --run latest --format bundle --redact-screenshots` creates
|
|
@@ -256,12 +294,22 @@ certified free of PII or PHI. A first-class PII/PHI detector is
|
|
|
256
294
|
Feedback commands require `share_ready`. A valid local run can still be
|
|
257
295
|
reviewed in Observer without being promoted into a public issue draft.
|
|
258
296
|
|
|
259
|
-
##
|
|
297
|
+
## Check the compatibility policy before upgrading
|
|
260
298
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
299
|
+
humanish is 0.x, so a minor release can break things. These rules limit how. They cover CLI
|
|
300
|
+
commands and flags, study file fields, the run bundle, `--json` output, exit and error codes,
|
|
301
|
+
and the library exports.
|
|
302
|
+
|
|
303
|
+
- A deprecated name keeps working for at least 30 days and through at least one minor release
|
|
304
|
+
before a release removes it.
|
|
305
|
+
- Breaking changes ship together in one minor release. Its release notes carry a migration
|
|
306
|
+
table: each old form, its new form, and the change to make.
|
|
307
|
+
- The study file and the run bundle carry versioned schemas, `humanish.study.v3` and
|
|
308
|
+
`humanish.run-bundle.v1`. A breaking change to either one bumps its version.
|
|
309
|
+
|
|
310
|
+
Pin the exact version in `package.json` and read [CHANGELOG.md](CHANGELOG.md) before upgrading.
|
|
311
|
+
|
|
312
|
+
## Know these limits before you depend on humanish
|
|
265
313
|
|
|
266
314
|
**CI runs nothing live.** CI runs the offline test suite, the Observer browser
|
|
267
315
|
proofs and, when guest files change, a guest-desktop container proof. It passes
|
|
@@ -292,7 +340,7 @@ fills from sources 2 to 4, never the value:
|
|
|
292
340
|
4. `$XDG_CONFIG_HOME/humanish/keys.env` (by default `~/.config/humanish/keys.env`),
|
|
293
341
|
which `humanish keys set` writes as plain text with mode `0600`.
|
|
294
342
|
|
|
295
|
-
A dry run reads no provider key, so `run
|
|
343
|
+
A dry run reads no provider key, so `run` and `watch` consult sources 2 to 4 only
|
|
296
344
|
for a live study. A live run fills every key it finds and prints the ones its plan reads: its model,
|
|
297
345
|
desktop and runtime keys, its `subject.env` names, the variable
|
|
298
346
|
`comms.email.external.authTokenEnv` names, `ANTHROPIC_API_KEY` for a Claude Code participant, and
|
|
@@ -310,31 +358,31 @@ never reads the subject app's own `.env` files.
|
|
|
310
358
|
**What leaves your machine.** [Trust boundaries](https://humanish.dev/docs/trust-boundaries)
|
|
311
359
|
has the providers' retention terms.
|
|
312
360
|
|
|
313
|
-
| Party | What it receives
|
|
314
|
-
| -------------------- |
|
|
315
|
-
| OpenAI | Each step's screenshot, the persona, the mission and the actions so far; after a run, selected text and captures for analysis; for an external-public shared-world study, the host participant's screen while its lobby code is read; for a terminal study, the agent's requests from the E2B sandbox
|
|
316
|
-
| Codex or Claude Code | The same screenshots and prompts, through your signed-in agent and its provider
|
|
317
|
-
| E2B | A desktop or shell running your app and everything on its screen; the repository a `clone` subject names; your working tree for a `local-tree` subject, minus gitignored files, `.env*` and other secret-shaped files; the credentials you declare for the subject app; for a terminal study
|
|
318
|
-
| PostHog | Telemetry events, described [below](#check-what-telemetry-sends)
|
|
361
|
+
| Party | What it receives | When |
|
|
362
|
+
| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------- |
|
|
363
|
+
| OpenAI | Each step's screenshot, the persona, the mission and the actions so far; after a run, selected text and captures for analysis; for an external-public shared-world study, the host participant's screen while its lobby code is read; for a terminal study, the agent's requests from the E2B sandbox | `openai-computer-use` participants; automatic or `humanish analyze` runs; external-public shared-world studies; terminal studies |
|
|
364
|
+
| Codex or Claude Code | The same screenshots and prompts, through your signed-in agent and its provider | `local-agent` participants, local browser studies, the Codex analyst |
|
|
365
|
+
| E2B | A desktop or shell running your app and everything on its screen; the repository a `clone` subject names; your working tree for a `local-tree` subject, minus gitignored files, `.env*` and other secret-shaped files; the credentials you declare for the subject app; for a terminal study with `execution.runtimeAuth: openai-env`, your OpenAI key | Every hosted study |
|
|
366
|
+
| PostHog | Telemetry events, described [below](#check-what-telemetry-sends) | By default |
|
|
319
367
|
|
|
320
368
|
## Find the main commands
|
|
321
369
|
|
|
322
370
|
Use `npx humanish` from your project. Full arguments and options are generated
|
|
323
371
|
from the shipped CLI in the [command reference](https://humanish.dev/docs/cli).
|
|
324
372
|
|
|
325
|
-
| Command | Purpose
|
|
326
|
-
| -------------------------------------------------------- |
|
|
327
|
-
| `humanish init --yes` | Scaffold study source and ignored runtime state.
|
|
328
|
-
| `humanish doctor --study <study> --json` | Check
|
|
329
|
-
| `humanish study list --json` | List available studies.
|
|
330
|
-
| `humanish study show <study> --json` | Read a study before running it.
|
|
331
|
-
| `humanish study check <study> --json` | Check configuration and route warnings.
|
|
332
|
-
| `humanish run <study>` | Run the named preview or live study.
|
|
333
|
-
| `humanish watch <study>` | Run a study with an attached Observer.
|
|
334
|
-
| `humanish runs --json` | List local run history.
|
|
335
|
-
| `humanish review --run latest --json` | Read
|
|
336
|
-
| `humanish verify --run latest --json` | Check evidence and share-safety gates.
|
|
337
|
-
| `humanish feedback issue --run latest --repo owner/repo` | Print an eligible feedback draft.
|
|
373
|
+
| Command | Purpose |
|
|
374
|
+
| -------------------------------------------------------- | -------------------------------------------------------------- |
|
|
375
|
+
| `humanish init --yes` | Scaffold study source and ignored runtime state. |
|
|
376
|
+
| `humanish doctor --study <study> --json` | Check a study's setup without exposing key values or spending. |
|
|
377
|
+
| `humanish study list --json` | List available studies. |
|
|
378
|
+
| `humanish study show <study> --json` | Read a study before running it. |
|
|
379
|
+
| `humanish study check <study> --json` | Check configuration and route warnings. |
|
|
380
|
+
| `humanish run <study>` | Run the named preview or live study. |
|
|
381
|
+
| `humanish watch <study>` | Run a study with an attached Observer. |
|
|
382
|
+
| `humanish runs --json` | List local run history. |
|
|
383
|
+
| `humanish review --run latest --json` | Read a run's outcome and its analysis findings. |
|
|
384
|
+
| `humanish verify --run latest --json` | Check evidence and share-safety gates. |
|
|
385
|
+
| `humanish feedback issue --run latest --repo owner/repo` | Print an eligible feedback draft. |
|
|
338
386
|
|
|
339
387
|
## Check exit codes
|
|
340
388
|
|
|
@@ -368,12 +416,14 @@ Source: [`skills/humanish/SKILL.md`](skills/humanish/SKILL.md).
|
|
|
368
416
|
|
|
369
417
|
## Find more guides
|
|
370
418
|
|
|
371
|
-
- [
|
|
372
|
-
|
|
419
|
+
- [How a study works](https://humanish.dev/docs/concepts): study, run, participant, persona,
|
|
420
|
+
actor, subject, route and the verify grades, on one annotated study file.
|
|
421
|
+
- [Study files](https://humanish.dev/docs/study-files): every field with its type and default,
|
|
422
|
+
persona files and traits, ignored private studies and
|
|
373
423
|
[scripted browser scenarios](https://humanish.dev/docs/study-files#scripted-browser-scenarios).
|
|
374
424
|
- [Computer use](https://humanish.dev/docs/computer-use): subjects, screenshots, devices, mobile
|
|
375
425
|
emulation, stop rules, dwell windows and reruns of failed participants. The
|
|
376
|
-
[cost model](https://humanish.dev/docs/
|
|
426
|
+
[cost model](https://humanish.dev/docs/what-a-study-costs#how-cost-estimates-work) explains
|
|
377
427
|
model selection, dated estimates, and study and per-participant caps.
|
|
378
428
|
- [A signed-in coding agent](https://humanish.dev/docs/local-agents): Codex or Claude Code supplies
|
|
379
429
|
the participant's model on your existing plan. E2B still needs a key and bills for desktops.
|
|
@@ -417,8 +467,8 @@ take which option.
|
|
|
417
467
|
humanish sends anonymous usage events to PostHog by default. Each event carries
|
|
418
468
|
the command, the humanish version, your OS and Node major version, whether it
|
|
419
469
|
ran in CI, the exit code, a duration bucket and a random machine id made on
|
|
420
|
-
first use. A study adds whether it was a dry run, an outcome word, the
|
|
421
|
-
participant's brain
|
|
470
|
+
first use. A study adds whether it was a dry run, an outcome word, where the
|
|
471
|
+
participant's model comes from (`brain`), stop and diagnostic categories, and humanish's error
|
|
422
472
|
code when it fails. The study id is sent only when it is a starter study that
|
|
423
473
|
`humanish init` writes; any other study is sent as `custom`. Your study ids,
|
|
424
474
|
subjects, personas, prompts, paths, run ids and evidence are never sent.
|
|
@@ -87,8 +87,8 @@ export function spendLimit(session, estimate, maxUsd) {
|
|
|
87
87
|
return {
|
|
88
88
|
completionReason: "budget_reached",
|
|
89
89
|
reason: materialActions > 0
|
|
90
|
-
? `Stopped before the next model turn: ${spend} passed
|
|
91
|
-
: `Stopped before the next model turn: ${spend} passed
|
|
90
|
+
? `Stopped before the next model turn: ${spend} passed caps.maxUsd=$${maxUsd} after productive activity (${plural(materialActions, "material action")}, ${plural(turns, "turn")})`
|
|
91
|
+
: `Stopped before the next model turn: ${spend} passed caps.maxUsd=$${maxUsd} with no material progress`,
|
|
92
92
|
stopCause: "spend_limit",
|
|
93
93
|
};
|
|
94
94
|
}
|
|
@@ -96,7 +96,7 @@ export function spendLimit(session, estimate, maxUsd) {
|
|
|
96
96
|
// trip.
|
|
97
97
|
export const nonFiniteEstimate = {
|
|
98
98
|
completionReason: "harness_error",
|
|
99
|
-
reason: "the injected estimateTurnCostUsd returned a non-finite estimate while
|
|
99
|
+
reason: "the injected estimateTurnCostUsd returned a non-finite estimate while caps.maxUsd is set; the estimator receives one ActorTokenUsage object. Failing closed instead of running uncapped.",
|
|
100
100
|
};
|
|
101
101
|
/** A study-level stop is a recruiting decision hitting its limit, not this participant's runaway. */
|
|
102
102
|
export function runSpendLimit(reason) {
|
|
@@ -131,7 +131,7 @@ export const usageUnreported = {
|
|
|
131
131
|
* limit is set, so its worst case is the model's whole output allowance.
|
|
132
132
|
*/
|
|
133
133
|
export function lostRequestUnbounded(turnNumber, lost) {
|
|
134
|
-
const reason = `provider turn ${turnNumber} ${lost}; its usage is unknown and no maxOutputTokens bounds what it cost, so the declared model-spend cap cannot be established; no further or closing request was dispatched. Set
|
|
134
|
+
const reason = `provider turn ${turnNumber} ${lost}; its usage is unknown and no maxOutputTokens bounds what it cost, so the declared model-spend cap cannot be established; no further or closing request was dispatched. Set actor.maxOutputTokens so a capped session can book the worst case and retry.`;
|
|
135
135
|
return {
|
|
136
136
|
completionReason: "harness_error",
|
|
137
137
|
reason,
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
type JsonObject = Record<string, unknown>;
|
|
2
|
+
/**
|
|
3
|
+
* The first release with `--restricted`. The changelog adds it in 2.1.248; `claude --help` of
|
|
4
|
+
* 2.1.247 does not list it and 2.1.248 lists it with `--tools`, `--strict-mcp-config`,
|
|
5
|
+
* `--no-session-persistence` and `--permission-mode dontAsk`.
|
|
6
|
+
*/
|
|
7
|
+
export declare const CLAUDE_CODE_MIN_VERSION = "2.1.248";
|
|
8
|
+
/**
|
|
9
|
+
* The flags that restrict a participant, shared by the session and one-shot paths. Each one was
|
|
10
|
+
* checked against `claude --help` of 2.1.248 and 2.1.289.
|
|
11
|
+
*/
|
|
12
|
+
export declare function claudeParticipantFlags(model?: string): string[];
|
|
13
|
+
/**
|
|
14
|
+
* Names the participant's environment keeps from the host. Measured on Linux: `HOME` alone signs
|
|
15
|
+
* in a `claude.ai` login, and `PATH` finds `claude`. `USER` and `LOGNAME` name the macOS keychain
|
|
16
|
+
* account, `CLAUDE_CONFIG_DIR` moves Claude Code's own login and config, and the proxy and CA
|
|
17
|
+
* names are for hosts that reach the API through a proxy. Everything else is dropped: provider
|
|
18
|
+
* keys, `GH_TOKEN`, every `--dotenv` name, `ANTHROPIC_API_KEY` and `CLAUDE_CODE_OAUTH_TOKEN`, and
|
|
19
|
+
* the `CLAUDE_CODE_*` variables of a Claude Code session humanish itself runs inside.
|
|
20
|
+
*/
|
|
21
|
+
export declare const CLAUDE_PARTICIPANT_ENV_NAMES: readonly string[];
|
|
22
|
+
/**
|
|
23
|
+
* The participant's whole environment: the allowlisted names `source` holds, minus any a `--dotenv`
|
|
24
|
+
* file set. The advisor is a server-side tool Claude Code enables by default on some plans; it is
|
|
25
|
+
* turned off so the only tool the stream can show is Read.
|
|
26
|
+
*/
|
|
27
|
+
export declare function claudeParticipantEnv(source: Readonly<Record<string, string | undefined>>): NodeJS.ProcessEnv;
|
|
28
|
+
/** `2.1.289` from `2.1.289 (Claude Code)`; undefined for anything else. */
|
|
29
|
+
export declare function parseClaudeCodeVersion(text: string): string | undefined;
|
|
30
|
+
/** Whether `version` is at or above CLAUDE_CODE_MIN_VERSION. */
|
|
31
|
+
export declare function admitsClaudeCodeVersion(version: string | undefined): boolean;
|
|
32
|
+
/** The refusal for a Claude Code release below the floor, or one whose version could not be read. */
|
|
33
|
+
export declare function claudeCodeVersionMessage(version: string | undefined): string;
|
|
34
|
+
export type ClaudeParticipantErrorCode =
|
|
35
|
+
/** Claude Code started with more than Read, with an MCP server, or in another permission mode. */
|
|
36
|
+
"HUMANISH_CLAUDE_PARTICIPANT_UNRESTRICTED"
|
|
37
|
+
/** The participant called a tool other than Read inside its folder, or Claude Code denied one. */
|
|
38
|
+
| "HUMANISH_CLAUDE_PARTICIPANT_TOOL_REFUSED";
|
|
39
|
+
/** The participant was stopped because its stream showed something it may not do. */
|
|
40
|
+
export declare class ClaudeParticipantError extends Error {
|
|
41
|
+
readonly code: ClaudeParticipantErrorCode;
|
|
42
|
+
constructor(code: ClaudeParticipantErrorCode, reason: string);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Checks every stream-json message from one participant process. `inspect` returns the error the
|
|
46
|
+
* message proves, or undefined. A `result` counts only after an init message was admitted.
|
|
47
|
+
*/
|
|
48
|
+
export declare class ClaudeStreamGuard {
|
|
49
|
+
private readonly folders;
|
|
50
|
+
private admitted;
|
|
51
|
+
/** `folders`: the session folder, as given to Claude Code and as resolved on disk. */
|
|
52
|
+
constructor(folders: readonly string[]);
|
|
53
|
+
inspect(message: JsonObject): ClaudeParticipantError | undefined;
|
|
54
|
+
private system;
|
|
55
|
+
private assistant;
|
|
56
|
+
private inside;
|
|
57
|
+
}
|
|
58
|
+
/** Transcript folders Claude Code saved for earlier participants, before `--no-session-persistence`. */
|
|
59
|
+
export type LeftoverClaudeTranscripts = {
|
|
60
|
+
/** Claude Code's projects folder. */
|
|
61
|
+
directory: string;
|
|
62
|
+
/** The matched folder names. */
|
|
63
|
+
names: string[];
|
|
64
|
+
/** Removes exactly `names`; absent when there are none or the folder path is not ASCII. */
|
|
65
|
+
removeCommand?: string;
|
|
66
|
+
}
|
|
67
|
+
/** The folder exists but could not be listed; `code` is the error code, such as `EACCES`. */
|
|
68
|
+
| {
|
|
69
|
+
directory: string;
|
|
70
|
+
unreadable: string;
|
|
71
|
+
};
|
|
72
|
+
/** Lists leftover participant transcripts. Reads folder names only, never their contents. */
|
|
73
|
+
export declare function leftoverClaudeTranscripts(env: Readonly<Record<string, string | undefined>>, home: string, list: (directory: string) => Promise<string[]>): Promise<LeftoverClaudeTranscripts>;
|
|
74
|
+
export {};
|