explorbot 0.2.4 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/bin/explorbot-cli.ts +19 -7
  2. package/boat/api-tester/src/cli.ts +17 -0
  3. package/boat/doc-collector/src/cli.ts +14 -1
  4. package/boat/prima/README.md +96 -0
  5. package/boat/prima/package.json +14 -10
  6. package/boat/prima/src/cli.ts +29 -12
  7. package/boat/prima/src/envelope.ts +35 -13
  8. package/boat/prima/src/prima.ts +78 -45
  9. package/dist/bin/explorbot-cli.js +19 -7
  10. package/dist/boat/api-tester/src/cli.js +17 -0
  11. package/dist/boat/doc-collector/src/cli.js +14 -1
  12. package/dist/boat/prima/src/cli.js +26 -7
  13. package/dist/boat/prima/src/envelope.js +32 -8
  14. package/dist/boat/prima/src/prima.js +75 -43
  15. package/dist/models.json +4 -4
  16. package/dist/package.json +6 -2
  17. package/dist/src/action-result.d.ts +13 -0
  18. package/dist/src/action-result.js +46 -15
  19. package/dist/src/action.d.ts +5 -2
  20. package/dist/src/action.js +53 -18
  21. package/dist/src/ai/captain/web-mode.js +1 -2
  22. package/dist/src/ai/captain.d.ts +20 -0
  23. package/dist/src/ai/captain.js +10 -1
  24. package/dist/src/ai/driller.js +6 -2
  25. package/dist/src/ai/fisherman-tools.d.ts +40 -1
  26. package/dist/src/ai/fisherman-tools.js +39 -0
  27. package/dist/src/ai/fisherman.js +2 -1
  28. package/dist/src/ai/navigator.d.ts +28 -0
  29. package/dist/src/ai/navigator.js +223 -175
  30. package/dist/src/ai/pilot.d.ts +7 -4
  31. package/dist/src/ai/pilot.js +89 -30
  32. package/dist/src/ai/planner/subpages.js +2 -16
  33. package/dist/src/ai/planner.js +1 -1
  34. package/dist/src/ai/provider.d.ts +2 -2
  35. package/dist/src/ai/provider.js +28 -22
  36. package/dist/src/ai/researcher/cache.d.ts +10 -3
  37. package/dist/src/ai/researcher/cache.js +23 -10
  38. package/dist/src/ai/researcher/deep-analysis.js +1 -1
  39. package/dist/src/ai/researcher/fingerprint-worker.js +21 -4
  40. package/dist/src/ai/researcher.js +6 -4
  41. package/dist/src/ai/rules.js +1 -5
  42. package/dist/src/ai/session-analyst.js +2 -0
  43. package/dist/src/ai/tester.d.ts +6 -3
  44. package/dist/src/ai/tester.js +30 -35
  45. package/dist/src/ai/tools.d.ts +8 -5
  46. package/dist/src/ai/tools.js +83 -57
  47. package/dist/src/commands/config-command.d.ts +51 -0
  48. package/dist/src/commands/config-command.js +117 -0
  49. package/dist/src/commands/index.js +2 -0
  50. package/dist/src/commands/init-command.js +13 -20
  51. package/dist/src/config.d.ts +8 -1
  52. package/dist/src/config.js +43 -1
  53. package/dist/src/experience-tracker.d.ts +2 -0
  54. package/dist/src/experience-tracker.js +12 -0
  55. package/dist/src/explorbot.js +5 -2
  56. package/dist/src/playwright-recorder.js +6 -12
  57. package/dist/src/remote.d.ts +3 -2
  58. package/dist/src/remote.js +8 -2
  59. package/dist/src/state-manager.d.ts +1 -1
  60. package/dist/src/state-manager.js +3 -1
  61. package/dist/src/test-plan.d.ts +9 -0
  62. package/dist/src/test-plan.js +30 -0
  63. package/dist/src/utils/html-diff.d.ts +5 -0
  64. package/dist/src/utils/html-diff.js +65 -6
  65. package/dist/src/utils/logger.d.ts +1 -1
  66. package/dist/src/utils/logger.js +8 -0
  67. package/dist/src/utils/strings.d.ts +2 -0
  68. package/dist/src/utils/strings.js +32 -0
  69. package/dist/src/utils/url-matcher.d.ts +1 -0
  70. package/dist/src/utils/url-matcher.js +31 -2
  71. package/docs/basics/getting-started.md +33 -10
  72. package/docs/basics/providers.md +6 -4
  73. package/docs/contributing/npm-package.md +73 -4
  74. package/docs/index.json +2 -1
  75. package/docs/reference/commands.md +3 -0
  76. package/docs/reference/websocket.md +50 -0
  77. package/docs/superpowers/specs/2026-08-18-prima-false-verdicts.md +159 -0
  78. package/models.json +4 -4
  79. package/package.json +6 -2
  80. package/src/action-result.ts +61 -16
  81. package/src/action.ts +56 -18
  82. package/src/ai/captain/web-mode.ts +1 -2
  83. package/src/ai/captain.ts +9 -1
  84. package/src/ai/driller.ts +6 -2
  85. package/src/ai/fisherman-tools.ts +35 -0
  86. package/src/ai/fisherman.ts +2 -1
  87. package/src/ai/navigator.ts +238 -179
  88. package/src/ai/pilot.ts +104 -36
  89. package/src/ai/planner/subpages.ts +2 -13
  90. package/src/ai/planner.ts +1 -1
  91. package/src/ai/provider.ts +29 -21
  92. package/src/ai/researcher/cache.ts +29 -11
  93. package/src/ai/researcher/deep-analysis.ts +1 -1
  94. package/src/ai/researcher/fingerprint-worker.ts +23 -5
  95. package/src/ai/researcher.ts +6 -4
  96. package/src/ai/rules.ts +1 -5
  97. package/src/ai/session-analyst.ts +2 -0
  98. package/src/ai/tester.ts +33 -34
  99. package/src/ai/tools.ts +88 -61
  100. package/src/commands/config-command.ts +146 -0
  101. package/src/commands/index.ts +2 -0
  102. package/src/commands/init-command.ts +14 -20
  103. package/src/config.ts +47 -2
  104. package/src/experience-tracker.ts +13 -0
  105. package/src/explorbot.ts +4 -2
  106. package/src/playwright-recorder.ts +6 -11
  107. package/src/remote.ts +8 -2
  108. package/src/state-manager.ts +5 -2
  109. package/src/test-plan.ts +38 -0
  110. package/src/utils/html-diff.ts +72 -7
  111. package/src/utils/logger.ts +9 -1
  112. package/src/utils/strings.ts +36 -0
  113. package/src/utils/url-matcher.ts +27 -2
@@ -32,30 +32,53 @@ OPENROUTER_API_KEY=sk-...
32
32
  Then open `explorbot.config.js` and set your app's base URL — the host only, no path:
33
33
 
34
34
  ```javascript
35
- import { createOpenRouter } from '@openrouter/ai-sdk-provider';
36
-
37
- const openrouter = createOpenRouter({
38
- apiKey: process.env.OPENROUTER_API_KEY,
39
- });
40
-
41
35
  export default {
42
36
  web: {
43
37
  url: 'http://localhost:3000',
44
38
  },
45
39
  ai: {
46
- model: openrouter('openai/gpt-oss-20b:nitro'),
47
- visionModel: openrouter('google/gemma-4-31b-it'),
48
- agenticModel: openrouter('minimax/minimax-m2.5:nitro'),
40
+ model: 'openrouter/openai/gpt-oss-20b:nitro',
41
+ visionModel: 'openrouter/openai/gpt-5.6-luna',
42
+ agenticModel: 'openrouter/openai/gpt-5.6-luna',
49
43
  },
50
44
  };
51
45
  ```
52
46
 
47
+ That shorthand — `'provider/model-id'` — uses a provider package Explorbot ships, so nothing extra is installed. Bundled providers:
48
+
49
+ - `openai`
50
+ - `anthropic`
51
+ - `google`
52
+ - `groq`
53
+ - `mistral`
54
+ - `openrouter`
55
+ - `sambanova`
56
+
57
+ The other style is explicit: install a Vercel AI SDK package (`npm i @ai-sdk/openai`) and build the client yourself. It works for the providers above too, and it is the only way to reach one that isn't bundled, a custom `baseURL`, or extra client options:
58
+
59
+ ```javascript
60
+ import { createOpenAI } from '@ai-sdk/openai';
61
+
62
+ const poolside = createOpenAI({
63
+ apiKey: process.env.POOLSIDE_API_KEY,
64
+ baseURL: 'https://inference.poolside.ai/v1',
65
+ });
66
+
67
+ export default {
68
+ ai: {
69
+ model: poolside('poolside/laguna-xs-2.1'),
70
+ },
71
+ };
72
+ ```
73
+
74
+ Both styles mix freely across the three keys. See [Providers](./providers.md).
75
+
53
76
  Explorbot uses three models. Pick each one for speed and cost:
54
77
 
55
78
  | Model | Config key | Used by | Pick |
56
79
  |-------|-----------|---------|------|
57
80
  | `model` | `ai.model` | Tester, Navigator, Researcher — they read HTML and ARIA on every step | a fast, cheap model (e.g. `openai/gpt-oss-20b:nitro`) |
58
- | `visionModel` | `ai.visionModel` | screenshot analysis | a vision model (e.g. `google/gemma-4-31b-it`) |
81
+ | `visionModel` | `ai.visionModel` | screenshot analysis | a vision model (e.g. `openai/gpt-5.6-luna`) |
59
82
  | `agenticModel` | `ai.agenticModel` | Captain and Pilot — they read short action logs and make the big decisions | a smarter model (e.g. MiniMax 2.5, Grok Fast) |
60
83
 
61
84
  Captain and Pilot barely use tokens, so a smarter `agenticModel` improves results for almost no extra cost. OpenRouter is the simplest start — one key, many models. To use OpenAI, Anthropic, Groq, or others, see [Providers](./providers.md). For every config option, see [Configuration](../reference/configuration.md).
@@ -2,6 +2,8 @@
2
2
 
3
3
  Explorbot connects to AI providers through the [Vercel AI SDK](https://sdk.vercel.ai/). Use any supported provider, and mix providers across different models.
4
4
 
5
+ Every provider below is set up the classical way: install its package, import it, build the client. Explorbot bundles some of these packages — for those you can skip the install and name the model as `'provider/model-id'` instead. See [Getting Started](./getting-started.md#2-configure) for that list and the two styles side by side.
6
+
5
7
  > The `export default` config block inside each `<!-- START/END provider -->` marker is generated from [`models.json`](../../models.json). After editing that file, run `bunosh docs:sync`. Everything else — including the import blocks — is hand-written.
6
8
 
7
9
  ## Requirements
@@ -51,8 +53,8 @@ Set the recommended models in the exported config:
51
53
  export default {
52
54
  ai: {
53
55
  model: openrouter('openai/gpt-oss-20b:nitro'),
54
- visionModel: openrouter('google/gemma-4-31b-it:nitro'),
55
- agenticModel: openrouter('google/gemma-4-31b-it:nitro'),
56
+ visionModel: openrouter('openai/gpt-5.6-luna'),
57
+ agenticModel: openrouter('openai/gpt-5.6-luna'),
56
58
  },
57
59
  };
58
60
  ```
@@ -118,8 +120,8 @@ Set the recommended models in the exported config:
118
120
  ```javascript
119
121
  export default {
120
122
  ai: {
121
- model: openai('gpt-5.4-nano'),
122
- visionModel: openai('gpt-5.4-nano'),
123
+ model: openai('gpt-5-nano'),
124
+ visionModel: openai('gpt-5.6-luna'),
123
125
  agenticModel: openai('gpt-5.6-luna'),
124
126
  },
125
127
  };
@@ -6,7 +6,7 @@ Explorbot develops on Bun but ships to npm as a Node.js-compatible package. This
6
6
 
7
7
  - Bun (for development and running the build)
8
8
  - Node.js >= 24 (for verifying the build output)
9
- - npm account with publish access to `explorbot` package
9
+ - npm account with publish access to `explorbot` and `prima-cli`, for publishing by hand; releases go out over OIDC (see [Trusted Publishing and Provenance](#trusted-publishing-and-provenance))
10
10
 
11
11
  ## How the Build Works
12
12
 
@@ -90,14 +90,16 @@ Key `package.json` fields:
90
90
  "boat/prima/src/**/*.ts",
91
91
  "boat/prima/bin/**/*.ts",
92
92
  "boat/prima/package.json",
93
+ "boat/prima/README.md",
93
94
  "rules/",
94
- "assets/sample-files/"
95
+ "assets/sample-files/",
96
+ "models.json"
95
97
  ],
96
98
  "engines": { "node": ">=24.0.0" }
97
99
  }
98
100
  ```
99
101
 
100
- The package ships two commands: `explorbot`, and `prima` for the [prima boat](../reference/commands.md#prima-boat), so `npx -p explorbot prima <command>` works without a separate install.
102
+ The package ships two commands, `explorbot` and `prima` for the [prima boat](../reference/commands.md#prima-boat). Prima is also mounted as a subcommand, so `npx explorbot prima <command>`, `npx -p explorbot prima <command>` and the standalone [`prima-cli`](#publishing-prima-cli) package all reach the same code.
101
103
 
102
104
  Explorbot is both a CLI (`bin`) and a library (`exports`). The `.` entry point is `src/index.ts`, a side-effect-free barrel that re-exports the public API (`ExplorBot`, `Plan`, `Test`, and their types). The `exports` conditions are ordered so each consumer gets the right entry: `types` (the emitted `.d.ts`) for type-checking, `bun` (the TypeScript source) under Bun, and `import` (the compiled JS) under Node.js. This is why the source `src/**` files ship alongside `dist/`.
103
105
 
@@ -127,6 +129,73 @@ npm version patch # or minor, major
127
129
  npm publish
128
130
  ```
129
131
 
132
+ ## Publishing prima-cli
133
+
134
+ Prima ships three ways, all the same code:
135
+
136
+ | | |
137
+ |---|---|
138
+ | `npx prima-cli` | its own package |
139
+ | `npx explorbot prima` | subcommand of the explorbot CLI |
140
+ | `npx -p explorbot prima` | the `prima` bin explorbot installs |
141
+
142
+ Prima is compiled into `dist/` by the same `tsc` run as everything else; only the packaging differs. `bun run build:prima` (`scripts/build-prima-npm.ts`) runs after `build:npm` and stages a second package:
143
+
144
+ ```
145
+ dist-prima/
146
+ ├── package.json # boat/prima/package.json + version and dependencies from the root manifest
147
+ ├── README.md # boat/prima/README.md, the npm page
148
+ └── dist/
149
+ ├── boat/prima/
150
+ ├── src/
151
+ ├── models.json
152
+ ├── rules/
153
+ └── assets/sample-files/
154
+ ```
155
+
156
+ The `dist/` layout is copied, not flattened: `config.js` reads `../models.json`, `rules-loader.js` reads `../../rules`, and `tester.js` reads `../../assets/sample-files`. Dependencies are copied verbatim from the root manifest rather than pruned to prima's closure — pruning saves little next to playwright and codeceptjs, and breaks on the first moved import. Edit `boat/prima/package.json` for the package name, bin, keywords or engines.
157
+
158
+ `publish-prima.yml` publishes it when a GitHub release is **published**, so a draft ships nothing. The version is the release tag with `prima-` and a leading `v` stripped: `0.2.6`, `v0.2.6` and `prima-v0.2.6` all publish `prima-cli@0.2.6`. A `prima-v*` tag is how prima ships on its own — `publish.yml` ignores it. A pre-release release, or a version containing `beta`, `alpha`, `pre` or `rc`, goes to the `beta` dist-tag.
159
+
160
+ Before publishing, the workflow packs the staged package, installs the tarball into an empty directory and runs it there with an empty `HOME` — a real consumer install, which is what catches a missing file, dependency or asset. It skips a version already on the registry, so a failed run can be re-run.
161
+
162
+ To check it locally:
163
+
164
+ ```bash
165
+ bun run build:npm && bun run build:prima
166
+ npm pack ./dist-prima --pack-destination /tmp
167
+ cd $(mktemp -d) && npm init -y && npm install --ignore-scripts /tmp/prima-cli-*.tgz
168
+ ./node_modules/.bin/prima-cli --help
169
+ ```
170
+
171
+ ## Trusted Publishing and Provenance
172
+
173
+ Both packages publish over OIDC, with no npm token in the repository. npm checks GitHub's identity token against a trusted publisher registered on the package, then attaches a provenance attestation. No `--provenance` flag is needed; trusted publishing does it.
174
+
175
+ The workflows are already set up for this: `id-token: write`, `ubuntu-latest`, and `npm@latest` for the npm 11.5.1+ requirement. The rest is per package on npmjs.com, because a trusted publisher names one package and one workflow file — `explorbot`'s does not cover `prima-cli`. The package has to exist before it can be configured, which is why a new one is claimed with a manual publish first.
176
+
177
+ On the package's **Settings** tab, under **Trusted Publisher**, choose **GitHub Actions**:
178
+
179
+ | Field | `explorbot` | `prima-cli` |
180
+ |---|---|---|
181
+ | Organization or user | `testomatio` | `testomatio` |
182
+ | Repository | `explorbot` | `explorbot` |
183
+ | Workflow filename | `publish.yml` | `publish-prima.yml` |
184
+ | Allowed actions | `npm publish` | `npm publish` |
185
+ | Environment name | empty | empty |
186
+
187
+ The workflow filename is a basename, and it is the field that differs — pointing `prima-cli` at `publish.yml` fails every release. Leave the environment empty unless the publish job gains an `environment:` key; a mismatch fails the publish.
188
+
189
+ To check a publish was attested:
190
+
191
+ ```bash
192
+ npm view <package>@<version> dist.attestations
193
+ ```
194
+
195
+ A `slsa.dev/provenance/v1` predicate means it worked. Nothing printed means the version went out unattested. `npm audit signatures` checks an installed tree.
196
+
197
+ Once no workflow needs a token, revoke the package's automation tokens and set its publishing access to require two-factor authentication and disallow tokens.
198
+
130
199
  ## Known Limitations
131
200
 
132
201
  - **Type declarations are transform-generated** - Declarations come from a transformed copy of the source (see [Type Declarations](#type-declarations)), not from `tsc --declaration` directly, because the mixin-based agents can't emit declarations as written. The published `.d.ts` types are exact; the workaround only concerns how they're produced.
@@ -135,4 +204,4 @@ npm publish
135
204
 
136
205
  The `test.yml` workflow verifies the npm build on every push. On Node.js 24 it runs `bun run build:npm`, then the Node smoke tests: `node --test tests/node/*.mjs`. The `publish.yml` workflow additionally checks `node dist/bin/explorbot-cli.js --help` before publishing.
137
206
 
138
- The `publish.yml` workflow publishes to npm when you push a version tag (`v*` or a bare `1.2.3`-style tag). It overwrites the package version from the tag; tags containing `beta`, `alpha`, `pre`, or `rc` publish to the `beta` dist-tag instead of `latest`.
207
+ The `publish.yml` workflow publishes `explorbot` when you push a version tag (`v*` or a bare `1.2.3`-style tag). It overwrites the package version from the tag; tags containing `beta`, `alpha`, `pre`, or `rc` publish to the `beta` dist-tag instead of `latest`. `publish-prima.yml` publishes `prima-cli` when a GitHub release is published. Both go out over OIDC and with provenance — see [Publishing prima-cli](#publishing-prima-cli) and [Trusted Publishing and Provenance](#trusted-publishing-and-provenance).
package/docs/index.json CHANGED
@@ -66,7 +66,8 @@
66
66
  "pages": [
67
67
  { "title": "Commands", "file": "reference/commands.md", "description": "Every CLI and terminal command" },
68
68
  { "title": "Configuration", "file": "reference/configuration.md", "description": "The config file, top to bottom" },
69
- { "title": "Scripting", "file": "reference/scripting.md", "description": "The programmatic API" }
69
+ { "title": "Scripting", "file": "reference/scripting.md", "description": "The programmatic API" },
70
+ { "title": "WebSocket stream", "file": "reference/websocket.md", "description": "Every frame a listener can read from a run" }
70
71
  ]
71
72
  },
72
73
  {
@@ -51,6 +51,7 @@ Inside the TUI, use the matching slash command: `/explore`, `/research`, `/plan`
51
51
  | Manage persistent browser | `npx explorbot browser {start\|stop\|status}` | — | Share browser across runs |
52
52
  | Initialize project | `npx explorbot init` | — | Generates `explorbot.config.*`, or `~/.explorbot` with `--global` |
53
53
  | List registered sites | `npx explorbot sites` | — | Sites stored in the global installation |
54
+ | Show resolved configuration | `npx explorbot config [url] [--json]` | `/config` | Models, config file, paths and `EXPLORBOT_*` in effect |
54
55
  | Clean generated files | `npx explorbot clean [target]` | `/clean [target]` | Same targets both ways |
55
56
 
56
57
  ## Common CLI Options
@@ -107,6 +108,8 @@ EXPLORBOT_AI_PROVIDER=openrouter \
107
108
  | `EXPLORBOT_NO_BANNER` | Suppress the startup banner, for machine-readable output |
108
109
  <!-- END env -->
109
110
 
111
+ `npx explorbot config` prints the values a run actually uses — models per role, the config file behind them, the output, knowledge and experience directories, and every `EXPLORBOT_*` variable currently set. The boats answer for their own configuration the same way: `npx explorbot api config`, `npx explorbot docs config`, `npx explorbot prima config`. Add `--json` on any of them to get the same values as an object a script can read.
112
+
110
113
  Explorbot resolves its configuration in this order: the path given to `--config`, then `explorbot.config.*` in the working directory, then the `EXPLORBOT_*` variables, and finally `~/.explorbot/config.*` from the global installation. A bare provider name fills every model role from the recommendations in [Providers](../basics/providers.md); a `provider/model-id` spec pins one model and splits on the first slash, so `openrouter/openai/gpt-oss-120b:nitro` selects OpenRouter with model `openai/gpt-oss-120b:nitro`. Supported providers: `openai`, `anthropic`, `google`, `groq`, `mistral`, `openrouter`, `sambanova`.
111
114
 
112
115
  In this mode output goes to `~/.explorbot/sites/<host>/output/` (or `EXPLORBOT_OUTPUT`, or a temp directory with `EXPLORBOT_EPHEMERAL=1`), experience is kept beside it unless the run is ephemeral, and the Historian is off, so no generated test files appear. See [Agentic Usage](../workflow/agentic-usage.md) for the full picture.
@@ -0,0 +1,50 @@
1
+ # WebSocket Stream
2
+
3
+ `--ws <url>` (or `EXPLORBOT_WS_URL`) streams a run to your own UI. Explorbot dials **out** — your side is the server — so the same flag covers a child process you spawned and a CI bot connecting from elsewhere.
4
+
5
+ ```bash
6
+ npx explorbot explore /dashboard --ws ws://127.0.0.1:8787
7
+ ```
8
+
9
+ Every message is JSON with a `type` and a `ts`, plus whatever that type carries. Nothing is validated on either side: render the types you know, ignore the rest, and expect new ones.
10
+
11
+ ## What a run sends
12
+
13
+ | Type | What it is |
14
+ |---|---|
15
+ | `hello` | the run itself: command, working directory, pid |
16
+ | `state` | the page under test: url, path, title, heading |
17
+ | `test` | a test starting or finishing, with its status, result and plan |
18
+ | `plan` | the current plan and the status of every test in it |
19
+ | `screenshot` | the screenshot file just written |
20
+ | `research` | the researcher's map of a page: markdown and its file |
21
+ | `report` | the analyst's end-of-session report: markdown and its file |
22
+ | `activity` | what the run is doing this second |
23
+ | `log` | a log line and its level |
24
+ | `ask` | a question for a human, carrying an `askId` |
25
+ | `result` | the run exited, with its code |
26
+
27
+ Each type carries the latest truth, so keep the last frame per type — that is what the terminal itself shows.
28
+
29
+ ## What you can send back
30
+
31
+ | Type | What it does |
32
+ |---|---|
33
+ | `answer` | answers an `ask`: same `askId`, plus `value` — or `null` to skip the question |
34
+ | `interrupt` | stops the current step; the run then asks what to do instead |
35
+
36
+ A run with nobody to answer never asks in the first place, so connecting a listener that answers is what makes a headless run interactive.
37
+
38
+ ## Delivery
39
+
40
+ Frames queue while disconnected — the last 1000 — and the connection retries on its own, so treat it as a live feed rather than a history. On exit the queue is flushed after `result`.
41
+
42
+ ## Adding a frame
43
+
44
+ Frames are logged, not published:
45
+
46
+ ```ts
47
+ tag('data').log('coverage', { visited: 12, total: 30 });
48
+ ```
49
+
50
+ That reaches listeners as a `coverage` frame. A `data` entry never goes to the console, the log file, or the TUI.
@@ -0,0 +1,159 @@
1
+ # Prima False Verdicts — No Reload, Vision-Confirmed Outcomes, Unconfirmed ≠ Failed
2
+
3
+ **Date:** 2026-08-18
4
+ **Status:** Implemented
5
+ **Follows:** `2026-08-07-prima-fixes-design.md`
6
+ **Evidence:** field feedback from an orchestrator driving prima over eight calls
7
+
8
+ ## Problem
9
+
10
+ Prima returned verdicts that did not match what happened, in both directions.
11
+
12
+ - `prima do` reported `error: open: <instruction>` for clicks that had landed. The caller only
13
+ found out by screenshotting anyway — which is the cost the boat exists to remove.
14
+ - `prima check` returned `ok: false` for an environmental reason and said nothing about it.
15
+ - No command answered a visual question with a verdict. `check` and `verify` never read a
16
+ screenshot; `ask` read one but returned prose.
17
+
18
+ The skill tells callers to trust `### Result`. A false red trains them out of that, and then the
19
+ greens stop meaning anything either.
20
+
21
+ ## Mechanisms found
22
+
23
+ 1. **`check` reloaded the page before checking it.** `prima.check()` → `tester.test()` →
24
+ `runTestSession` → `explorer.visit(task.startUrl!)` (`tester.ts:192`, unconditional) →
25
+ `I.amOnPage()` (`explorer.ts:431`) → `page.goto()`, which reloads even on the same URL.
26
+ `task.startUrl` is the page the caller is already on. Any transient state — an open dialog, a
27
+ selected tab, an unsaved form — was destroyed by the command asked to inspect it. This was
28
+ guaranteed, not a race with a dev-server reload.
29
+ 2. **`check` could not say why it failed.** `reportEnvelope` never sets `failure`, and
30
+ `envelope.steps` was built only from notes with `status === FAILED`. An abort produced
31
+ `ok: false` with an empty Steps section and no Failure section.
32
+ 3. **`do`'s verdict was the model's bookkeeping, not the page.** An instruction the model never
33
+ passed to `completed()` was rendered as an error, so an envelope could show every step green
34
+ and `ok: false` at once. `settleLedger`'s `.catch(() => null)` made a provider error
35
+ indistinguishable from a model that would not report.
36
+ 4. **Vision routing was per command, not per question.** `verify` produced DOM assertions only;
37
+ `check`'s verdict came from that same tool plus `settleExpectations`, which judged a text log.
38
+ The `inexpressible` branch told the model to "check it with `see()`" with no model in the loop
39
+ to act on the suggestion.
40
+ 5. **`prima status` printed the page tree.** `saveStatus` stored the full compact ARIA under
41
+ `changes`, so the command whose job is to cite artifact paths dumped the tree inline instead.
42
+
43
+ ## Changes
44
+
45
+ ### 1. `check` starts where the caller is
46
+
47
+ `Tester.test(task, opts)` takes `startOnCurrentPage`, which skips the initial visit. Prima passes
48
+ it. Nothing else changes for the explore flow, where reload-to-start-url is intentional.
49
+
50
+ `reset` needs no guard: it already refuses when the current URL equals the start URL, which is
51
+ the case for a check that starts in place. Once a check has navigated away, resetting back is
52
+ the right behaviour anyway.
53
+
54
+ ### 2. The screenshot is the proof, and a disagreement is a finding
55
+
56
+ `settleExpectations` is called from exactly one place, `prima.ts`, so it is prima's final judge
57
+ and can change without touching the explore flow. It now takes the final `ActionResult` and, when
58
+ that carries a screenshot and a vision model is configured, settles every outcome in one
59
+ structured call on the vision model with the image attached.
60
+
61
+ The screenshot is not one of two equal inputs. An outcome is satisfied when the page shows it to
62
+ somebody looking at it; the log only says what the run did. The prompt says so.
63
+
64
+ **Where the two disagree, the judge does not choose.** It reports `contradiction` and says what each
65
+ side shows. An assertion that matched an element nobody can see is a defect in the application, and
66
+ it is exactly the case both other verdicts destroy: `passed` hides it behind an assertion that
67
+ happens to match, `failed` mislabels a feature that half works. It comes back as its own status
68
+ with both sides quoted, it fails the command, and `### Artifacts` names the html, aria and screenshot
69
+ on disk so the caller can settle it on the page itself rather than on the judge's word.
70
+
71
+ **Absence in the picture is not a contradiction.** Review of the first pass raised this: a
72
+ screenshot is not proof that a thing is missing, only that the judge could not make it out. A
73
+ contradiction now requires the picture to show something *incompatible* — a list visibly empty, an error
74
+ where a result was expected, the old value still displayed. "I cannot find it" is `unverified`,
75
+ which does not fail the command.
76
+
77
+ The screenshot is the final page only. An outcome the run established earlier stays established
78
+ even when the page has moved past it, and the prompt says that is not a contradiction — otherwise every
79
+ "record deleted, then navigated away" scenario reports one.
80
+
81
+ ### 3. `check`'s verdict is its outcomes
82
+
83
+ `ok` no longer comes from `tester.test()`'s success flag, which could contradict the outcomes
84
+ printed beside it. `ok: true` when no outcome failed and none was contradicted; each failure and
85
+ each contradiction names itself in `### Failure`. `unverified` is not a failure — it is a statement about
86
+ the run, matching what the help text already promised.
87
+
88
+ A run that could not complete is reported separately from an application failure: when the test
89
+ never finished or was skipped, the envelope says the run established nothing and cites the last
90
+ step recorded.
91
+
92
+ ### 4. `do` distinguishes failed from unconfirmed
93
+
94
+ `ok` is a function of what ran, not of the paperwork. An action error or a `blocked()` fails the
95
+ command. An instruction the model never reported becomes a `??` row in `### Steps` — the actions
96
+ that ran are listed above it — and does not fail the command. An AI error while settling the
97
+ ledger is reported as its own step rather than attributed to the instruction.
98
+
99
+ The `<proof>` block gains one general line: how much of the page moved is not evidence of whether
100
+ something happened. A change confined to one region proves an instruction as well as one that
101
+ redraws everything.
102
+
103
+ ### 5. `verify` reaches for vision when no assertion can express the claim
104
+
105
+ The `inexpressible` branch now judges the claim from a screenshot and reports the judgement,
106
+ instead of dead-ending on a suggestion nothing acts on. Because Tester's `verify` tool calls the
107
+ same `navigator.verifyState`, this covers `check` as well.
108
+
109
+ `Prima.visionEnabled()` also honours `Stats.visionDisabled`, so a session that lost vision
110
+ mid-run stops claiming to have it.
111
+
112
+ ### 6. `status` cites artifacts instead of reprinting the page
113
+
114
+ The ARIA blob is gone from `status.json`. `status` returns the page block and the artifact paths,
115
+ which is its whole job.
116
+
117
+ ## Decisions Log
118
+
119
+ - `check` starts on the current page. A command that inspects transient UI must not destroy it.
120
+ - Vision is not a fallback in `check`; it closes every run that has a vision model. The
121
+ screenshot is the proof — what a user can see — and the run log only says what was done.
122
+ - A disagreement between the picture and the run is reported as `contradiction`, never settled one
123
+ way. An assertion matching an element nobody can see is a defect, and both `passed` and `failed`
124
+ would bury it. A contradiction fails the command, and hands the caller the page files to judge on.
125
+ The word is `contradiction` rather than `conflict` because it names what happened, and rather than
126
+ `ambiguity` because that is what `unverified` already means.
127
+ - A contradiction needs the picture to show something incompatible. Not finding something is `unverified`;
128
+ absence of evidence is not evidence of absence, and treating it as one is how a false-verdict fix
129
+ becomes a false-verdict generator.
130
+ - Nothing probes the page to explain *why* something is invisible. A first attempt walked the DOM
131
+ comparing colours, sniffing screen-reader patterns and hit-testing every element; it was a pile of
132
+ heuristics guessing at an answer the contradiction already states. "The run says it is there and
133
+ the picture does not show it" is the finding, and the caller is better placed to say why.
134
+ - `settleExpectations` judges all outcomes, not only undecided ones, when it has a screenshot —
135
+ otherwise a DOM assertion the run already made could never be contradicted by the page.
136
+ - `unverified` is not a failure, in `check` outcomes and in `do` instructions alike. A statement
137
+ about the run is not a statement about the application.
138
+ - Bookkeeping is not evidence. `do`'s `ok` follows actions and blocks; an unreported instruction
139
+ is surfaced, never rendered as an error.
140
+ - A run that could not complete is reported as such, never as an application failure.
141
+ - `provider.getVisionModel()` is added for symmetry with `getModelForAgent`/`getAgenticModel`;
142
+ `processImage` returns free text and cannot carry a per-outcome verdict.
143
+ - A failed vision judgement falls back to the text model and flips `Stats.visionDisabled` — the
144
+ existing global for "vision is not usable this session" — which prima reads through
145
+ `visionEnabled()`.
146
+ - No line reports which evidence the judge had; that is plumbing. Only the degraded case is
147
+ stated, as a `### Warning`, because only that case is a fact the caller must act on.
148
+ - The verdict vocabulary lives in `prima <command> --help`. A marker the caller can see in an
149
+ envelope but cannot look up is not documented.
150
+
151
+ ## Not done
152
+
153
+ - `explorer.beginTest` still calls `closeOtherTabs()`, so `check` closes other tabs of an
154
+ attached session. Guarding it means threading an option through `beginTest`, which every flow
155
+ shares.
156
+ - `do` gets no vision confirmation pass. Its `completed()` proof is the same kind of unverified
157
+ paperwork, but a per-instruction vision call is a different cost profile.
158
+ - The `prima` skill in `testomatio/skills` documents the old envelope vocabulary. It needs the
159
+ `??` row, the `CONTRADICTION` status, and the unconfirmed-is-not-failed rule.
package/models.json CHANGED
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "openrouter": {
3
3
  "model": "openai/gpt-oss-20b:nitro",
4
- "visionModel": "google/gemma-4-31b-it:nitro",
5
- "agenticModel": "google/gemma-4-31b-it:nitro"
4
+ "visionModel": "openai/gpt-5.6-luna",
5
+ "agenticModel": "openai/gpt-5.6-luna"
6
6
  },
7
7
  "poolside": {
8
8
  "model": "poolside/laguna-xs-2.1"
@@ -13,8 +13,8 @@
13
13
  "agenticModel": "qwen/qwen3.6-27b"
14
14
  },
15
15
  "openai": {
16
- "model": "gpt-5.4-nano",
17
- "visionModel": "gpt-5.4-nano",
16
+ "model": "gpt-5-nano",
17
+ "visionModel": "gpt-5.6-luna",
18
18
  "agenticModel": "gpt-5.6-luna"
19
19
  },
20
20
  "anthropic": {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "explorbot",
3
- "version": "0.2.4",
3
+ "version": "0.3.0",
4
4
  "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
5
5
  "license": "Elastic-2.0",
6
6
  "type": "module",
@@ -30,6 +30,7 @@
30
30
  "boat/prima/src/**/*.ts",
31
31
  "boat/prima/bin/**/*.ts",
32
32
  "boat/prima/package.json",
33
+ "boat/prima/README.md",
33
34
  "rules/",
34
35
  "assets/sample-files/",
35
36
  "models.json"
@@ -58,7 +59,8 @@
58
59
  "lint:fix": "biome lint --write .",
59
60
  "check": "biome check .",
60
61
  "check:fix": "biome check --write .",
61
- "langfuse:export": "bun run .claude/skills/explorbot-debug/langfuse-export.ts"
62
+ "langfuse:export": "bun run .claude/skills/explorbot-debug/langfuse-export.ts",
63
+ "build:prima": "bun run scripts/build-prima-npm.ts"
62
64
  },
63
65
  "keywords": [
64
66
  "cli",
@@ -97,6 +99,7 @@
97
99
  "ai": "^7.0.2",
98
100
  "axe-core": "^4.11.1",
99
101
  "bash-tool": "^1.3.15",
102
+ "chalk": "^5.6.2",
100
103
  "cli-highlight": "^2.1.11",
101
104
  "codeceptjs": "4.0.0-rc.16",
102
105
  "commander": "^14.0.1",
@@ -121,6 +124,7 @@
121
124
  "parse5": "^8.0.0",
122
125
  "pixelmatch": "^7.2.0",
123
126
  "playwright": "^1.62",
127
+ "playwright-core": "^1.62",
124
128
  "pngjs": "^7.0.0",
125
129
  "react": "^19.1.1",
126
130
  "sambanova-ai-provider": "^1.2.2",