explorbot 0.2.2 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. package/README.md +1 -1
  2. package/bin/explorbot-cli.ts +59 -38
  3. package/boat/api-tester/src/apibot.ts +4 -2
  4. package/boat/api-tester/src/cli.ts +2 -2
  5. package/boat/api-tester/src/config.ts +43 -10
  6. package/boat/doc-collector/bin/doc-collector-cli.ts +2 -0
  7. package/boat/doc-collector/src/ai/documentarian.ts +61 -31
  8. package/boat/doc-collector/src/cli.ts +1 -0
  9. package/boat/doc-collector/src/config.ts +4 -2
  10. package/boat/doc-collector/src/docs-renderer.ts +18 -4
  11. package/boat/doc-collector/src/state-diagram.ts +61 -14
  12. package/boat/prima/bin/prima-cli.ts +5 -0
  13. package/boat/prima/package.json +16 -0
  14. package/boat/prima/src/activity-line.ts +33 -0
  15. package/boat/prima/src/cli.ts +251 -0
  16. package/boat/prima/src/envelope.ts +169 -0
  17. package/boat/prima/src/prima.ts +1124 -0
  18. package/boat/prima/src/pw-parser.ts +27 -0
  19. package/boat/prima/src/pw-registry.ts +74 -0
  20. package/boat/prima/src/session-log.ts +126 -0
  21. package/dist/bin/explorbot-cli.js +51 -32
  22. package/dist/boat/api-tester/bin/apibot-cli.js +2 -0
  23. package/dist/boat/api-tester/src/apibot.js +3 -2
  24. package/dist/boat/api-tester/src/cli.js +2 -2
  25. package/dist/boat/api-tester/src/config.js +40 -10
  26. package/dist/boat/doc-collector/bin/doc-collector-cli.js +2 -0
  27. package/dist/boat/doc-collector/src/ai/documentarian.js +44 -19
  28. package/dist/boat/doc-collector/src/cli.js +1 -0
  29. package/dist/boat/doc-collector/src/config.js +4 -2
  30. package/dist/boat/doc-collector/src/docs-renderer.js +17 -3
  31. package/dist/boat/doc-collector/src/state-diagram.js +57 -13
  32. package/dist/boat/prima/bin/prima-cli.js +4 -0
  33. package/dist/boat/prima/src/activity-line.js +30 -0
  34. package/dist/boat/prima/src/cli.js +220 -0
  35. package/dist/boat/prima/src/envelope.js +142 -0
  36. package/dist/boat/prima/src/prima.js +1031 -0
  37. package/dist/boat/prima/src/pw-parser.js +30 -0
  38. package/dist/boat/prima/src/pw-registry.js +65 -0
  39. package/dist/boat/prima/src/session-log.js +108 -0
  40. package/dist/models.json +3 -0
  41. package/dist/package.json +8 -3
  42. package/dist/rules/navigator/verification-actions.md +20 -0
  43. package/dist/src/action-result.d.ts +7 -0
  44. package/dist/src/action-result.js +4 -0
  45. package/dist/src/action.d.ts +7 -2
  46. package/dist/src/action.js +41 -6
  47. package/dist/src/ai/captain/mixin.js +3 -4
  48. package/dist/src/ai/captain/web-mode.js +7 -4
  49. package/dist/src/ai/captain.js +2 -0
  50. package/dist/src/ai/navigator.d.ts +11 -0
  51. package/dist/src/ai/navigator.js +21 -12
  52. package/dist/src/ai/pilot.d.ts +4 -0
  53. package/dist/src/ai/pilot.js +48 -2
  54. package/dist/src/ai/planner.d.ts +1 -0
  55. package/dist/src/ai/planner.js +6 -0
  56. package/dist/src/ai/provider.js +2 -2
  57. package/dist/src/ai/rerunner.js +2 -1
  58. package/dist/src/ai/researcher.js +2 -2
  59. package/dist/src/ai/rules.js +17 -10
  60. package/dist/src/ai/task-agent.js +5 -2
  61. package/dist/src/ai/tester.d.ts +1 -0
  62. package/dist/src/ai/tester.js +33 -20
  63. package/dist/src/ai/tools.d.ts +14 -0
  64. package/dist/src/ai/tools.js +113 -36
  65. package/dist/src/application-spec-contract.d.ts +8 -0
  66. package/dist/src/application-spec-contract.js +8 -0
  67. package/dist/src/application-spec.d.ts +15 -0
  68. package/dist/src/application-spec.js +71 -0
  69. package/dist/src/browser-server.d.ts +12 -6
  70. package/dist/src/browser-server.js +74 -19
  71. package/dist/src/commands/clean-command.js +2 -7
  72. package/dist/src/commands/init-command.d.ts +5 -0
  73. package/dist/src/commands/init-command.js +119 -1
  74. package/dist/src/commands/navigate-command.js +1 -1
  75. package/dist/src/commands/research-command.js +1 -1
  76. package/dist/src/commands/sites-command.d.ts +6 -0
  77. package/dist/src/commands/sites-command.js +23 -0
  78. package/dist/src/components/InitWizard.d.ts +10 -0
  79. package/dist/src/components/InitWizard.js +133 -0
  80. package/dist/src/components/InputReadline.d.ts +1 -0
  81. package/dist/src/components/InputReadline.js +7 -4
  82. package/dist/src/config.d.ts +25 -5
  83. package/dist/src/config.js +158 -40
  84. package/dist/src/execution-controller.d.ts +2 -0
  85. package/dist/src/execution-controller.js +6 -0
  86. package/dist/src/explorbot.d.ts +11 -1
  87. package/dist/src/explorbot.js +27 -6
  88. package/dist/src/explorer.d.ts +4 -1
  89. package/dist/src/explorer.js +42 -9
  90. package/dist/src/global-config.d.ts +22 -0
  91. package/dist/src/global-config.js +117 -0
  92. package/dist/src/knowledge-tracker.d.ts +5 -1
  93. package/dist/src/knowledge-tracker.js +14 -1
  94. package/dist/src/playwright-recorder.js +30 -0
  95. package/dist/src/remote.d.ts +54 -0
  96. package/dist/src/remote.js +229 -0
  97. package/dist/src/reporter.d.ts +1 -0
  98. package/dist/src/reporter.js +7 -1
  99. package/dist/src/state-manager.d.ts +2 -1
  100. package/dist/src/stats.d.ts +1 -0
  101. package/dist/src/stats.js +1 -0
  102. package/dist/src/test-plan.d.ts +2 -0
  103. package/dist/src/test-plan.js +7 -0
  104. package/dist/src/utils/aria.d.ts +2 -8
  105. package/dist/src/utils/aria.js +69 -40
  106. package/dist/src/utils/cli-name.js +6 -2
  107. package/dist/src/utils/html.js +1 -0
  108. package/dist/src/utils/logger.d.ts +6 -0
  109. package/dist/src/utils/logger.js +24 -0
  110. package/dist/src/utils/page-readiness.js +18 -1
  111. package/dist/src/utils/test-files.js +1 -2
  112. package/dist/src/utils/url-matcher.d.ts +1 -0
  113. package/dist/src/utils/url-matcher.js +12 -0
  114. package/dist/src/utils/web-element.d.ts +2 -0
  115. package/dist/src/utils/web-element.js +8 -0
  116. package/dist/src/utils/web-sandbox.d.ts +1 -1
  117. package/dist/src/utils/web-sandbox.js +2 -3
  118. package/docs/api-testing/basics.md +90 -0
  119. package/docs/api-testing/planning.md +57 -0
  120. package/docs/api-testing/running-tests.md +55 -0
  121. package/docs/assets/cloud-report.png +0 -0
  122. package/docs/assets/html-report.png +0 -0
  123. package/docs/assets/langfuse-trace.png +0 -0
  124. package/docs/assets/successful-explore-run.png +0 -0
  125. package/docs/basics/getting-started.md +140 -0
  126. package/docs/basics/prerequisites.md +63 -0
  127. package/docs/basics/providers.md +362 -0
  128. package/docs/basics/running.md +78 -0
  129. package/docs/contributing/ai-integration-tests.md +57 -0
  130. package/docs/contributing/contributing.md +90 -0
  131. package/docs/contributing/demo-videos.md +36 -0
  132. package/docs/contributing/npm-package.md +138 -0
  133. package/docs/contributing/observability.md +227 -0
  134. package/docs/contributing/regression-tests.md +103 -0
  135. package/docs/contributing/testing.md +95 -0
  136. package/docs/doc-collection/basics.md +128 -0
  137. package/docs/doc-collection/crawling.md +67 -0
  138. package/docs/doc-collection/interactive-mode.md +99 -0
  139. package/docs/index.json +86 -0
  140. package/docs/reference/commands.md +994 -0
  141. package/docs/reference/configuration.md +569 -0
  142. package/docs/reference/scripting.md +303 -0
  143. package/docs/superpowers/plans/2026-08-01-actor-boat.md +925 -0
  144. package/docs/superpowers/plans/2026-08-01-prima-boat.md +1120 -0
  145. package/docs/superpowers/reviews/2026-08-06-prima-vs-playwright-cli.md +268 -0
  146. package/docs/superpowers/specs/2026-08-01-actor-boat-design.md +204 -0
  147. package/docs/superpowers/specs/2026-08-01-prima-boat-design.md +242 -0
  148. package/docs/superpowers/specs/2026-08-03-global-config-design.md +138 -0
  149. package/docs/superpowers/specs/2026-08-07-prima-fixes-design.md +394 -0
  150. package/docs/web-testing/agents.md +158 -0
  151. package/docs/web-testing/automated-tests.md +134 -0
  152. package/docs/web-testing/basics.md +91 -0
  153. package/docs/web-testing/customization.md +131 -0
  154. package/docs/web-testing/hooks.md +238 -0
  155. package/docs/web-testing/page-interaction.md +84 -0
  156. package/docs/web-testing/planner.md +122 -0
  157. package/docs/web-testing/rerun.md +164 -0
  158. package/docs/web-testing/researcher.md +380 -0
  159. package/docs/workflow/agentic-usage.md +233 -0
  160. package/docs/workflow/application-spec.md +73 -0
  161. package/docs/workflow/ci.md +202 -0
  162. package/docs/workflow/knowledge.md +310 -0
  163. package/docs/workflow/planning-styles.md +67 -0
  164. package/docs/workflow/reporting.md +133 -0
  165. package/docs/workflow/test-plans.md +90 -0
  166. package/models.json +3 -0
  167. package/package.json +8 -3
  168. package/rules/navigator/verification-actions.md +20 -0
  169. package/src/action-result.ts +11 -0
  170. package/src/action.ts +47 -7
  171. package/src/ai/captain/mixin.ts +3 -3
  172. package/src/ai/captain/web-mode.ts +7 -4
  173. package/src/ai/captain.ts +3 -0
  174. package/src/ai/navigator.ts +26 -15
  175. package/src/ai/pilot.ts +53 -2
  176. package/src/ai/planner.ts +7 -0
  177. package/src/ai/provider.ts +2 -4
  178. package/src/ai/rerunner.ts +2 -1
  179. package/src/ai/researcher.ts +2 -2
  180. package/src/ai/rules.ts +17 -10
  181. package/src/ai/task-agent.ts +4 -2
  182. package/src/ai/tester.ts +32 -18
  183. package/src/ai/tools.ts +132 -36
  184. package/src/application-spec-contract.ts +10 -0
  185. package/src/application-spec.ts +87 -0
  186. package/src/browser-server.ts +74 -19
  187. package/src/commands/clean-command.ts +1 -6
  188. package/src/commands/init-command.ts +146 -1
  189. package/src/commands/navigate-command.ts +1 -1
  190. package/src/commands/research-command.ts +1 -1
  191. package/src/commands/sites-command.ts +27 -0
  192. package/src/components/InitWizard.tsx +166 -0
  193. package/src/components/InputReadline.tsx +8 -4
  194. package/src/config.ts +177 -43
  195. package/src/execution-controller.ts +8 -0
  196. package/src/explorbot.ts +34 -7
  197. package/src/explorer.ts +47 -9
  198. package/src/global-config.ts +148 -0
  199. package/src/knowledge-tracker.ts +17 -1
  200. package/src/playwright-recorder.ts +23 -0
  201. package/src/remote.ts +238 -0
  202. package/src/reporter.ts +7 -1
  203. package/src/state-manager.ts +2 -1
  204. package/src/stats.ts +1 -0
  205. package/src/test-plan.ts +9 -0
  206. package/src/utils/aria.ts +65 -45
  207. package/src/utils/cli-name.ts +5 -2
  208. package/src/utils/html.ts +1 -0
  209. package/src/utils/logger.ts +24 -1
  210. package/src/utils/page-readiness.ts +24 -1
  211. package/src/utils/test-files.ts +1 -2
  212. package/src/utils/url-matcher.ts +13 -0
  213. package/src/utils/web-element.ts +9 -0
  214. package/src/utils/web-sandbox.ts +3 -4
@@ -0,0 +1,394 @@
1
+ # Prima Fixes — Perception Ladder, No Heal, Proof-Carrying Envelopes
2
+
3
+ **Date:** 2026-08-07
4
+ **Status:** Draft for review
5
+ **Supersedes parts of:** `2026-08-01-prima-boat-design.md`
6
+ **Evidence:** `docs/superpowers/reviews/2026-08-06-prima-vs-playwright-cli.md`
7
+
8
+ ## Problem
9
+
10
+ A field run of prima against a live app (Testeiya, `localhost:3050`) found the boat working but
11
+ not trustworthy as an executor. Four defects matter:
12
+
13
+ - A `pw` call on a selector that does not exist returned `ok: true`, `healed: true`, having
14
+ clicked an unrelated control ("New agent"). Heal substituted the target and reported success.
15
+ - `### Changes` never appeared in any successful envelope, so a successful action proved
16
+ nothing and every step had to be confirmed with a playwright-cli snapshot.
17
+ - `do` given two instructions performed four, typing into a live chat box actions nobody asked
18
+ for.
19
+ - Auto-discovery of a playwright-cli session never matched, while the failure message advised
20
+ opening the session the user already had open.
21
+
22
+ Underneath the third defect is a perception problem. `action.ts:139` captures
23
+ `page.locator('body').ariaSnapshot()` **without** `mode: 'ai'`, so `do` sees roles and text but
24
+ no element handles and must invent locators. That is why the `click` tool's schema demands
25
+ "multiple commands targeting the SAME element" and why one click produced five ladder attempts,
26
+ one of them invalid JavaScript.
27
+
28
+ ## Goals
29
+
30
+ - Prima is the executor for an expensive orchestrator: page data never enters that context on
31
+ the happy path, intent is never guessed, and every action carries proof of what changed.
32
+ - `do` acts on element handles it was given, not on locators it invented.
33
+ - Failures are failures. No code path may reach a different element than the one asked for.
34
+ - Attachment to an existing playwright-cli session works without hand-editing files.
35
+
36
+ ## Non-Goals
37
+
38
+ - Replacing playwright-cli. It remains the fallback for direct driving and the tool prima
39
+ points back to when it cannot help.
40
+ - Reducing per-command latency, and improving `research` locator quality. Separate concerns.
41
+ - Changing Tester, Navigator, or Researcher behaviour outside the shared pieces named below.
42
+
43
+ ---
44
+
45
+ ## 1. Perception ladder for `do`
46
+
47
+ Four tiers, tried in order. Each tier answers "what can I act on here?" at a different cost.
48
+
49
+ | Tier | Source | When |
50
+ |---|---|---|
51
+ | 1 | Research UI map | a stored map exists for this state hash and the state is well-visited |
52
+ | 2 | **ARIA snapshot with refs** | default |
53
+ | 3 | Compact HTML tree | a ref action failed, or the control is absent from the ARIA tree |
54
+ | 4 | Vision (`visualClick`) | click only, when the target is identifiable solely by appearance |
55
+
56
+ ### Tier 1 — research map
57
+
58
+ Read through `getPreviousResearch(hash)` — a TTL-free disk read. Not `getCachedResearch`, whose
59
+ six-hour TTL carries session-scoped semantics that must not be stretched for cross-invocation
60
+ reuse. The trigger is the StateManager visit count for the current state hash, the same number
61
+ the envelope already prints as `visit #N`: use the map from the **third visit onward**, so a
62
+ state has to prove it is worth the map before prima prefers one. Below that, skip to tier 2.
63
+ The threshold is a single config field under `ai.agents.prima`, per the one-knob convention.
64
+
65
+ Research cost is real — 41s on the vision model in the field run — so tier 1 only pays off
66
+ amortized across repeat visits to one state. Prima does not run research on the caller's behalf
67
+ inside `do`; it consumes a map that already exists.
68
+
69
+ ### Tier 2 — ARIA snapshot with refs (the default)
70
+
71
+ `page.locator('body').ariaSnapshot({ mode: 'ai' })` emits `[ref=eN]` handles. Playwright resolves
72
+ them natively through the `aria-ref=` selector engine, verified live against the running app:
73
+
74
+ ```
75
+ aria-ref=e1 → 1 match
76
+ ```
77
+
78
+ No DOM mutation, no attribute stamping. The model is given the ref-bearing tree and names a ref;
79
+ resolution to a CodeceptJS command is §1.1. One ref becomes one command and the
80
+ multiple-locator fallback ladder collapses.
81
+
82
+ **Ref lifetime is one context injection.** Refs are snapshot-scoped and shift between calls —
83
+ in the field run the same sidebar button was `ref=e13` in one snapshot and `ref=e284` in a later
84
+ one on the same page. Prima re-snapshots whenever the `do` loop re-injects context on state-hash
85
+ change (the hook exists at `prima.ts:104`), and the prompt states that refs from an earlier
86
+ injection are dead.
87
+
88
+ **Ref shape.** Refs are frame-qualified: every ref this app emits is `f1e13`-shaped, not `e13`.
89
+ A validator that accepts only `e\d+` rejects every real ref and silently disables the whole tier,
90
+ so the accepted shape is `(f\d+)?e\d+`. Refs are never adapted or invented — a ref that does not
91
+ parse, or that resolves to nothing, is a failure with its own message.
92
+
93
+ ### Tier 3 — compact HTML tree
94
+
95
+ The current `simplifiedHtml` path. Entered when a ref action fails or the target is not
96
+ represented in the accessibility tree.
97
+
98
+ ### Descending the ladder — `context()`
99
+
100
+ The tiers above are only reachable if something can move between them mid-run. The `do` loop
101
+ re-injects context between iterations when the state hash changes, which leaves no way to
102
+ recover from a ref that died *within* an iteration — the model's only remaining moves would be
103
+ to guess a locator, which the prompt forbids, or to stop.
104
+
105
+ `do` therefore carries a `context` tool. First call returns the page as it is now with fresh
106
+ refs, which replace every ref the model was holding; a later call on the same page drops to
107
+ capped markup for elements the accessibility tree does not describe. It is the tier descent, not
108
+ a page dump.
109
+
110
+ This is deliberately **not** the `context` tool from `createAgentTools`. That one returns
111
+ `getInteractiveARIA()` — `compactAriaSnapshot` over `ActionResult.ariaSnapshot`, captured without
112
+ `mode: 'ai'`, so it carries **no refs** — plus a 6k-char HTML dump beside it. Offering it here
113
+ would hand the model a ref-less tree and push it straight back to guessed locators, undoing
114
+ tier 2.
115
+
116
+ ### Tier 4 — vision
117
+
118
+ `visualClick` (`src/ai/tools.ts:791`), already implemented. Click only — a coordinate is not a
119
+ handle, so it cannot serve fills, selects, or assertions.
120
+
121
+ ### 1.1 Acting on a ref — `clickRef` / `hoverRef` in `boat/prima/src/tools.ts`
122
+
123
+ CodeceptJS has no `aria-ref` locator, so a ref cannot be handed to `I.click()` directly. It is
124
+ resolved to a locator CodeceptJS does understand, using machinery that already exists:
125
+
126
+ ```
127
+ page.locator(`aria-ref=${ref}`)
128
+ → WebElement.fromPlaywrightLocator(...) (web-element.ts:112)
129
+ → I.click(<xpath>)
130
+ ```
131
+
132
+ **Which XPath, and the one that is not available.** `WebElement` declares two — `clickXPath`,
133
+ built attribute-first by `buildClickableXPath` (`utils/xpath.ts`), and `xpath`, the absolute
134
+ positional path. On this path only `clickXPath` exists: `fromPlaywrightLocator` builds through
135
+ `fromRawData`, which hardcodes `xpath: ''` (`web-element.ts:89`). Only `fromXPathMatch`, the
136
+ static-HTML path, populates the absolute form. So there is no positional fallback to reach for,
137
+ and that is the right outcome anyway — an absolute path is deduplication machinery, fragile the
138
+ moment the DOM shifts, and the ref resolution and the click happen on separate round-trips.
139
+
140
+ Verified live, ref → `clickXPath` → match count:
141
+
142
+ ```
143
+ e15 link "Checking the proxy…" //*[self::a and contains(.,"Checking the proxy and the firewall")] → 1
144
+ e19 button "Reload" //*[@id="reload-button"] → 1
145
+ e20 button "Details" //*[@id="details-button"] → 1
146
+ ```
147
+
148
+ **Require a unique match.** Because there is no fallback, the resolved `clickXPath` is checked
149
+ to match exactly one element before it is used. Zero or many is a failure, reported as such —
150
+ never a click on an ambiguous match and never a retreat to guessed locators.
151
+
152
+ Ref acting lives in **separate tools in the boat** — `clickRef` and `hoverRef`, from a
153
+ `createRefTools` factory in `boat/prima/src/tools.ts`. `click` and `hover` in `src/ai/tools.ts`
154
+ are left byte-identical. A prima-only tool belongs to prima; core keeps only what every caller
155
+ uses, and lends the boat its result-shaping helpers (`successToolResult`, `failedToolResult`,
156
+ `commitNote`) rather than having them copied.
157
+
158
+ Adding an optional `ref` to the existing tools looks cheaper and is wrong. A tool's schema and
159
+ description are shared with every caller, and Tester never receives ref-bearing snapshots, so it
160
+ would be shown a parameter it can only fill by inventing one. Making `commands` optional to
161
+ accommodate the new field weakens the contract for Tester too. "Prima enables ref mode, Tester is
162
+ untouched" is not achievable through one definition: there is one definition, and Tester sees it.
163
+
164
+ Each ref tool takes a ref and nothing else, resolves it, executes one command, and reports that
165
+ command as `used:` — real CodeceptJS a generated test can keep. A ref that does not resolve is a
166
+ failure, not a cue to fall back to a guessed locator: it means the context is stale.
167
+
168
+ `hoverRef` exists for the same reason as `hover` — revealing hover-only controls is a
169
+ prerequisite for clicking them.
170
+
171
+ Prima keeps the locator tools alongside the ref tools: tier 3 works from markup that carries no
172
+ refs, and needs them. Tier 4 keeps the coordinate input.
173
+
174
+ ---
175
+
176
+ ## 2. Heal is deleted
177
+
178
+ `heal()` (`prima.ts:428-460`) is removed, along with `--no-heal`, `PrimaOptions.heal`, the
179
+ `healed:` / `healNote` envelope fields, the `HealAttempt` type, and the
180
+ `### Healing attempts` section. The three call sites — `pw` (`:79`), `do` (`:135`), `go` (`:211`)
181
+ — go straight to `failureEnvelope`.
182
+
183
+ The failure envelope already does the right thing and becomes the only failure path: `ok: false`,
184
+ the exact error with its call log, and compact ARIA inline. Measured at 8.6s in the field run
185
+ against 34.8s for the heal path that got the answer wrong.
186
+
187
+ This removes the substitution defect at the root. No remaining code path can select an element
188
+ other than the one asked for, so no envelope can report success for an action the caller did not
189
+ request. Routine obstructions — an overlay covering a button, an element not yet visible — now
190
+ return the failure envelope, and the orchestrating model decides. That is the accepted cost: the
191
+ compact ARIA block is the one place page data deliberately enters the expensive context.
192
+
193
+ ---
194
+
195
+ ## 3. Proof-carrying envelopes
196
+
197
+ Two defects with one cause. `renderOutcome` (`envelope.ts:76`) returns the **first** of
198
+ changes / answer / research / verdict, so `### Changes` is structurally impossible alongside
199
+ `### Verdict`, `### Answer`, or `### Research`. And `pageChanges` returns `ariaChanges ?? null`,
200
+ which renders nothing when the diff is empty or `previousState` is null.
201
+
202
+ - `### Changes` renders on every action envelope, showing `no change` explicitly when the tree is
203
+ identical. A caller can then tell "nothing happened" from "prima did not say".
204
+ - `renderOutcome` stops being mutually exclusive: `### Changes` renders alongside the command's
205
+ own outcome section.
206
+ - Prima captures an explicit before-snapshot rather than relying on whatever `stateManager`
207
+ holds at process start.
208
+ - New `### Steps` block for `do`: one line per instruction, each naming the ariaDiff entry that
209
+ proves it, or marked `unproven`.
210
+
211
+ ### Refs must never reach the diff or hash pipeline
212
+
213
+ The ref-bearing snapshot is a **context artifact only**. Refs are stripped before hashing and
214
+ diffing. Storing the `mode: 'ai'` output in `ActionResult.ariaSnapshot` would poison every
215
+ `### Changes` block, because Playwright renumbers refs on each call. Measured on an identical
216
+ page whose refs merely shifted:
217
+
218
+ ```
219
+ ref churn only, page identical → diff count = 6
220
+ added: button "Cancel" [ref=e21], button "Save" [ref=e20], textbox "Name" [ref=e22]
221
+ removed: button "Cancel" [ref=e11], button "Save" [ref=e10], textbox "Name" [ref=e12]
222
+ ```
223
+
224
+ Six phantom entries for a page that did not change. Capture ref-bearing and ref-free variants,
225
+ and feed only the ref-free one to `diffAriaSnapshots` and `getStateHash`.
226
+
227
+ ---
228
+
229
+ ## 4. Executor prompt
230
+
231
+ `instructionSystemPrompt` (`prima.ts:493`) already says to stop when every instruction is done,
232
+ but nothing tracks per-instruction completion, so the loop runs until the model stops calling
233
+ tools. Two rules are added, stated as general principles rather than as counter-examples from
234
+ any debugging session:
235
+
236
+ - Act only on the listed instructions. An adjacent action that appears helpful is out of scope;
237
+ report it as an observation instead of performing it.
238
+ - For each instruction, cite the observed page change that proves it. An instruction that cannot
239
+ be tied to an observed change is reported unproven rather than claimed as done.
240
+
241
+ The prompt also states the ref contract: act on refs from the current context injection; refs
242
+ from an earlier injection are dead.
243
+
244
+ ---
245
+
246
+ ## 5. Attachment and discovery
247
+
248
+ Three separate faults, all confirmed empirically.
249
+
250
+ **Connect with the daemon's own build when attached.** `connectDescriptor` (`prima.ts:374`)
251
+ tries prima's own playwright first and only falls back to `descriptor.playwrightLib` if connect
252
+ fails. Connect *succeeds* across builds, so the fallback is never reached — and then the tier-2
253
+ snapshot breaks:
254
+
255
+ | client lib | `connect()` | `ariaSnapshot({mode:'ai'})` |
256
+ |---|---|---|
257
+ | own playwright 1.62.1 | ok | `locator.ariaSnapshot: timeout: expected float, got undefined` |
258
+ | daemon playwright-core 1.62.0-alpha | ok | ok, 3376 bytes, `aria-ref=e1` → 1 match |
259
+
260
+ In attached mode, prefer `descriptor.playwrightLib` when present and fall back to our own. Carry
261
+ `playwrightLib` through the `--endpoint` path too, which currently hardcodes `''`.
262
+
263
+ **Stop keying discovery on `workspaceDir`.** No `@playwright/cli` release writes that field —
264
+ verified on 0.1.13 and 0.1.17. `parseDescriptor` (`pw-registry.ts:56`) requires it and
265
+ `selectDescriptor` filters on it, so every descriptor is dropped and discovery finds nothing no
266
+ matter what is installed. Resolution becomes: `--endpoint` → `--pw-session <title>` →
267
+ `PLAYWRIGHT_CLI_SESSION` → live descriptor titled `default` → the single live descriptor →
268
+ tool error listing candidate titles. Liveness-probe before selecting.
269
+
270
+ **Bump the playwright pin to `^1.62`.** `playwright@^1.60` could not `connect()` to a 1.61/1.62
271
+ browser server at all; 1.62.1 connects. The pin is the reason the original review concluded
272
+ attach was broken outright.
273
+
274
+ Consequences: `browser list` shows attachable sessions rather than reporting none while one is
275
+ live, and the no-browser error stops advising `playwright-cli open <url>` to someone who already
276
+ ran it.
277
+
278
+ ---
279
+
280
+ ## 6. `verify` honesty
281
+
282
+ `rules/navigator/verification-actions.md` offers nine assertions — `see`, `seeElement`,
283
+ `seeInField`, `seeInTitle`, `seeInSource` and their `dontSee*` counterparts — none of which
284
+ express enabled, disabled, checked, selected, or expanded. A claim about interactive state is
285
+ therefore unprovable by construction, and the field run reported a working feature as failing:
286
+
287
+ ```
288
+ verify "the Save button in the skill editor is now enabled ..."
289
+ → passed: false, code: (empty), evidence: no assertion held on the current page
290
+ ```
291
+
292
+ The app was correct — `● unsaved` was displayed and the button was enabled.
293
+
294
+ - Add state assertions to the rule.
295
+ - `verify` distinguishes **assertion failed** from **could not express this assertion**. The
296
+ second is not a test failure and must not be reported as one.
297
+ - With §7's compaction fix, `verify` reads state from the same ref-bearing snapshot the tier-2
298
+ ladder produces.
299
+
300
+ ---
301
+
302
+ ## 7. Prerequisite fix — ARIA compaction drops refs and state
303
+
304
+ `compactAriaSnapshot` keeps only the first bracket group on a line. Playwright emits state
305
+ attributes before `[ref=]`, so every stateful control loses its handle. Measured:
306
+
307
+ ```
308
+ button "Plain" [ref=e10] → ref KEPT
309
+ button "Active" [active] [ref=e13] → ref LOST
310
+ button "Disabled" [disabled] [ref=e14] → ref LOST
311
+ button "Pressed" [pressed] [ref=e15] → ref LOST
312
+ button "Expanded" [expanded] [ref=e16] → ref LOST
313
+ checkbox "Checked" [checked] [ref=e17] → ref LOST
314
+ button "Cursor" [ref=e18] [cursor=pointer] → ref KEPT
315
+ ```
316
+
317
+ The controls most worth acting on and asserting about are exactly the ones stripped of their
318
+ handle. **Keep every bracket group on a line.** No allow-list, no drop-list — parsing one group
319
+ and discarding the rest is the whole bug, and any rule about which groups survive re-creates it
320
+ the next time Playwright adds an attribute. This fix is a prerequisite for §1 tier 2 and enables
321
+ §6.
322
+
323
+ ---
324
+
325
+ ## 8. Envelope hygiene
326
+
327
+ - `used:` carries the winning line only — no concatenated ladder attempts, no comment lines. The
328
+ field run produced `I.click(".sidebar button:has-text("Workflows")")`, which is not valid
329
+ JavaScript, and a `// 1. Open dialog...` comment inside the code block.
330
+ - `click` and `fill` stop labelling themselves `do` in `command:`.
331
+ - `network.jsonl` is written and advertised only when requests were actually captured. It was
332
+ 0 bytes across all 18 runs while being advertised in every envelope; an artifact line that
333
+ points at an empty file is worse than no line.
334
+ - Research annotation overlays are removed from the DOM after use. The browser is shared with
335
+ playwright-cli, and leftover `Legend` / `e8` / `e10` nodes appeared in its next snapshot.
336
+ - Commands stop requiring a URL when attached to a browser already on a page.
337
+ - A redirect that preserves origin and path counts as navigation success. The app's
338
+ `/` → `/?session=<uuid>&ws=1` redirect cost 3m00s and eight attempts under the old heal path,
339
+ and a hard tool failure against a prima-owned browser — in an envelope whose own inlined ARIA
340
+ proved the page had loaded.
341
+
342
+ ---
343
+
344
+ ## Testing
345
+
346
+ - **Unit** — envelope rendering with `### Changes` always present, including the `no change`
347
+ form and the combination with `### Verdict` / `### Answer` / `### Research`; compaction
348
+ preserving every bracket group across the §7 combinations and any order of them; ref-free
349
+ diffing (the six-phantom case must yield zero); descriptor selection without `workspaceDir`,
350
+ including the ambiguous multi-session error.
351
+ - **Ref resolution** — a ref resolves to an attribute-based `clickXPath` matching exactly one
352
+ element, and still matching after unrelated siblings are added or removed; a ref that resolves
353
+ to zero or many fails rather than falling back to a guessed locator.
354
+ - **Integration** — `do` prompt behaviour through the existing `@copilotkit/aimock` harness per
355
+ `docs/contributing/ai-integration-tests.md`: instructions performed and nothing beyond them,
356
+ per-instruction proof citation, unproven reporting. Fictional fixture data only.
357
+ - **End-to-end** — against a local fixture: a ref named from the snapshot clicks the element it
358
+ names and no other; a failing `pw` returns `ok: false` with compact ARIA and never a
359
+ substituted action; attach to a live playwright-cli session with no descriptor editing; a
360
+ session-param redirect resolves as success.
361
+
362
+ ---
363
+
364
+ ## Decisions Log
365
+
366
+ - Heal is deleted outright rather than constrained to same-target recovery. No flag, no opt-in.
367
+ - `do` perception is a four-tier ladder: research map, ARIA with refs, compact HTML, vision for
368
+ click.
369
+ - Refs come from Playwright's native `aria-ref=` engine. No eidx attribute stamping, so the
370
+ shared browser's DOM is not mutated.
371
+ - Ref acting lives in `boat/prima/src/tools.ts`, not in core; `click` and `hover` are untouched.
372
+ Extending a shared tool would show Tester a `ref` parameter it can only fill by inventing one,
373
+ and would weaken `commands` for every caller.
374
+ - A ref resolves through `WebElement` to an attribute-based `clickXPath`. The absolute positional
375
+ `xpath` is not a fallback — `fromRawData` never populates it on the live-locator path, and it
376
+ would be the fragile choice regardless.
377
+ - A resolved `clickXPath` must match exactly one element. A dead or ambiguous ref is a failure,
378
+ never a fallback to guessed locators.
379
+ - ARIA compaction keeps every bracket group. No allow-list of attributes to preserve.
380
+ - Refs are a context artifact only, stripped before hashing and diffing.
381
+ - Ref lifetime is one context injection; the loop re-snapshots on state-hash change.
382
+ - Tier 1 reads maps via `getPreviousResearch`; the trigger is StateManager visit count.
383
+ - Discovery matches on title plus liveness; `workspaceDir` is abandoned as a key.
384
+ - Attached mode connects with `descriptor.playwrightLib` first, because connect succeeds across
385
+ builds but `ariaSnapshot` does not.
386
+ - The playwright pin moves to `^1.62`.
387
+ - `verify` reports inexpressible assertions as inexpressible, never as failures, and does not
388
+ record them as verifications — a claim that could not be checked must not be remembered as
389
+ one that failed.
390
+ - `do` carries a prima-shaped `context` tool that descends the ladder — fresh refs first, capped
391
+ markup on a repeat call. The shared `context` from `createAgentTools` is not reused: it returns
392
+ a ref-less tree and an HTML dump.
393
+ - The baseline snapshot is captured lazily when a command needs a diff, not in `start()`, which
394
+ runs before a page is loaded.
@@ -0,0 +1,158 @@
1
+ # AI Agents
2
+
3
+ Explorbot splits the testing workflow across specialized AI agents. Each agent handles one part of the work, which keeps it focused and keeps token costs down.
4
+
5
+ ## Agent Overview
6
+
7
+ ```mermaid
8
+ flowchart LR
9
+ A[Navigator] --> B[Researcher] --> C[Planner] --> D[Tester]
10
+ A -- "goes to page" --> B
11
+ B -- "analyzes UI" --> C
12
+ C -- "suggests tests" --> D
13
+ D -- "runs tests" --> A
14
+ E[Pilot] -.->|supervises| D
15
+ ```
16
+
17
+ ## Navigator Agent
18
+
19
+ Handles browser interactions: clicks, form fills, and navigation.
20
+
21
+ The Navigator runs CodeceptJS commands in the browser. When a selector fails, it tries other locator strategies and resolves the interaction without stopping the run. It remembers what worked and what didn't, so failed selectors don't keep killing your runs and tests survive UI changes.
22
+
23
+ Commands that use Navigator:
24
+ - `/navigate <target>`
25
+ - `I.click()`, `I.fillField()`, `I.amOnPage()`, etc.
26
+
27
+ ## Researcher Agent
28
+
29
+ Analyzes pages to find what's actually on them.
30
+
31
+ The Researcher discovers every interactive element, including content hidden in accordions, dropdowns, and modals. It maps navigation paths and form structures, extracts data from tables and lists, and filters out noise like cookie banners and ads. The result is a complete map of what you can test, with form validation rules documented.
32
+
33
+ Commands that use Researcher:
34
+ - `npx explorbot research /path` (CLI)
35
+ - `/research [path]` (TUI)
36
+ - `/research --deep` — expand hidden elements
37
+ - `/research --data` — extract page data
38
+
39
+ Explicit research always captures a screenshot; a configured vision model analyzes it.
40
+
41
+ See [Researcher Agent](./researcher.md) for configuration and usage.
42
+
43
+ ## Planner Agent
44
+
45
+ Generates test scenarios from research findings.
46
+
47
+ The Planner writes business-focused scenarios with priority levels (critical/important/high/normal/low) and expected outcomes for verification. It balances positive and negative cases, skips scenarios you already have, and cycles through planning styles (normal, psycho, curious) to broaden coverage across iterations. You can add your own styles and page-specific rules.
48
+
49
+ Commands that use Planner:
50
+ - `/plan [--focus <feature>]`
51
+ - `/explore`
52
+
53
+ See [Planner Agent](./planner.md) for planning styles, customization, and configuration.
54
+
55
+ ## Tester Agent
56
+
57
+ Runs the planned scenarios.
58
+
59
+ The Tester executes scenarios step by step and adapts when something goes wrong. It tracks state changes during execution, records actual results against expected ones, and uses research context to make decisions. It handles unexpected modals and popups, recovers from minor failures on its own, and produces detailed execution logs.
60
+
61
+ Commands that use Tester:
62
+ - `/test [scenario]`
63
+ - `/explore`
64
+
65
+ ## Pilot Agent
66
+
67
+ Supervises the Tester and steps in when a test gets stuck.
68
+
69
+ The Pilot keeps a separate conversation to track progress over time. It detects stuck patterns — loops, repeated failures, no page changes — and decides what context the Tester needs next (HTML, ARIA, UI map). When automated recovery fails, it asks the user for help instead of giving up.
70
+
71
+ Because the Pilot sees only tool summaries rather than raw HTML, you can run it on a smarter model without a token cost explosion.
72
+
73
+ The Pilot intervenes when:
74
+ - Actions succeed but the page doesn't change (wrong element)
75
+ - The same action repeats several times (loop)
76
+ - The same locator keeps failing (need a different approach)
77
+ - Only research or context calls run, with no action tools (no progress)
78
+
79
+ ## Analyst Agent
80
+
81
+ Produces a human-readable session report after `/explore` and `/freesail` runs.
82
+
83
+ The Analyst reads every test in the session — scenario, expected outcome, final result, notes, and step log. It clusters findings by root cause: three tests that fail on the same control become one defect with three test references, not three rows. It separates product defects from automation problems, so an element the test could not click is not reported as a bug.
84
+
85
+ The report is plain markdown: a one-to-two sentence headline about the feature state (no test counts), then `## Coverage`, `## What works`, `## Defects` (each with a `[High]`/`[Medium]`/`[Low]` severity tag, reproduce steps, and one-line evidence), `## UX issues`, and `## Execution Issues`. Empty sections are omitted.
86
+
87
+ It runs automatically at the end of `/explore` (per-run) and on app exit (session-wide, across multiple `/explore` or `/freesail` runs). The same markdown goes to the console, to `output/reports/<mode>-<sessionName>.md`, and — when the reporter is enabled — to the Testomat.io run description.
88
+
89
+ Configuration:
90
+
91
+ ```javascript
92
+ export default {
93
+ ai: {
94
+ agents: {
95
+ analyst: {
96
+ // model: openai('gpt-4o'), // override the model
97
+ // systemPrompt: 'Focus on...', // append guidance to the prompt
98
+ // enabled: false, // disable the analyst entirely
99
+ },
100
+ },
101
+ },
102
+ };
103
+ ```
104
+
105
+ The Analyst resolves its model like other agentic tasks: the agent-specific `model`, then `ai.agenticModel`, then `ai.model`.
106
+
107
+ ## Captain Agent
108
+
109
+ Handles your direct requests in the TUI and recovers the session when something breaks.
110
+
111
+ The Captain steps in when a slash command isn't enough — answering questions about your setup, inspecting tests and page states, and reading recent output before it replies. It works in four modes:
112
+
113
+ - **idle** — plan management, project inspection, knowledge and experience files; available before any page loads
114
+ - **web** — page interaction, navigation, and browser diagnostics
115
+ - **test** — test timeline, state inspection, generated code and logs
116
+ - **heal** — browser and test recovery when an active test loses its page or browser
117
+
118
+ When a test hits a fatal browser error, the Captain tries to recover — reload, restart the browser, open a fresh tab, or close extra tabs — before the test is stopped, then tells the Tester how to continue. It runs on explicit TUI requests and on test interrupts (stop, pass, skip, or redirect a running test).
119
+
120
+ ## Per-Agent Model Configuration
121
+
122
+ Use different models for different agents to control cost:
123
+
124
+ ```javascript
125
+ export default {
126
+ ai: {
127
+ model: groq('openai/gpt-oss-20b'),
128
+ visionModel: groq('meta-llama/llama-4-scout-17b-16e-instruct'),
129
+ agents: {
130
+ navigator: { model: groq('openai/gpt-oss-20b') },
131
+ researcher: { model: groq('openai/gpt-oss-20b') },
132
+ planner: { model: groq('openai/gpt-oss-20b') },
133
+ tester: { model: groq('openai/gpt-oss-20b'), progressCheckInterval: 5 },
134
+ pilot: { stepsToReview: 5 },
135
+ },
136
+ },
137
+ };
138
+ ```
139
+
140
+ Typical choices:
141
+ - Navigator needs fast responses for real-time interaction
142
+ - Researcher benefits from vision
143
+ - Planner can use a larger model for better test design
144
+ - Tester needs tool use for execution
145
+ - Pilot can use a smarter model — it only processes tool summaries, not HTML or ARIA
146
+
147
+ ## How Agents Communicate
148
+
149
+ Agents share context through four channels:
150
+
151
+ 1. **State Manager** — tracks the current page, URL, and navigation history
152
+ 2. **Research Results** — structured page analysis available to Planner and Tester
153
+ 3. **Experience Files** — patterns learned across sessions. Injected as a compact table of contents (file tags plus section headings) rather than full bodies; agents pull individual sections on demand via the `learnExperience` tool
154
+ 4. **Knowledge Files** — domain knowledge you provide
155
+
156
+ Each agent keeps its context minimal to hold costs down. Agents request specific information when they need it instead of carrying full conversation history.
157
+
158
+ The Pilot keeps a separate conversation from the Tester. The Tester's conversation carries heavy HTML and ARIA context; the Pilot sees only tool execution summaries (what succeeded, what failed, what changed). That lets the Pilot run on expensive models without a token cost explosion.