explorbot 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/explorbot-cli.ts +7 -1
- package/boat/api-tester/src/config.ts +4 -2
- package/boat/doc-collector/bin/doc-collector-cli.ts +2 -0
- package/boat/doc-collector/src/ai/documentarian.ts +61 -31
- package/boat/doc-collector/src/config.ts +4 -2
- package/boat/prima/bin/prima-cli.ts +0 -0
- package/boat/prima/src/activity-line.ts +33 -0
- package/boat/prima/src/cli.ts +115 -86
- package/boat/prima/src/envelope.ts +80 -52
- package/boat/prima/src/prima.ts +547 -128
- package/boat/prima/src/pw-parser.ts +11 -1
- package/boat/prima/src/pw-registry.ts +4 -5
- package/boat/prima/src/session-log.ts +126 -0
- package/dist/bin/explorbot-cli.js +7 -1
- package/dist/boat/api-tester/bin/apibot-cli.js +2 -0
- package/dist/boat/api-tester/src/config.js +4 -2
- package/dist/boat/doc-collector/bin/doc-collector-cli.js +2 -0
- package/dist/boat/doc-collector/src/ai/documentarian.js +44 -19
- package/dist/boat/doc-collector/src/config.js +4 -2
- package/dist/boat/prima/src/activity-line.js +30 -0
- package/dist/boat/prima/src/cli.js +97 -77
- package/dist/boat/prima/src/envelope.js +70 -44
- package/dist/boat/prima/src/prima.js +515 -119
- package/dist/boat/prima/src/pw-parser.js +13 -1
- package/dist/boat/prima/src/pw-registry.js +4 -5
- package/dist/boat/prima/src/session-log.js +108 -0
- package/dist/package.json +3 -2
- package/dist/rules/navigator/verification-actions.md +20 -0
- package/dist/src/action-result.d.ts +7 -0
- package/dist/src/action-result.js +4 -0
- package/dist/src/action.d.ts +2 -0
- package/dist/src/action.js +36 -1
- package/dist/src/ai/captain/web-mode.js +6 -3
- package/dist/src/ai/captain.js +2 -0
- package/dist/src/ai/navigator.d.ts +7 -0
- package/dist/src/ai/navigator.js +10 -6
- package/dist/src/ai/pilot.d.ts +4 -0
- package/dist/src/ai/pilot.js +48 -2
- package/dist/src/ai/provider.js +2 -2
- package/dist/src/ai/rerunner.js +2 -1
- package/dist/src/ai/researcher.js +1 -1
- package/dist/src/ai/rules.js +17 -10
- package/dist/src/ai/task-agent.js +4 -1
- package/dist/src/ai/tester.d.ts +1 -1
- package/dist/src/ai/tester.js +33 -33
- package/dist/src/ai/tools.d.ts +14 -0
- package/dist/src/ai/tools.js +113 -36
- package/dist/src/config.d.ts +1 -0
- package/dist/src/config.js +13 -4
- package/dist/src/execution-controller.d.ts +2 -0
- package/dist/src/execution-controller.js +6 -0
- package/dist/src/explorbot.d.ts +2 -1
- package/dist/src/explorbot.js +3 -1
- package/dist/src/explorer.js +2 -3
- package/dist/src/playwright-recorder.js +30 -0
- package/dist/src/remote.d.ts +54 -0
- package/dist/src/remote.js +229 -0
- package/dist/src/reporter.d.ts +1 -0
- package/dist/src/reporter.js +7 -1
- package/dist/src/state-manager.d.ts +2 -1
- package/dist/src/stats.d.ts +1 -0
- package/dist/src/stats.js +1 -0
- package/dist/src/test-plan.d.ts +2 -0
- package/dist/src/test-plan.js +7 -0
- package/dist/src/utils/aria.d.ts +2 -8
- package/dist/src/utils/aria.js +69 -40
- package/dist/src/utils/html.js +1 -0
- package/dist/src/utils/logger.d.ts +6 -0
- package/dist/src/utils/logger.js +24 -0
- package/dist/src/utils/page-readiness.js +18 -1
- package/dist/src/utils/url-matcher.js +3 -0
- package/dist/src/utils/web-element.d.ts +2 -0
- package/dist/src/utils/web-element.js +8 -0
- package/dist/src/utils/web-sandbox.d.ts +1 -1
- package/dist/src/utils/web-sandbox.js +2 -3
- package/docs/api-testing/basics.md +90 -0
- package/docs/api-testing/planning.md +57 -0
- package/docs/api-testing/running-tests.md +55 -0
- package/docs/assets/cloud-report.png +0 -0
- package/docs/assets/html-report.png +0 -0
- package/docs/assets/langfuse-trace.png +0 -0
- package/docs/assets/successful-explore-run.png +0 -0
- package/docs/basics/getting-started.md +140 -0
- package/docs/basics/prerequisites.md +63 -0
- package/docs/basics/providers.md +362 -0
- package/docs/basics/running.md +78 -0
- package/docs/contributing/ai-integration-tests.md +57 -0
- package/docs/contributing/contributing.md +90 -0
- package/docs/contributing/demo-videos.md +36 -0
- package/docs/contributing/npm-package.md +138 -0
- package/docs/contributing/observability.md +227 -0
- package/docs/contributing/regression-tests.md +103 -0
- package/docs/contributing/testing.md +95 -0
- package/docs/doc-collection/basics.md +128 -0
- package/docs/doc-collection/crawling.md +67 -0
- package/docs/doc-collection/interactive-mode.md +99 -0
- package/docs/index.json +86 -0
- package/docs/reference/commands.md +994 -0
- package/docs/reference/configuration.md +569 -0
- package/docs/reference/scripting.md +303 -0
- package/docs/superpowers/plans/2026-08-01-actor-boat.md +925 -0
- package/docs/superpowers/plans/2026-08-01-prima-boat.md +1120 -0
- package/docs/superpowers/reviews/2026-08-06-prima-vs-playwright-cli.md +268 -0
- package/docs/superpowers/specs/2026-08-01-actor-boat-design.md +204 -0
- package/docs/superpowers/specs/2026-08-01-prima-boat-design.md +242 -0
- package/docs/superpowers/specs/2026-08-03-global-config-design.md +138 -0
- package/docs/superpowers/specs/2026-08-07-prima-fixes-design.md +394 -0
- package/docs/web-testing/agents.md +158 -0
- package/docs/web-testing/automated-tests.md +134 -0
- package/docs/web-testing/basics.md +91 -0
- package/docs/web-testing/customization.md +131 -0
- package/docs/web-testing/hooks.md +238 -0
- package/docs/web-testing/page-interaction.md +84 -0
- package/docs/web-testing/planner.md +122 -0
- package/docs/web-testing/rerun.md +164 -0
- package/docs/web-testing/researcher.md +380 -0
- package/docs/workflow/agentic-usage.md +233 -0
- package/docs/workflow/application-spec.md +73 -0
- package/docs/workflow/ci.md +202 -0
- package/docs/workflow/knowledge.md +310 -0
- package/docs/workflow/planning-styles.md +67 -0
- package/docs/workflow/reporting.md +133 -0
- package/docs/workflow/test-plans.md +90 -0
- package/package.json +3 -2
- package/rules/navigator/verification-actions.md +20 -0
- package/src/action-result.ts +11 -0
- package/src/action.ts +38 -2
- package/src/ai/captain/web-mode.ts +6 -3
- package/src/ai/captain.ts +3 -0
- package/src/ai/navigator.ts +14 -8
- package/src/ai/pilot.ts +53 -2
- package/src/ai/provider.ts +2 -4
- package/src/ai/rerunner.ts +2 -1
- package/src/ai/researcher.ts +1 -1
- package/src/ai/rules.ts +17 -10
- package/src/ai/task-agent.ts +3 -1
- package/src/ai/tester.ts +32 -33
- package/src/ai/tools.ts +132 -36
- package/src/config.ts +15 -4
- package/src/execution-controller.ts +8 -0
- package/src/explorbot.ts +4 -2
- package/src/explorer.ts +2 -2
- package/src/playwright-recorder.ts +23 -0
- package/src/remote.ts +238 -0
- package/src/reporter.ts +7 -1
- package/src/state-manager.ts +2 -1
- package/src/stats.ts +1 -0
- package/src/test-plan.ts +9 -0
- package/src/utils/aria.ts +65 -45
- package/src/utils/html.ts +1 -0
- package/src/utils/logger.ts +24 -1
- package/src/utils/page-readiness.ts +24 -1
- package/src/utils/url-matcher.ts +3 -0
- package/src/utils/web-element.ts +9 -0
- package/src/utils/web-sandbox.ts +3 -4
|
@@ -0,0 +1,394 @@
|
|
|
1
|
+
# Prima Fixes — Perception Ladder, No Heal, Proof-Carrying Envelopes
|
|
2
|
+
|
|
3
|
+
**Date:** 2026-08-07
|
|
4
|
+
**Status:** Draft for review
|
|
5
|
+
**Supersedes parts of:** `2026-08-01-prima-boat-design.md`
|
|
6
|
+
**Evidence:** `docs/superpowers/reviews/2026-08-06-prima-vs-playwright-cli.md`
|
|
7
|
+
|
|
8
|
+
## Problem
|
|
9
|
+
|
|
10
|
+
A field run of prima against a live app (Testeiya, `localhost:3050`) found the boat working but
|
|
11
|
+
not trustworthy as an executor. Four defects matter:
|
|
12
|
+
|
|
13
|
+
- A `pw` call on a selector that does not exist returned `ok: true`, `healed: true`, having
|
|
14
|
+
clicked an unrelated control ("New agent"). Heal substituted the target and reported success.
|
|
15
|
+
- `### Changes` never appeared in any successful envelope, so a successful action proved
|
|
16
|
+
nothing and every step had to be confirmed with a playwright-cli snapshot.
|
|
17
|
+
- `do` given two instructions performed four, typing into a live chat box actions nobody asked
|
|
18
|
+
for.
|
|
19
|
+
- Auto-discovery of a playwright-cli session never matched, while the failure message advised
|
|
20
|
+
opening the session the user already had open.
|
|
21
|
+
|
|
22
|
+
Underneath the third defect is a perception problem. `action.ts:139` captures
|
|
23
|
+
`page.locator('body').ariaSnapshot()` **without** `mode: 'ai'`, so `do` sees roles and text but
|
|
24
|
+
no element handles and must invent locators. That is why the `click` tool's schema demands
|
|
25
|
+
"multiple commands targeting the SAME element" and why one click produced five ladder attempts,
|
|
26
|
+
one of them invalid JavaScript.
|
|
27
|
+
|
|
28
|
+
## Goals
|
|
29
|
+
|
|
30
|
+
- Prima is the executor for an expensive orchestrator: page data never enters that context on
|
|
31
|
+
the happy path, intent is never guessed, and every action carries proof of what changed.
|
|
32
|
+
- `do` acts on element handles it was given, not on locators it invented.
|
|
33
|
+
- Failures are failures. No code path may reach a different element than the one asked for.
|
|
34
|
+
- Attachment to an existing playwright-cli session works without hand-editing files.
|
|
35
|
+
|
|
36
|
+
## Non-Goals
|
|
37
|
+
|
|
38
|
+
- Replacing playwright-cli. It remains the fallback for direct driving and the tool prima
|
|
39
|
+
points back to when it cannot help.
|
|
40
|
+
- Reducing per-command latency, and improving `research` locator quality. Separate concerns.
|
|
41
|
+
- Changing Tester, Navigator, or Researcher behaviour outside the shared pieces named below.
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## 1. Perception ladder for `do`
|
|
46
|
+
|
|
47
|
+
Four tiers, tried in order. Each tier answers "what can I act on here?" at a different cost.
|
|
48
|
+
|
|
49
|
+
| Tier | Source | When |
|
|
50
|
+
|---|---|---|
|
|
51
|
+
| 1 | Research UI map | a stored map exists for this state hash and the state is well-visited |
|
|
52
|
+
| 2 | **ARIA snapshot with refs** | default |
|
|
53
|
+
| 3 | Compact HTML tree | a ref action failed, or the control is absent from the ARIA tree |
|
|
54
|
+
| 4 | Vision (`visualClick`) | click only, when the target is identifiable solely by appearance |
|
|
55
|
+
|
|
56
|
+
### Tier 1 — research map
|
|
57
|
+
|
|
58
|
+
Read through `getPreviousResearch(hash)` — a TTL-free disk read. Not `getCachedResearch`, whose
|
|
59
|
+
six-hour TTL carries session-scoped semantics that must not be stretched for cross-invocation
|
|
60
|
+
reuse. The trigger is the StateManager visit count for the current state hash, the same number
|
|
61
|
+
the envelope already prints as `visit #N`: use the map from the **third visit onward**, so a
|
|
62
|
+
state has to prove it is worth the map before prima prefers one. Below that, skip to tier 2.
|
|
63
|
+
The threshold is a single config field under `ai.agents.prima`, per the one-knob convention.
|
|
64
|
+
|
|
65
|
+
Research cost is real — 41s on the vision model in the field run — so tier 1 only pays off
|
|
66
|
+
amortized across repeat visits to one state. Prima does not run research on the caller's behalf
|
|
67
|
+
inside `do`; it consumes a map that already exists.
|
|
68
|
+
|
|
69
|
+
### Tier 2 — ARIA snapshot with refs (the default)
|
|
70
|
+
|
|
71
|
+
`page.locator('body').ariaSnapshot({ mode: 'ai' })` emits `[ref=eN]` handles. Playwright resolves
|
|
72
|
+
them natively through the `aria-ref=` selector engine, verified live against the running app:
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
aria-ref=e1 → 1 match
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
No DOM mutation, no attribute stamping. The model is given the ref-bearing tree and names a ref;
|
|
79
|
+
resolution to a CodeceptJS command is §1.1. One ref becomes one command and the
|
|
80
|
+
multiple-locator fallback ladder collapses.
|
|
81
|
+
|
|
82
|
+
**Ref lifetime is one context injection.** Refs are snapshot-scoped and shift between calls —
|
|
83
|
+
in the field run the same sidebar button was `ref=e13` in one snapshot and `ref=e284` in a later
|
|
84
|
+
one on the same page. Prima re-snapshots whenever the `do` loop re-injects context on state-hash
|
|
85
|
+
change (the hook exists at `prima.ts:104`), and the prompt states that refs from an earlier
|
|
86
|
+
injection are dead.
|
|
87
|
+
|
|
88
|
+
**Ref shape.** Refs are frame-qualified: every ref this app emits is `f1e13`-shaped, not `e13`.
|
|
89
|
+
A validator that accepts only `e\d+` rejects every real ref and silently disables the whole tier,
|
|
90
|
+
so the accepted shape is `(f\d+)?e\d+`. Refs are never adapted or invented — a ref that does not
|
|
91
|
+
parse, or that resolves to nothing, is a failure with its own message.
|
|
92
|
+
|
|
93
|
+
### Tier 3 — compact HTML tree
|
|
94
|
+
|
|
95
|
+
The current `simplifiedHtml` path. Entered when a ref action fails or the target is not
|
|
96
|
+
represented in the accessibility tree.
|
|
97
|
+
|
|
98
|
+
### Descending the ladder — `context()`
|
|
99
|
+
|
|
100
|
+
The tiers above are only reachable if something can move between them mid-run. The `do` loop
|
|
101
|
+
re-injects context between iterations when the state hash changes, which leaves no way to
|
|
102
|
+
recover from a ref that died *within* an iteration — the model's only remaining moves would be
|
|
103
|
+
to guess a locator, which the prompt forbids, or to stop.
|
|
104
|
+
|
|
105
|
+
`do` therefore carries a `context` tool. First call returns the page as it is now with fresh
|
|
106
|
+
refs, which replace every ref the model was holding; a later call on the same page drops to
|
|
107
|
+
capped markup for elements the accessibility tree does not describe. It is the tier descent, not
|
|
108
|
+
a page dump.
|
|
109
|
+
|
|
110
|
+
This is deliberately **not** the `context` tool from `createAgentTools`. That one returns
|
|
111
|
+
`getInteractiveARIA()` — `compactAriaSnapshot` over `ActionResult.ariaSnapshot`, captured without
|
|
112
|
+
`mode: 'ai'`, so it carries **no refs** — plus a 6k-char HTML dump beside it. Offering it here
|
|
113
|
+
would hand the model a ref-less tree and push it straight back to guessed locators, undoing
|
|
114
|
+
tier 2.
|
|
115
|
+
|
|
116
|
+
### Tier 4 — vision
|
|
117
|
+
|
|
118
|
+
`visualClick` (`src/ai/tools.ts:791`), already implemented. Click only — a coordinate is not a
|
|
119
|
+
handle, so it cannot serve fills, selects, or assertions.
|
|
120
|
+
|
|
121
|
+
### 1.1 Acting on a ref — `clickRef` / `hoverRef` in `boat/prima/src/tools.ts`
|
|
122
|
+
|
|
123
|
+
CodeceptJS has no `aria-ref` locator, so a ref cannot be handed to `I.click()` directly. It is
|
|
124
|
+
resolved to a locator CodeceptJS does understand, using machinery that already exists:
|
|
125
|
+
|
|
126
|
+
```
|
|
127
|
+
page.locator(`aria-ref=${ref}`)
|
|
128
|
+
→ WebElement.fromPlaywrightLocator(...) (web-element.ts:112)
|
|
129
|
+
→ I.click(<xpath>)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
**Which XPath, and the one that is not available.** `WebElement` declares two — `clickXPath`,
|
|
133
|
+
built attribute-first by `buildClickableXPath` (`utils/xpath.ts`), and `xpath`, the absolute
|
|
134
|
+
positional path. On this path only `clickXPath` exists: `fromPlaywrightLocator` builds through
|
|
135
|
+
`fromRawData`, which hardcodes `xpath: ''` (`web-element.ts:89`). Only `fromXPathMatch`, the
|
|
136
|
+
static-HTML path, populates the absolute form. So there is no positional fallback to reach for,
|
|
137
|
+
and that is the right outcome anyway — an absolute path is deduplication machinery, fragile the
|
|
138
|
+
moment the DOM shifts, and the ref resolution and the click happen on separate round-trips.
|
|
139
|
+
|
|
140
|
+
Verified live, ref → `clickXPath` → match count:
|
|
141
|
+
|
|
142
|
+
```
|
|
143
|
+
e15 link "Checking the proxy…" //*[self::a and contains(.,"Checking the proxy and the firewall")] → 1
|
|
144
|
+
e19 button "Reload" //*[@id="reload-button"] → 1
|
|
145
|
+
e20 button "Details" //*[@id="details-button"] → 1
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
**Require a unique match.** Because there is no fallback, the resolved `clickXPath` is checked
|
|
149
|
+
to match exactly one element before it is used. Zero or many is a failure, reported as such —
|
|
150
|
+
never a click on an ambiguous match and never a retreat to guessed locators.
|
|
151
|
+
|
|
152
|
+
Ref acting lives in **separate tools in the boat** — `clickRef` and `hoverRef`, from a
|
|
153
|
+
`createRefTools` factory in `boat/prima/src/tools.ts`. `click` and `hover` in `src/ai/tools.ts`
|
|
154
|
+
are left byte-identical. A prima-only tool belongs to prima; core keeps only what every caller
|
|
155
|
+
uses, and lends the boat its result-shaping helpers (`successToolResult`, `failedToolResult`,
|
|
156
|
+
`commitNote`) rather than having them copied.
|
|
157
|
+
|
|
158
|
+
Adding an optional `ref` to the existing tools looks cheaper and is wrong. A tool's schema and
|
|
159
|
+
description are shared with every caller, and Tester never receives ref-bearing snapshots, so it
|
|
160
|
+
would be shown a parameter it can only fill by inventing one. Making `commands` optional to
|
|
161
|
+
accommodate the new field weakens the contract for Tester too. "Prima enables ref mode, Tester is
|
|
162
|
+
untouched" is not achievable through one definition: there is one definition, and Tester sees it.
|
|
163
|
+
|
|
164
|
+
Each ref tool takes a ref and nothing else, resolves it, executes one command, and reports that
|
|
165
|
+
command as `used:` — real CodeceptJS a generated test can keep. A ref that does not resolve is a
|
|
166
|
+
failure, not a cue to fall back to a guessed locator: it means the context is stale.
|
|
167
|
+
|
|
168
|
+
`hoverRef` exists for the same reason as `hover` — revealing hover-only controls is a
|
|
169
|
+
prerequisite for clicking them.
|
|
170
|
+
|
|
171
|
+
Prima keeps the locator tools alongside the ref tools: tier 3 works from markup that carries no
|
|
172
|
+
refs, and needs them. Tier 4 keeps the coordinate input.
|
|
173
|
+
|
|
174
|
+
---
|
|
175
|
+
|
|
176
|
+
## 2. Heal is deleted
|
|
177
|
+
|
|
178
|
+
`heal()` (`prima.ts:428-460`) is removed, along with `--no-heal`, `PrimaOptions.heal`, the
|
|
179
|
+
`healed:` / `healNote` envelope fields, the `HealAttempt` type, and the
|
|
180
|
+
`### Healing attempts` section. The three call sites — `pw` (`:79`), `do` (`:135`), `go` (`:211`)
|
|
181
|
+
— go straight to `failureEnvelope`.
|
|
182
|
+
|
|
183
|
+
The failure envelope already does the right thing and becomes the only failure path: `ok: false`,
|
|
184
|
+
the exact error with its call log, and compact ARIA inline. Measured at 8.6s in the field run
|
|
185
|
+
against 34.8s for the heal path that got the answer wrong.
|
|
186
|
+
|
|
187
|
+
This removes the substitution defect at the root. No remaining code path can select an element
|
|
188
|
+
other than the one asked for, so no envelope can report success for an action the caller did not
|
|
189
|
+
request. Routine obstructions — an overlay covering a button, an element not yet visible — now
|
|
190
|
+
return the failure envelope, and the orchestrating model decides. That is the accepted cost: the
|
|
191
|
+
compact ARIA block is the one place page data deliberately enters the expensive context.
|
|
192
|
+
|
|
193
|
+
---
|
|
194
|
+
|
|
195
|
+
## 3. Proof-carrying envelopes
|
|
196
|
+
|
|
197
|
+
Two defects with one cause. `renderOutcome` (`envelope.ts:76`) returns the **first** of
|
|
198
|
+
changes / answer / research / verdict, so `### Changes` is structurally impossible alongside
|
|
199
|
+
`### Verdict`, `### Answer`, or `### Research`. And `pageChanges` returns `ariaChanges ?? null`,
|
|
200
|
+
which renders nothing when the diff is empty or `previousState` is null.
|
|
201
|
+
|
|
202
|
+
- `### Changes` renders on every action envelope, showing `no change` explicitly when the tree is
|
|
203
|
+
identical. A caller can then tell "nothing happened" from "prima did not say".
|
|
204
|
+
- `renderOutcome` stops being mutually exclusive: `### Changes` renders alongside the command's
|
|
205
|
+
own outcome section.
|
|
206
|
+
- Prima captures an explicit before-snapshot rather than relying on whatever `stateManager`
|
|
207
|
+
holds at process start.
|
|
208
|
+
- New `### Steps` block for `do`: one line per instruction, each naming the ariaDiff entry that
|
|
209
|
+
proves it, or marked `unproven`.
|
|
210
|
+
|
|
211
|
+
### Refs must never reach the diff or hash pipeline
|
|
212
|
+
|
|
213
|
+
The ref-bearing snapshot is a **context artifact only**. Refs are stripped before hashing and
|
|
214
|
+
diffing. Storing the `mode: 'ai'` output in `ActionResult.ariaSnapshot` would poison every
|
|
215
|
+
`### Changes` block, because Playwright renumbers refs on each call. Measured on an identical
|
|
216
|
+
page whose refs merely shifted:
|
|
217
|
+
|
|
218
|
+
```
|
|
219
|
+
ref churn only, page identical → diff count = 6
|
|
220
|
+
added: button "Cancel" [ref=e21], button "Save" [ref=e20], textbox "Name" [ref=e22]
|
|
221
|
+
removed: button "Cancel" [ref=e11], button "Save" [ref=e10], textbox "Name" [ref=e12]
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
Six phantom entries for a page that did not change. Capture ref-bearing and ref-free variants,
|
|
225
|
+
and feed only the ref-free one to `diffAriaSnapshots` and `getStateHash`.
|
|
226
|
+
|
|
227
|
+
---
|
|
228
|
+
|
|
229
|
+
## 4. Executor prompt
|
|
230
|
+
|
|
231
|
+
`instructionSystemPrompt` (`prima.ts:493`) already says to stop when every instruction is done,
|
|
232
|
+
but nothing tracks per-instruction completion, so the loop runs until the model stops calling
|
|
233
|
+
tools. Two rules are added, stated as general principles rather than as counter-examples from
|
|
234
|
+
any debugging session:
|
|
235
|
+
|
|
236
|
+
- Act only on the listed instructions. An adjacent action that appears helpful is out of scope;
|
|
237
|
+
report it as an observation instead of performing it.
|
|
238
|
+
- For each instruction, cite the observed page change that proves it. An instruction that cannot
|
|
239
|
+
be tied to an observed change is reported unproven rather than claimed as done.
|
|
240
|
+
|
|
241
|
+
The prompt also states the ref contract: act on refs from the current context injection; refs
|
|
242
|
+
from an earlier injection are dead.
|
|
243
|
+
|
|
244
|
+
---
|
|
245
|
+
|
|
246
|
+
## 5. Attachment and discovery
|
|
247
|
+
|
|
248
|
+
Three separate faults, all confirmed empirically.
|
|
249
|
+
|
|
250
|
+
**Connect with the daemon's own build when attached.** `connectDescriptor` (`prima.ts:374`)
|
|
251
|
+
tries prima's own playwright first and only falls back to `descriptor.playwrightLib` if connect
|
|
252
|
+
fails. Connect *succeeds* across builds, so the fallback is never reached — and then the tier-2
|
|
253
|
+
snapshot breaks:
|
|
254
|
+
|
|
255
|
+
| client lib | `connect()` | `ariaSnapshot({mode:'ai'})` |
|
|
256
|
+
|---|---|---|
|
|
257
|
+
| own playwright 1.62.1 | ok | `locator.ariaSnapshot: timeout: expected float, got undefined` |
|
|
258
|
+
| daemon playwright-core 1.62.0-alpha | ok | ok, 3376 bytes, `aria-ref=e1` → 1 match |
|
|
259
|
+
|
|
260
|
+
In attached mode, prefer `descriptor.playwrightLib` when present and fall back to our own. Carry
|
|
261
|
+
`playwrightLib` through the `--endpoint` path too, which currently hardcodes `''`.
|
|
262
|
+
|
|
263
|
+
**Stop keying discovery on `workspaceDir`.** No `@playwright/cli` release writes that field —
|
|
264
|
+
verified on 0.1.13 and 0.1.17. `parseDescriptor` (`pw-registry.ts:56`) requires it and
|
|
265
|
+
`selectDescriptor` filters on it, so every descriptor is dropped and discovery finds nothing no
|
|
266
|
+
matter what is installed. Resolution becomes: `--endpoint` → `--pw-session <title>` →
|
|
267
|
+
`PLAYWRIGHT_CLI_SESSION` → live descriptor titled `default` → the single live descriptor →
|
|
268
|
+
tool error listing candidate titles. Liveness-probe before selecting.
|
|
269
|
+
|
|
270
|
+
**Bump the playwright pin to `^1.62`.** `playwright@^1.60` could not `connect()` to a 1.61/1.62
|
|
271
|
+
browser server at all; 1.62.1 connects. The pin is the reason the original review concluded
|
|
272
|
+
attach was broken outright.
|
|
273
|
+
|
|
274
|
+
Consequences: `browser list` shows attachable sessions rather than reporting none while one is
|
|
275
|
+
live, and the no-browser error stops advising `playwright-cli open <url>` to someone who already
|
|
276
|
+
ran it.
|
|
277
|
+
|
|
278
|
+
---
|
|
279
|
+
|
|
280
|
+
## 6. `verify` honesty
|
|
281
|
+
|
|
282
|
+
`rules/navigator/verification-actions.md` offers nine assertions — `see`, `seeElement`,
|
|
283
|
+
`seeInField`, `seeInTitle`, `seeInSource` and their `dontSee*` counterparts — none of which
|
|
284
|
+
express enabled, disabled, checked, selected, or expanded. A claim about interactive state is
|
|
285
|
+
therefore unprovable by construction, and the field run reported a working feature as failing:
|
|
286
|
+
|
|
287
|
+
```
|
|
288
|
+
verify "the Save button in the skill editor is now enabled ..."
|
|
289
|
+
→ passed: false, code: (empty), evidence: no assertion held on the current page
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
The app was correct — `● unsaved` was displayed and the button was enabled.
|
|
293
|
+
|
|
294
|
+
- Add state assertions to the rule.
|
|
295
|
+
- `verify` distinguishes **assertion failed** from **could not express this assertion**. The
|
|
296
|
+
second is not a test failure and must not be reported as one.
|
|
297
|
+
- With §7's compaction fix, `verify` reads state from the same ref-bearing snapshot the tier-2
|
|
298
|
+
ladder produces.
|
|
299
|
+
|
|
300
|
+
---
|
|
301
|
+
|
|
302
|
+
## 7. Prerequisite fix — ARIA compaction drops refs and state
|
|
303
|
+
|
|
304
|
+
`compactAriaSnapshot` keeps only the first bracket group on a line. Playwright emits state
|
|
305
|
+
attributes before `[ref=]`, so every stateful control loses its handle. Measured:
|
|
306
|
+
|
|
307
|
+
```
|
|
308
|
+
button "Plain" [ref=e10] → ref KEPT
|
|
309
|
+
button "Active" [active] [ref=e13] → ref LOST
|
|
310
|
+
button "Disabled" [disabled] [ref=e14] → ref LOST
|
|
311
|
+
button "Pressed" [pressed] [ref=e15] → ref LOST
|
|
312
|
+
button "Expanded" [expanded] [ref=e16] → ref LOST
|
|
313
|
+
checkbox "Checked" [checked] [ref=e17] → ref LOST
|
|
314
|
+
button "Cursor" [ref=e18] [cursor=pointer] → ref KEPT
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
The controls most worth acting on and asserting about are exactly the ones stripped of their
|
|
318
|
+
handle. **Keep every bracket group on a line.** No allow-list, no drop-list — parsing one group
|
|
319
|
+
and discarding the rest is the whole bug, and any rule about which groups survive re-creates it
|
|
320
|
+
the next time Playwright adds an attribute. This fix is a prerequisite for §1 tier 2 and enables
|
|
321
|
+
§6.
|
|
322
|
+
|
|
323
|
+
---
|
|
324
|
+
|
|
325
|
+
## 8. Envelope hygiene
|
|
326
|
+
|
|
327
|
+
- `used:` carries the winning line only — no concatenated ladder attempts, no comment lines. The
|
|
328
|
+
field run produced `I.click(".sidebar button:has-text("Workflows")")`, which is not valid
|
|
329
|
+
JavaScript, and a `// 1. Open dialog...` comment inside the code block.
|
|
330
|
+
- `click` and `fill` stop labelling themselves `do` in `command:`.
|
|
331
|
+
- `network.jsonl` is written and advertised only when requests were actually captured. It was
|
|
332
|
+
0 bytes across all 18 runs while being advertised in every envelope; an artifact line that
|
|
333
|
+
points at an empty file is worse than no line.
|
|
334
|
+
- Research annotation overlays are removed from the DOM after use. The browser is shared with
|
|
335
|
+
playwright-cli, and leftover `Legend` / `e8` / `e10` nodes appeared in its next snapshot.
|
|
336
|
+
- Commands stop requiring a URL when attached to a browser already on a page.
|
|
337
|
+
- A redirect that preserves origin and path counts as navigation success. The app's
|
|
338
|
+
`/` → `/?session=<uuid>&ws=1` redirect cost 3m00s and eight attempts under the old heal path,
|
|
339
|
+
and a hard tool failure against a prima-owned browser — in an envelope whose own inlined ARIA
|
|
340
|
+
proved the page had loaded.
|
|
341
|
+
|
|
342
|
+
---
|
|
343
|
+
|
|
344
|
+
## Testing
|
|
345
|
+
|
|
346
|
+
- **Unit** — envelope rendering with `### Changes` always present, including the `no change`
|
|
347
|
+
form and the combination with `### Verdict` / `### Answer` / `### Research`; compaction
|
|
348
|
+
preserving every bracket group across the §7 combinations and any order of them; ref-free
|
|
349
|
+
diffing (the six-phantom case must yield zero); descriptor selection without `workspaceDir`,
|
|
350
|
+
including the ambiguous multi-session error.
|
|
351
|
+
- **Ref resolution** — a ref resolves to an attribute-based `clickXPath` matching exactly one
|
|
352
|
+
element, and still matching after unrelated siblings are added or removed; a ref that resolves
|
|
353
|
+
to zero or many fails rather than falling back to a guessed locator.
|
|
354
|
+
- **Integration** — `do` prompt behaviour through the existing `@copilotkit/aimock` harness per
|
|
355
|
+
`docs/contributing/ai-integration-tests.md`: instructions performed and nothing beyond them,
|
|
356
|
+
per-instruction proof citation, unproven reporting. Fictional fixture data only.
|
|
357
|
+
- **End-to-end** — against a local fixture: a ref named from the snapshot clicks the element it
|
|
358
|
+
names and no other; a failing `pw` returns `ok: false` with compact ARIA and never a
|
|
359
|
+
substituted action; attach to a live playwright-cli session with no descriptor editing; a
|
|
360
|
+
session-param redirect resolves as success.
|
|
361
|
+
|
|
362
|
+
---
|
|
363
|
+
|
|
364
|
+
## Decisions Log
|
|
365
|
+
|
|
366
|
+
- Heal is deleted outright rather than constrained to same-target recovery. No flag, no opt-in.
|
|
367
|
+
- `do` perception is a four-tier ladder: research map, ARIA with refs, compact HTML, vision for
|
|
368
|
+
click.
|
|
369
|
+
- Refs come from Playwright's native `aria-ref=` engine. No eidx attribute stamping, so the
|
|
370
|
+
shared browser's DOM is not mutated.
|
|
371
|
+
- Ref acting lives in `boat/prima/src/tools.ts`, not in core; `click` and `hover` are untouched.
|
|
372
|
+
Extending a shared tool would show Tester a `ref` parameter it can only fill by inventing one,
|
|
373
|
+
and would weaken `commands` for every caller.
|
|
374
|
+
- A ref resolves through `WebElement` to an attribute-based `clickXPath`. The absolute positional
|
|
375
|
+
`xpath` is not a fallback — `fromRawData` never populates it on the live-locator path, and it
|
|
376
|
+
would be the fragile choice regardless.
|
|
377
|
+
- A resolved `clickXPath` must match exactly one element. A dead or ambiguous ref is a failure,
|
|
378
|
+
never a fallback to guessed locators.
|
|
379
|
+
- ARIA compaction keeps every bracket group. No allow-list of attributes to preserve.
|
|
380
|
+
- Refs are a context artifact only, stripped before hashing and diffing.
|
|
381
|
+
- Ref lifetime is one context injection; the loop re-snapshots on state-hash change.
|
|
382
|
+
- Tier 1 reads maps via `getPreviousResearch`; the trigger is StateManager visit count.
|
|
383
|
+
- Discovery matches on title plus liveness; `workspaceDir` is abandoned as a key.
|
|
384
|
+
- Attached mode connects with `descriptor.playwrightLib` first, because connect succeeds across
|
|
385
|
+
builds but `ariaSnapshot` does not.
|
|
386
|
+
- The playwright pin moves to `^1.62`.
|
|
387
|
+
- `verify` reports inexpressible assertions as inexpressible, never as failures, and does not
|
|
388
|
+
record them as verifications — a claim that could not be checked must not be remembered as
|
|
389
|
+
one that failed.
|
|
390
|
+
- `do` carries a prima-shaped `context` tool that descends the ladder — fresh refs first, capped
|
|
391
|
+
markup on a repeat call. The shared `context` from `createAgentTools` is not reused: it returns
|
|
392
|
+
a ref-less tree and an HTML dump.
|
|
393
|
+
- The baseline snapshot is captured lazily when a command needs a diff, not in `start()`, which
|
|
394
|
+
runs before a page is loaded.
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# AI Agents
|
|
2
|
+
|
|
3
|
+
Explorbot splits the testing workflow across specialized AI agents. Each agent handles one part of the work, which keeps it focused and keeps token costs down.
|
|
4
|
+
|
|
5
|
+
## Agent Overview
|
|
6
|
+
|
|
7
|
+
```mermaid
|
|
8
|
+
flowchart LR
|
|
9
|
+
A[Navigator] --> B[Researcher] --> C[Planner] --> D[Tester]
|
|
10
|
+
A -- "goes to page" --> B
|
|
11
|
+
B -- "analyzes UI" --> C
|
|
12
|
+
C -- "suggests tests" --> D
|
|
13
|
+
D -- "runs tests" --> A
|
|
14
|
+
E[Pilot] -.->|supervises| D
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Navigator Agent
|
|
18
|
+
|
|
19
|
+
Handles browser interactions: clicks, form fills, and navigation.
|
|
20
|
+
|
|
21
|
+
The Navigator runs CodeceptJS commands in the browser. When a selector fails, it tries other locator strategies and resolves the interaction without stopping the run. It remembers what worked and what didn't, so failed selectors don't keep killing your runs and tests survive UI changes.
|
|
22
|
+
|
|
23
|
+
Commands that use Navigator:
|
|
24
|
+
- `/navigate <target>`
|
|
25
|
+
- `I.click()`, `I.fillField()`, `I.amOnPage()`, etc.
|
|
26
|
+
|
|
27
|
+
## Researcher Agent
|
|
28
|
+
|
|
29
|
+
Analyzes pages to find what's actually on them.
|
|
30
|
+
|
|
31
|
+
The Researcher discovers every interactive element, including content hidden in accordions, dropdowns, and modals. It maps navigation paths and form structures, extracts data from tables and lists, and filters out noise like cookie banners and ads. The result is a complete map of what you can test, with form validation rules documented.
|
|
32
|
+
|
|
33
|
+
Commands that use Researcher:
|
|
34
|
+
- `npx explorbot research /path` (CLI)
|
|
35
|
+
- `/research [path]` (TUI)
|
|
36
|
+
- `/research --deep` — expand hidden elements
|
|
37
|
+
- `/research --data` — extract page data
|
|
38
|
+
|
|
39
|
+
Explicit research always captures a screenshot; a configured vision model analyzes it.
|
|
40
|
+
|
|
41
|
+
See [Researcher Agent](./researcher.md) for configuration and usage.
|
|
42
|
+
|
|
43
|
+
## Planner Agent
|
|
44
|
+
|
|
45
|
+
Generates test scenarios from research findings.
|
|
46
|
+
|
|
47
|
+
The Planner writes business-focused scenarios with priority levels (critical/important/high/normal/low) and expected outcomes for verification. It balances positive and negative cases, skips scenarios you already have, and cycles through planning styles (normal, psycho, curious) to broaden coverage across iterations. You can add your own styles and page-specific rules.
|
|
48
|
+
|
|
49
|
+
Commands that use Planner:
|
|
50
|
+
- `/plan [--focus <feature>]`
|
|
51
|
+
- `/explore`
|
|
52
|
+
|
|
53
|
+
See [Planner Agent](./planner.md) for planning styles, customization, and configuration.
|
|
54
|
+
|
|
55
|
+
## Tester Agent
|
|
56
|
+
|
|
57
|
+
Runs the planned scenarios.
|
|
58
|
+
|
|
59
|
+
The Tester executes scenarios step by step and adapts when something goes wrong. It tracks state changes during execution, records actual results against expected ones, and uses research context to make decisions. It handles unexpected modals and popups, recovers from minor failures on its own, and produces detailed execution logs.
|
|
60
|
+
|
|
61
|
+
Commands that use Tester:
|
|
62
|
+
- `/test [scenario]`
|
|
63
|
+
- `/explore`
|
|
64
|
+
|
|
65
|
+
## Pilot Agent
|
|
66
|
+
|
|
67
|
+
Supervises the Tester and steps in when a test gets stuck.
|
|
68
|
+
|
|
69
|
+
The Pilot keeps a separate conversation to track progress over time. It detects stuck patterns — loops, repeated failures, no page changes — and decides what context the Tester needs next (HTML, ARIA, UI map). When automated recovery fails, it asks the user for help instead of giving up.
|
|
70
|
+
|
|
71
|
+
Because the Pilot sees only tool summaries rather than raw HTML, you can run it on a smarter model without a token cost explosion.
|
|
72
|
+
|
|
73
|
+
The Pilot intervenes when:
|
|
74
|
+
- Actions succeed but the page doesn't change (wrong element)
|
|
75
|
+
- The same action repeats several times (loop)
|
|
76
|
+
- The same locator keeps failing (need a different approach)
|
|
77
|
+
- Only research or context calls run, with no action tools (no progress)
|
|
78
|
+
|
|
79
|
+
## Analyst Agent
|
|
80
|
+
|
|
81
|
+
Produces a human-readable session report after `/explore` and `/freesail` runs.
|
|
82
|
+
|
|
83
|
+
The Analyst reads every test in the session — scenario, expected outcome, final result, notes, and step log. It clusters findings by root cause: three tests that fail on the same control become one defect with three test references, not three rows. It separates product defects from automation problems, so an element the test could not click is not reported as a bug.
|
|
84
|
+
|
|
85
|
+
The report is plain markdown: a one-to-two sentence headline about the feature state (no test counts), then `## Coverage`, `## What works`, `## Defects` (each with a `[High]`/`[Medium]`/`[Low]` severity tag, reproduce steps, and one-line evidence), `## UX issues`, and `## Execution Issues`. Empty sections are omitted.
|
|
86
|
+
|
|
87
|
+
It runs automatically at the end of `/explore` (per-run) and on app exit (session-wide, across multiple `/explore` or `/freesail` runs). The same markdown goes to the console, to `output/reports/<mode>-<sessionName>.md`, and — when the reporter is enabled — to the Testomat.io run description.
|
|
88
|
+
|
|
89
|
+
Configuration:
|
|
90
|
+
|
|
91
|
+
```javascript
|
|
92
|
+
export default {
|
|
93
|
+
ai: {
|
|
94
|
+
agents: {
|
|
95
|
+
analyst: {
|
|
96
|
+
// model: openai('gpt-4o'), // override the model
|
|
97
|
+
// systemPrompt: 'Focus on...', // append guidance to the prompt
|
|
98
|
+
// enabled: false, // disable the analyst entirely
|
|
99
|
+
},
|
|
100
|
+
},
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
The Analyst resolves its model like other agentic tasks: the agent-specific `model`, then `ai.agenticModel`, then `ai.model`.
|
|
106
|
+
|
|
107
|
+
## Captain Agent
|
|
108
|
+
|
|
109
|
+
Handles your direct requests in the TUI and recovers the session when something breaks.
|
|
110
|
+
|
|
111
|
+
The Captain steps in when a slash command isn't enough — answering questions about your setup, inspecting tests and page states, and reading recent output before it replies. It works in four modes:
|
|
112
|
+
|
|
113
|
+
- **idle** — plan management, project inspection, knowledge and experience files; available before any page loads
|
|
114
|
+
- **web** — page interaction, navigation, and browser diagnostics
|
|
115
|
+
- **test** — test timeline, state inspection, generated code and logs
|
|
116
|
+
- **heal** — browser and test recovery when an active test loses its page or browser
|
|
117
|
+
|
|
118
|
+
When a test hits a fatal browser error, the Captain tries to recover — reload, restart the browser, open a fresh tab, or close extra tabs — before the test is stopped, then tells the Tester how to continue. It runs on explicit TUI requests and on test interrupts (stop, pass, skip, or redirect a running test).
|
|
119
|
+
|
|
120
|
+
## Per-Agent Model Configuration
|
|
121
|
+
|
|
122
|
+
Use different models for different agents to control cost:
|
|
123
|
+
|
|
124
|
+
```javascript
|
|
125
|
+
export default {
|
|
126
|
+
ai: {
|
|
127
|
+
model: groq('openai/gpt-oss-20b'),
|
|
128
|
+
visionModel: groq('meta-llama/llama-4-scout-17b-16e-instruct'),
|
|
129
|
+
agents: {
|
|
130
|
+
navigator: { model: groq('openai/gpt-oss-20b') },
|
|
131
|
+
researcher: { model: groq('openai/gpt-oss-20b') },
|
|
132
|
+
planner: { model: groq('openai/gpt-oss-20b') },
|
|
133
|
+
tester: { model: groq('openai/gpt-oss-20b'), progressCheckInterval: 5 },
|
|
134
|
+
pilot: { stepsToReview: 5 },
|
|
135
|
+
},
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Typical choices:
|
|
141
|
+
- Navigator needs fast responses for real-time interaction
|
|
142
|
+
- Researcher benefits from vision
|
|
143
|
+
- Planner can use a larger model for better test design
|
|
144
|
+
- Tester needs tool use for execution
|
|
145
|
+
- Pilot can use a smarter model — it only processes tool summaries, not HTML or ARIA
|
|
146
|
+
|
|
147
|
+
## How Agents Communicate
|
|
148
|
+
|
|
149
|
+
Agents share context through four channels:
|
|
150
|
+
|
|
151
|
+
1. **State Manager** — tracks the current page, URL, and navigation history
|
|
152
|
+
2. **Research Results** — structured page analysis available to Planner and Tester
|
|
153
|
+
3. **Experience Files** — patterns learned across sessions. Injected as a compact table of contents (file tags plus section headings) rather than full bodies; agents pull individual sections on demand via the `learnExperience` tool
|
|
154
|
+
4. **Knowledge Files** — domain knowledge you provide
|
|
155
|
+
|
|
156
|
+
Each agent keeps its context minimal to hold costs down. Agents request specific information when they need it instead of carrying full conversation history.
|
|
157
|
+
|
|
158
|
+
The Pilot keeps a separate conversation from the Tester. The Tester's conversation carries heavy HTML and ARIA context; the Pilot sees only tool execution summaries (what succeeded, what failed, what changed). That lets the Pilot run on expensive models without a token cost explosion.
|