explorbot 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/explorbot-cli.ts +7 -1
- package/boat/api-tester/src/config.ts +4 -2
- package/boat/doc-collector/bin/doc-collector-cli.ts +2 -0
- package/boat/doc-collector/src/ai/documentarian.ts +61 -31
- package/boat/doc-collector/src/config.ts +4 -2
- package/boat/prima/bin/prima-cli.ts +0 -0
- package/boat/prima/src/activity-line.ts +33 -0
- package/boat/prima/src/cli.ts +115 -86
- package/boat/prima/src/envelope.ts +80 -52
- package/boat/prima/src/prima.ts +547 -128
- package/boat/prima/src/pw-parser.ts +11 -1
- package/boat/prima/src/pw-registry.ts +4 -5
- package/boat/prima/src/session-log.ts +126 -0
- package/dist/bin/explorbot-cli.js +7 -1
- package/dist/boat/api-tester/bin/apibot-cli.js +2 -0
- package/dist/boat/api-tester/src/config.js +4 -2
- package/dist/boat/doc-collector/bin/doc-collector-cli.js +2 -0
- package/dist/boat/doc-collector/src/ai/documentarian.js +44 -19
- package/dist/boat/doc-collector/src/config.js +4 -2
- package/dist/boat/prima/src/activity-line.js +30 -0
- package/dist/boat/prima/src/cli.js +97 -77
- package/dist/boat/prima/src/envelope.js +70 -44
- package/dist/boat/prima/src/prima.js +515 -119
- package/dist/boat/prima/src/pw-parser.js +13 -1
- package/dist/boat/prima/src/pw-registry.js +4 -5
- package/dist/boat/prima/src/session-log.js +108 -0
- package/dist/package.json +3 -2
- package/dist/rules/navigator/verification-actions.md +20 -0
- package/dist/src/action-result.d.ts +7 -0
- package/dist/src/action-result.js +4 -0
- package/dist/src/action.d.ts +2 -0
- package/dist/src/action.js +36 -1
- package/dist/src/ai/captain/web-mode.js +6 -3
- package/dist/src/ai/captain.js +2 -0
- package/dist/src/ai/navigator.d.ts +7 -0
- package/dist/src/ai/navigator.js +10 -6
- package/dist/src/ai/pilot.d.ts +4 -0
- package/dist/src/ai/pilot.js +48 -2
- package/dist/src/ai/provider.js +2 -2
- package/dist/src/ai/rerunner.js +2 -1
- package/dist/src/ai/researcher.js +1 -1
- package/dist/src/ai/rules.js +17 -10
- package/dist/src/ai/task-agent.js +4 -1
- package/dist/src/ai/tester.d.ts +1 -1
- package/dist/src/ai/tester.js +33 -33
- package/dist/src/ai/tools.d.ts +14 -0
- package/dist/src/ai/tools.js +113 -36
- package/dist/src/config.d.ts +1 -0
- package/dist/src/config.js +13 -4
- package/dist/src/execution-controller.d.ts +2 -0
- package/dist/src/execution-controller.js +6 -0
- package/dist/src/explorbot.d.ts +2 -1
- package/dist/src/explorbot.js +3 -1
- package/dist/src/explorer.js +2 -3
- package/dist/src/playwright-recorder.js +30 -0
- package/dist/src/remote.d.ts +54 -0
- package/dist/src/remote.js +229 -0
- package/dist/src/reporter.d.ts +1 -0
- package/dist/src/reporter.js +7 -1
- package/dist/src/state-manager.d.ts +2 -1
- package/dist/src/stats.d.ts +1 -0
- package/dist/src/stats.js +1 -0
- package/dist/src/test-plan.d.ts +2 -0
- package/dist/src/test-plan.js +7 -0
- package/dist/src/utils/aria.d.ts +2 -8
- package/dist/src/utils/aria.js +69 -40
- package/dist/src/utils/html.js +1 -0
- package/dist/src/utils/logger.d.ts +6 -0
- package/dist/src/utils/logger.js +24 -0
- package/dist/src/utils/page-readiness.js +18 -1
- package/dist/src/utils/url-matcher.js +3 -0
- package/dist/src/utils/web-element.d.ts +2 -0
- package/dist/src/utils/web-element.js +8 -0
- package/dist/src/utils/web-sandbox.d.ts +1 -1
- package/dist/src/utils/web-sandbox.js +2 -3
- package/docs/api-testing/basics.md +90 -0
- package/docs/api-testing/planning.md +57 -0
- package/docs/api-testing/running-tests.md +55 -0
- package/docs/assets/cloud-report.png +0 -0
- package/docs/assets/html-report.png +0 -0
- package/docs/assets/langfuse-trace.png +0 -0
- package/docs/assets/successful-explore-run.png +0 -0
- package/docs/basics/getting-started.md +140 -0
- package/docs/basics/prerequisites.md +63 -0
- package/docs/basics/providers.md +362 -0
- package/docs/basics/running.md +78 -0
- package/docs/contributing/ai-integration-tests.md +57 -0
- package/docs/contributing/contributing.md +90 -0
- package/docs/contributing/demo-videos.md +36 -0
- package/docs/contributing/npm-package.md +138 -0
- package/docs/contributing/observability.md +227 -0
- package/docs/contributing/regression-tests.md +103 -0
- package/docs/contributing/testing.md +95 -0
- package/docs/doc-collection/basics.md +128 -0
- package/docs/doc-collection/crawling.md +67 -0
- package/docs/doc-collection/interactive-mode.md +99 -0
- package/docs/index.json +86 -0
- package/docs/reference/commands.md +994 -0
- package/docs/reference/configuration.md +569 -0
- package/docs/reference/scripting.md +303 -0
- package/docs/superpowers/plans/2026-08-01-actor-boat.md +925 -0
- package/docs/superpowers/plans/2026-08-01-prima-boat.md +1120 -0
- package/docs/superpowers/reviews/2026-08-06-prima-vs-playwright-cli.md +268 -0
- package/docs/superpowers/specs/2026-08-01-actor-boat-design.md +204 -0
- package/docs/superpowers/specs/2026-08-01-prima-boat-design.md +242 -0
- package/docs/superpowers/specs/2026-08-03-global-config-design.md +138 -0
- package/docs/superpowers/specs/2026-08-07-prima-fixes-design.md +394 -0
- package/docs/web-testing/agents.md +158 -0
- package/docs/web-testing/automated-tests.md +134 -0
- package/docs/web-testing/basics.md +91 -0
- package/docs/web-testing/customization.md +131 -0
- package/docs/web-testing/hooks.md +238 -0
- package/docs/web-testing/page-interaction.md +84 -0
- package/docs/web-testing/planner.md +122 -0
- package/docs/web-testing/rerun.md +164 -0
- package/docs/web-testing/researcher.md +380 -0
- package/docs/workflow/agentic-usage.md +233 -0
- package/docs/workflow/application-spec.md +73 -0
- package/docs/workflow/ci.md +202 -0
- package/docs/workflow/knowledge.md +310 -0
- package/docs/workflow/planning-styles.md +67 -0
- package/docs/workflow/reporting.md +133 -0
- package/docs/workflow/test-plans.md +90 -0
- package/package.json +3 -2
- package/rules/navigator/verification-actions.md +20 -0
- package/src/action-result.ts +11 -0
- package/src/action.ts +38 -2
- package/src/ai/captain/web-mode.ts +6 -3
- package/src/ai/captain.ts +3 -0
- package/src/ai/navigator.ts +14 -8
- package/src/ai/pilot.ts +53 -2
- package/src/ai/provider.ts +2 -4
- package/src/ai/rerunner.ts +2 -1
- package/src/ai/researcher.ts +1 -1
- package/src/ai/rules.ts +17 -10
- package/src/ai/task-agent.ts +3 -1
- package/src/ai/tester.ts +32 -33
- package/src/ai/tools.ts +132 -36
- package/src/config.ts +15 -4
- package/src/execution-controller.ts +8 -0
- package/src/explorbot.ts +4 -2
- package/src/explorer.ts +2 -2
- package/src/playwright-recorder.ts +23 -0
- package/src/remote.ts +238 -0
- package/src/reporter.ts +7 -1
- package/src/state-manager.ts +2 -1
- package/src/stats.ts +1 -0
- package/src/test-plan.ts +9 -0
- package/src/utils/aria.ts +65 -45
- package/src/utils/html.ts +1 -0
- package/src/utils/logger.ts +24 -1
- package/src/utils/page-readiness.ts +24 -1
- package/src/utils/url-matcher.ts +3 -0
- package/src/utils/web-element.ts +9 -0
- package/src/utils/web-sandbox.ts +3 -4
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
# Knowledge System
|
|
2
|
+
|
|
3
|
+
Knowledge files tell explorbot facts about your app. Agents read them to make better decisions about authentication, special workflows, and app-specific behavior.
|
|
4
|
+
|
|
5
|
+
## Adding Knowledge
|
|
6
|
+
|
|
7
|
+
### Interactive Mode
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npx explorbot learn
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Opens a TUI form where you can:
|
|
14
|
+
- Enter a URL pattern
|
|
15
|
+
- See existing knowledge for that URL
|
|
16
|
+
- Add new knowledge
|
|
17
|
+
|
|
18
|
+
### CLI Mode
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
npx explorbot learn "<url-pattern>" "<description>"
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Examples:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
# Login credentials
|
|
28
|
+
npx explorbot learn "/login" "Use credentials: admin@example.com / secret123"
|
|
29
|
+
|
|
30
|
+
# General knowledge (applies to all pages)
|
|
31
|
+
npx explorbot learn "*" "This is a React SPA. Wait for loading spinners to disappear."
|
|
32
|
+
|
|
33
|
+
# Specific page behavior
|
|
34
|
+
npx explorbot learn "/checkout" "Credit card field requires format: XXXX-XXXX-XXXX-XXXX"
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
### Inside TUI
|
|
38
|
+
|
|
39
|
+
While exploring, use the `/learn` command.
|
|
40
|
+
|
|
41
|
+
```
|
|
42
|
+
/learn # Opens interactive form
|
|
43
|
+
/learn Test user: test@example.com # Adds to current page
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
### API Testing
|
|
47
|
+
|
|
48
|
+
[API testing](../api-testing/basics.md) shares the same `knowledge/` directory. `npx explorbot api know <endpoint> "<description>"` adds endpoint-scoped notes, stored with an `endpoint:` frontmatter field instead of `url:`.
|
|
49
|
+
|
|
50
|
+
## URL Patterns
|
|
51
|
+
|
|
52
|
+
| Pattern | Matches |
|
|
53
|
+
|---------|---------|
|
|
54
|
+
| `/login` | Exact path `/login` |
|
|
55
|
+
| `/admin/*` | Any path starting with `/admin/` |
|
|
56
|
+
| `*` | All pages (general knowledge) |
|
|
57
|
+
| `^/users/\d+` | Regex: `/users/` followed by digits |
|
|
58
|
+
| `~dashboard~` | Regex: "dashboard" anywhere in URL (tilde on both sides) |
|
|
59
|
+
|
|
60
|
+
## Knowledge File Format
|
|
61
|
+
|
|
62
|
+
Knowledge lives in `./knowledge/` as markdown files with frontmatter:
|
|
63
|
+
|
|
64
|
+
```markdown
|
|
65
|
+
---
|
|
66
|
+
url: /login
|
|
67
|
+
title: Login Page
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
Test credentials:
|
|
71
|
+
- email: admin@example.com
|
|
72
|
+
- password: secret123
|
|
73
|
+
|
|
74
|
+
Notes:
|
|
75
|
+
- Submit button disabled until email validates
|
|
76
|
+
- 3 failed attempts triggers captcha
|
|
77
|
+
- "Remember me" checkbox persists session for 30 days
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
### Frontmatter Fields
|
|
81
|
+
|
|
82
|
+
| Field | Purpose |
|
|
83
|
+
|-------|---------|
|
|
84
|
+
| `url` | URL pattern to match (optional, defaults to `*`) |
|
|
85
|
+
| `title` | Human-readable title (optional) |
|
|
86
|
+
| Custom fields | Any additional metadata for agents |
|
|
87
|
+
|
|
88
|
+
## Variables
|
|
89
|
+
|
|
90
|
+
Knowledge files support variable interpolation with `${namespace.key}` syntax. Explorbot resolves variables when it loads the knowledge.
|
|
91
|
+
|
|
92
|
+
### Environment Variables
|
|
93
|
+
|
|
94
|
+
Use `${env.VARNAME}` to reference environment variables. This keeps secrets out of knowledge files.
|
|
95
|
+
|
|
96
|
+
```markdown
|
|
97
|
+
---
|
|
98
|
+
url: /login
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
Login credentials:
|
|
102
|
+
- email: ${env.LOGIN}
|
|
103
|
+
- password: ${env.PASSWORD}
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Missing environment variables become an empty string.
|
|
107
|
+
|
|
108
|
+
### Config Variables
|
|
109
|
+
|
|
110
|
+
Use `${config.path}` to reference values from `explorbot.config.js` with dot notation.
|
|
111
|
+
|
|
112
|
+
```markdown
|
|
113
|
+
---
|
|
114
|
+
url: *
|
|
115
|
+
---
|
|
116
|
+
|
|
117
|
+
Base URL: ${config.playwright.url}
|
|
118
|
+
Browser: ${config.playwright.browser}
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
You can reference any scalar config value. Object values become an empty string.
|
|
122
|
+
|
|
123
|
+
### Supported Namespaces
|
|
124
|
+
|
|
125
|
+
| Namespace | Source | Example |
|
|
126
|
+
|-----------|--------|---------|
|
|
127
|
+
| `env` | `process.env` | `${env.API_KEY}` |
|
|
128
|
+
| `config` | `explorbot.config.js` | `${config.playwright.url}` |
|
|
129
|
+
|
|
130
|
+
Expressions with an unknown namespace (such as `${other.value}`) or no namespace (such as `${value}`) are left as-is.
|
|
131
|
+
|
|
132
|
+
## Page Automation
|
|
133
|
+
|
|
134
|
+
Knowledge files can run automation commands when explorbot navigates to a matching page. Use this for loading states, cookie banners, or page-specific setup.
|
|
135
|
+
|
|
136
|
+
### Available Fields
|
|
137
|
+
|
|
138
|
+
| Field | Type | Description |
|
|
139
|
+
|-------|------|-------------|
|
|
140
|
+
| `wait` | `number` | Wait for specified seconds after page load |
|
|
141
|
+
| `waitForElement` | `string` | Wait for element to appear (CSS selector) |
|
|
142
|
+
| `code` | `string` | Execute CodeceptJS code after navigation |
|
|
143
|
+
| `statePush` | `boolean` | Use `history.pushState` instead of full navigation |
|
|
144
|
+
|
|
145
|
+
### Wait for Page Load
|
|
146
|
+
|
|
147
|
+
```markdown
|
|
148
|
+
---
|
|
149
|
+
url: /dashboard
|
|
150
|
+
wait: 2
|
|
151
|
+
waitForElement: '.dashboard-loaded'
|
|
152
|
+
---
|
|
153
|
+
|
|
154
|
+
Dashboard requires data to load before interaction.
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
### Execute Custom Code
|
|
158
|
+
|
|
159
|
+
```markdown
|
|
160
|
+
---
|
|
161
|
+
url: /app/*
|
|
162
|
+
code: |
|
|
163
|
+
I.waitForElement('.app-ready');
|
|
164
|
+
I.click('.cookie-accept');
|
|
165
|
+
I.wait(1);
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
App pages need cookie consent dismissed and loading complete.
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### CodeceptJS Effects
|
|
172
|
+
|
|
173
|
+
Knowledge code can use CodeceptJS effects for error handling and retries:
|
|
174
|
+
|
|
175
|
+
| Effect | Purpose |
|
|
176
|
+
|--------|---------|
|
|
177
|
+
| `tryTo(fn)` | Execute without failing - returns `true`/`false` |
|
|
178
|
+
| `retryTo(fn, maxTries, interval)` | Retry on failure with polling |
|
|
179
|
+
| `within(context, fn)` | Execute within a specific element context |
|
|
180
|
+
|
|
181
|
+
**Example with effects:**
|
|
182
|
+
|
|
183
|
+
```markdown
|
|
184
|
+
---
|
|
185
|
+
url: /dashboard
|
|
186
|
+
code: |
|
|
187
|
+
await tryTo(() => I.click('.cookie-dismiss'));
|
|
188
|
+
await retryTo(() => {
|
|
189
|
+
I.click('Reload Data');
|
|
190
|
+
I.waitForElement('.data-loaded');
|
|
191
|
+
}, 5, 500);
|
|
192
|
+
---
|
|
193
|
+
|
|
194
|
+
Dashboard may show cookie banner. Data loads asynchronously - retry reload if needed.
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
> [!NOTE]
|
|
198
|
+
> Effects are async. Use `await` when you call them in knowledge code.
|
|
199
|
+
|
|
200
|
+
### SPA Navigation
|
|
201
|
+
|
|
202
|
+
For single-page apps where a full reload breaks state:
|
|
203
|
+
|
|
204
|
+
```markdown
|
|
205
|
+
---
|
|
206
|
+
url: /settings/*
|
|
207
|
+
statePush: true
|
|
208
|
+
---
|
|
209
|
+
|
|
210
|
+
Settings uses client-side routing. Use pushState to preserve app state.
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
> [!TIP]
|
|
214
|
+
> Use knowledge automation for page-specific behavior. For agent-specific logic, such as code that runs only during testing, use [Agent Hooks](../web-testing/hooks.md) instead.
|
|
215
|
+
|
|
216
|
+
### Execution Order
|
|
217
|
+
|
|
218
|
+
When explorbot navigates to a page, automation runs in this order:
|
|
219
|
+
|
|
220
|
+
1. Navigation (`I.amOnPage()` or `history.pushState`)
|
|
221
|
+
2. `wait` (if specified)
|
|
222
|
+
3. `waitForElement` (if specified)
|
|
223
|
+
4. `code` (if specified)
|
|
224
|
+
|
|
225
|
+
## What to Document
|
|
226
|
+
|
|
227
|
+
### Authentication
|
|
228
|
+
|
|
229
|
+
```markdown
|
|
230
|
+
---
|
|
231
|
+
url: /login
|
|
232
|
+
---
|
|
233
|
+
|
|
234
|
+
Credentials: test@example.com / testpass123
|
|
235
|
+
OAuth: Use "Continue with Google" for SSO testing
|
|
236
|
+
2FA: Code is always 123456 in test environment
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
### Form Behavior
|
|
240
|
+
|
|
241
|
+
```markdown
|
|
242
|
+
---
|
|
243
|
+
url: /checkout
|
|
244
|
+
---
|
|
245
|
+
|
|
246
|
+
Required fields: name, email, card number, expiry, CVV
|
|
247
|
+
Card format: XXXX-XXXX-XXXX-XXXX
|
|
248
|
+
Test card: 4111-1111-1111-1111, any future expiry, any CVV
|
|
249
|
+
Promo code "TEST10" gives 10% discount
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
### Navigation Quirks
|
|
253
|
+
|
|
254
|
+
```markdown
|
|
255
|
+
---
|
|
256
|
+
url: *
|
|
257
|
+
---
|
|
258
|
+
|
|
259
|
+
- App uses React Router, wait for route transitions
|
|
260
|
+
- Loading spinner class: .spinner-overlay
|
|
261
|
+
- Modals block interaction until dismissed
|
|
262
|
+
- Session expires after 15 minutes of inactivity
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
### Test Data
|
|
266
|
+
|
|
267
|
+
```markdown
|
|
268
|
+
---
|
|
269
|
+
url: /users
|
|
270
|
+
---
|
|
271
|
+
|
|
272
|
+
Test users available:
|
|
273
|
+
- admin@test.com (admin role)
|
|
274
|
+
- user@test.com (standard user)
|
|
275
|
+
- readonly@test.com (view-only permissions)
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
## How Agents Use Knowledge
|
|
279
|
+
|
|
280
|
+
When an agent works on a page, it gets the knowledge whose URL pattern matches:
|
|
281
|
+
|
|
282
|
+
1. **Navigator** — uses credentials and knows about special interactions
|
|
283
|
+
2. **Researcher** — reads page structure and hidden elements
|
|
284
|
+
3. **Planner** — adds edge cases and validation rules to test scenarios
|
|
285
|
+
4. **Tester** — uses test data and expected behaviors
|
|
286
|
+
|
|
287
|
+
## Best Practices
|
|
288
|
+
|
|
289
|
+
1. **Start with auth** — add login credentials before exploring protected areas
|
|
290
|
+
2. **Use `*` for globals** — document app-wide behavior such as loading states and timeouts
|
|
291
|
+
3. **Be specific** — give exact selectors, formats, and values when you know them
|
|
292
|
+
4. **Update as you learn** — add knowledge when agents struggle with an interaction
|
|
293
|
+
|
|
294
|
+
## File Organization
|
|
295
|
+
|
|
296
|
+
```
|
|
297
|
+
./knowledge/
|
|
298
|
+
├── login.md # /login page
|
|
299
|
+
├── checkout.md # /checkout page
|
|
300
|
+
├── general.md # * (all pages)
|
|
301
|
+
└── admin_users.md # /admin/users/*
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
Files are named after the URL pattern. Multiple entries for the same URL append to the same file.
|
|
305
|
+
|
|
306
|
+
## See Also
|
|
307
|
+
|
|
308
|
+
- [Agent Hooks](../web-testing/hooks.md) — per-agent custom code execution
|
|
309
|
+
- [Configuration](../reference/configuration.md) — full configuration reference
|
|
310
|
+
- [Page Interaction](../web-testing/page-interaction.md) — how agents interact with pages
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Planning Styles
|
|
2
|
+
|
|
3
|
+
A planning style is a markdown rule file that shapes what scenarios get planned. The whole file becomes the planning approach in the agent's prompt — written as instructions to a QA engineer on how to think about test scenarios.
|
|
4
|
+
|
|
5
|
+
Two agents use styles through the same mechanism: the web [Planner](../web-testing/planner.md) and the API [Chief](../api-testing/basics.md).
|
|
6
|
+
|
|
7
|
+
## Built-in Styles
|
|
8
|
+
|
|
9
|
+
Bundled style files live in `rules/planner/styles/` (web) and `rules/chief/styles/` (API):
|
|
10
|
+
|
|
11
|
+
| Style | Intent |
|
|
12
|
+
|-------|--------|
|
|
13
|
+
| `normal` | Complete user workflows: CRUD and full commit flows that end in a data or state change |
|
|
14
|
+
| `curious` | Coverage gaps: mines previous test results and research to find paths earlier tests missed |
|
|
15
|
+
| `psycho` | Stress tests: feeds empty, invalid, and extreme values to every reachable control, then commits |
|
|
16
|
+
| `hacker` | API only: probes beyond the spec — hidden endpoints, undocumented fields, unprotected actions |
|
|
17
|
+
|
|
18
|
+
## Cycling
|
|
19
|
+
|
|
20
|
+
Each planning iteration uses the next style in the list, cycling by index.
|
|
21
|
+
|
|
22
|
+
**Web Planner** — default order is `normal`, `curious`, `psycho`. The first `/plan` uses normal, the second curious, the third psycho, then the cycle repeats. Override the list and order with the `styles` option in [Planner configuration](../web-testing/planner.md#configuration); a name may appear more than once.
|
|
23
|
+
|
|
24
|
+
**API Chief** — order is `normal`, `curious`, `psycho`, `hacker`. `api plan` cycles them the same way. `api explore` doesn't cycle: it runs every style once, generating and executing a plan per style.
|
|
25
|
+
|
|
26
|
+
## Selecting a Style
|
|
27
|
+
|
|
28
|
+
Force a style for a single run:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
npx explorbot plan /users --style psycho # web
|
|
32
|
+
npx explorbot api plan /users --style hacker # API
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
The web TUI accepts the same flag: `/plan --style psycho`. `api explore` has no `--style` flag — it always runs all styles.
|
|
36
|
+
|
|
37
|
+
## Custom Styles
|
|
38
|
+
|
|
39
|
+
Styles load from `rules/<agent>/styles/<name>.md` in your project first, falling back to the bundled file of the same name. To edit built-in styles, copy the bundled rules into your project:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
npx explorbot extract-rules planner # or: chief
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
This copies the agent's bundled rule files (including styles) to `./rules/<agent>/`, skipping files that already exist. Edit the copies — they take precedence over the bundled versions. Extract only what you want to change; anything you delete falls back to the bundled file.
|
|
46
|
+
|
|
47
|
+
Style files are plain markdown with no frontmatter. Write the mindset, the patterns to test, what counts as a test, and what to skip.
|
|
48
|
+
|
|
49
|
+
To add a new web style, create `rules/planner/styles/<name>.md` and add its name to the rotation:
|
|
50
|
+
|
|
51
|
+
```javascript
|
|
52
|
+
ai: {
|
|
53
|
+
agents: {
|
|
54
|
+
planner: {
|
|
55
|
+
styles: ['normal', 'curious', 'psycho', 'security'],
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
}
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The API Chief has no `styles` config option — customize it by overriding the built-in style files in `rules/chief/styles/`.
|
|
62
|
+
|
|
63
|
+
## See Also
|
|
64
|
+
|
|
65
|
+
- [Planner](../web-testing/planner.md) — how web test plans are generated
|
|
66
|
+
- [API Testing](../api-testing/basics.md) — the Chief agent and API commands
|
|
67
|
+
- [Test Plans](./test-plans.md) — the plan file format
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# Test Reporting
|
|
2
|
+
|
|
3
|
+
Explorbot generates test reports with [@testomatio/reporter](https://github.com/testomatio/reporter). Reports include test steps, screenshots, and result messages for every run. The [API tester](../api-testing/basics.md) reports through the same reporter, so the HTML and Testomat.io settings below apply to API runs too.
|
|
4
|
+
|
|
5
|
+
The [Testomat.io cloud reporter](#testomatio-cloud-report) is the recommended way to review runs, especially on CI: pair it with the [Analyst](#session-analysis) and [screencasts](#screencasts) and each run opens with a written overview you can drill from down to a video of any single test. The [HTML](#html-report-local) and [markdown](#markdown-report-local) reports are the local alternatives. For the full CI setup, see [Continuous Integration](./ci.md#reporting-on-ci).
|
|
6
|
+
|
|
7
|
+
## Session analysis
|
|
8
|
+
|
|
9
|
+
After `/explore` and `/freesail` runs, the [Analyst agent](../web-testing/agents.md#analyst-agent) writes a summary that clusters findings by root cause.
|
|
10
|
+
|
|
11
|
+
The same markdown is printed to the console, written to disk, and set as the run description on Testomat.io when the cloud reporter is enabled.
|
|
12
|
+
|
|
13
|
+
**On disk:** `output/reports/<mode>-<sessionName>.md` — for example `explore-WiseFox42.md` or `freesail-CleverOwl91.md`. Each run gets a unique name, so nothing is overwritten.
|
|
14
|
+
|
|
15
|
+
**On Testomat.io:** the markdown becomes the run description, so the analysis sits next to the test list in the cloud dashboard with no extra setup.
|
|
16
|
+
|
|
17
|
+
See [Analyst Agent](../web-testing/agents.md#analyst-agent) for the report format and configuration options.
|
|
18
|
+
|
|
19
|
+
## Screencasts
|
|
20
|
+
|
|
21
|
+
Explorbot can record a video of every test instead of leaving you a trail of screenshots. Enable it on the Historian agent:
|
|
22
|
+
|
|
23
|
+
```js
|
|
24
|
+
export default {
|
|
25
|
+
ai: {
|
|
26
|
+
agents: {
|
|
27
|
+
historian: {
|
|
28
|
+
screencast: true, // or { size: { width: 1280, height: 720 }, quality: 95 }
|
|
29
|
+
},
|
|
30
|
+
},
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Each test gets one video in `output/screencasts/`, with the executed actions overlaid in the corner and the current scenario step shown as a chapter title — the video narrates itself. The file is attached to the test as an artifact, so with [artifact storage configured](#artifacts-in-cloud-reports) it appears on the test in Testomat.io, ready to watch next to the failure message.
|
|
36
|
+
|
|
37
|
+
## HTML report (local)
|
|
38
|
+
|
|
39
|
+
With `html: true` under `reporter` in `explorbot.config.js`, Explorbot creates an HTML report after each run in `output/reports/<mode>-<sessionName>.html`, for example `explore-WiseFox42.html`. Each session gets its own file, so nothing is overwritten. Open it in a browser to review results. Configs generated by `explorbot init` enable it, so new projects get HTML reports out of the box.
|
|
40
|
+
|
|
41
|
+

|
|
42
|
+
|
|
43
|
+
For cloud reporting with history and team features, see [Testomat.io cloud report](#testomatio-cloud-report).
|
|
44
|
+
|
|
45
|
+
## Markdown report (local)
|
|
46
|
+
|
|
47
|
+
Explorbot can write a markdown report next to the HTML one. It's plain text, so you can paste it into a PR description, a chat thread, or a CI summary.
|
|
48
|
+
|
|
49
|
+
Opt in via `explorbot.config.js`:
|
|
50
|
+
|
|
51
|
+
```js
|
|
52
|
+
export default {
|
|
53
|
+
reporter: {
|
|
54
|
+
enabled: true,
|
|
55
|
+
html: true,
|
|
56
|
+
markdown: true,
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Output: `output/reports/<mode>-<sessionName>-tests.md`, for example `explore-WiseFox42-tests.md`. Like the HTML report, the filename is session-scoped, so successive runs don't overwrite each other. The `-tests` suffix keeps it distinct from the [Analyst report](#session-analysis), which writes `<mode>-<sessionName>.md` in the same folder.
|
|
62
|
+
|
|
63
|
+
The markdown report isn't generated unless `markdown: true` is set. Configs generated by `explorbot init` enable it as well.
|
|
64
|
+
|
|
65
|
+
## Testomat.io cloud report
|
|
66
|
+
|
|
67
|
+
[Testomat.io](https://testomat.io) provides a cloud dashboard with test history, analytics, and team collaboration. It's free with unlimited test runs.
|
|
68
|
+
|
|
69
|
+

|
|
70
|
+
|
|
71
|
+
### Setup
|
|
72
|
+
|
|
73
|
+
1. Register at [app.testomat.io](https://app.testomat.io).
|
|
74
|
+
2. Create an empty project.
|
|
75
|
+
3. Copy the project API key.
|
|
76
|
+
|
|
77
|
+
### Run with the key
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
TESTOMATIO=tstmt_your_key_here npx explorbot explore /
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Set the key in your shell profile or CI environment so it's always active.
|
|
84
|
+
|
|
85
|
+
### Options
|
|
86
|
+
|
|
87
|
+
| Variable | Description |
|
|
88
|
+
|----------|-------------|
|
|
89
|
+
| `TESTOMATIO` | Project API key (required) |
|
|
90
|
+
| `TESTOMATIO_TITLE` | Custom name for the test run |
|
|
91
|
+
| `TESTOMATIO_ENV` | Environment label (e.g. `staging`, `production`) |
|
|
92
|
+
| `TESTOMATIO_SHARED_RUN` | Merge parallel executions into one run |
|
|
93
|
+
| `TESTOMATIO_RUNGROUP_TITLE` | Group successive runs under one heading (overrides `reporter.runGroup`) |
|
|
94
|
+
|
|
95
|
+
See the [@testomatio/reporter docs](https://github.com/testomatio/reporter/blob/2.x/docs/pipes/testomatio.md) for the full list.
|
|
96
|
+
|
|
97
|
+
### Run group
|
|
98
|
+
|
|
99
|
+
Set `reporter.runGroup` to file successive runs under one heading on Testomat.io. There is no built-in default, but configs generated by `explorbot init` set it to today's date (`YYYY-MM-DD`), so all sessions from one day appear together in the dashboard.
|
|
100
|
+
|
|
101
|
+
Change or remove it via `explorbot.config.js`:
|
|
102
|
+
|
|
103
|
+
```js
|
|
104
|
+
export default {
|
|
105
|
+
reporter: {
|
|
106
|
+
enabled: true,
|
|
107
|
+
runGroup: 'Smoke Suite', // any string; remove to disable grouping
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
If set, `TESTOMATIO_RUNGROUP_TITLE` from the environment takes precedence over the config.
|
|
113
|
+
|
|
114
|
+
## Artifacts in cloud reports
|
|
115
|
+
|
|
116
|
+
Explorbot attaches screenshots to test steps and, when [screencasts](#screencasts) are enabled, a video to every test. To see them in Testomat.io, configure an S3-compatible storage provider under Settings > Artifacts in Testomat.io. Without it, screenshots and screencasts won't appear in cloud reports.
|
|
117
|
+
|
|
118
|
+
Set these environment variables:
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
S3_ACCESS_KEY_ID=your_access_key
|
|
122
|
+
S3_SECRET_ACCESS_KEY=your_secret_key
|
|
123
|
+
S3_BUCKET=your-bucket-name
|
|
124
|
+
S3_REGION=us-east-1
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
For non-AWS providers (DigitalOcean Spaces, Cloudflare R2, Google Cloud Storage, Minio), also set:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
S3_ENDPOINT=https://your-provider-endpoint.com
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
See the [@testomatio/reporter artifacts docs](https://github.com/testomatio/reporter/blob/2.x/docs/artifacts.md) for provider-specific examples.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# Test Plans
|
|
2
|
+
|
|
3
|
+
A test plan is a markdown file with a suite of scenarios for the Tester to run. The [Planner](../web-testing/planner.md) generates plans, but the format is plain markdown. You can write plans by hand, edit generated ones, or check them into version control.
|
|
4
|
+
|
|
5
|
+
Explorbot saves plans to `output/plans/` by default. The same parser loads them whether they were generated or written by hand. [API testing](../api-testing/basics.md) saves and loads its plans in this same format.
|
|
6
|
+
|
|
7
|
+
The format is a dialect of the [Testomat.io classical markdown format](https://docs.testomat.io/project/import-export/export-tests/classical-tests-markdown-format/). It adds a `### Prerequisite` block that tells Explorbot which page to open before each test.
|
|
8
|
+
|
|
9
|
+
## Format
|
|
10
|
+
|
|
11
|
+
```markdown
|
|
12
|
+
<!-- suite -->
|
|
13
|
+
# Plan Title
|
|
14
|
+
|
|
15
|
+
### Prerequisite
|
|
16
|
+
|
|
17
|
+
* URL: /relative-path
|
|
18
|
+
|
|
19
|
+
<!-- test
|
|
20
|
+
priority: critical
|
|
21
|
+
-->
|
|
22
|
+
# Scenario written as a user-facing sentence
|
|
23
|
+
|
|
24
|
+
## Requirements
|
|
25
|
+
/relative-path
|
|
26
|
+
|
|
27
|
+
## Steps
|
|
28
|
+
* First step in plain language
|
|
29
|
+
* Second step
|
|
30
|
+
|
|
31
|
+
## Expected
|
|
32
|
+
* First expected outcome
|
|
33
|
+
* Second expected outcome
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
One file can hold several suites. Each begins with its own `<!-- suite -->` marker and parses as an independent plan.
|
|
37
|
+
|
|
38
|
+
## Elements
|
|
39
|
+
|
|
40
|
+
### `<!-- suite -->`
|
|
41
|
+
|
|
42
|
+
Marks the start of a plan. The `#` heading on the next line becomes the plan's title. This follows the Testomat.io convention of HTML-comment metadata blocks.
|
|
43
|
+
|
|
44
|
+
### `### Prerequisite`
|
|
45
|
+
|
|
46
|
+
Holds the suite-level URL as a single bullet:
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
* URL: /relative-path
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The URL is required. Without it, the suite's tests do not run. Make it relative to the configured base URL (start with `/`), so the same plan runs against staging, production, or a local dev server without edits.
|
|
53
|
+
|
|
54
|
+
Tests without their own `## Requirements` URL use this URL as their start page. Explorbot navigates to it before each scenario.
|
|
55
|
+
|
|
56
|
+
### `<!-- test priority: … -->`
|
|
57
|
+
|
|
58
|
+
Opens a test block. Valid priorities: `critical`, `important`, `high`, `normal`, `low`. Omit it and the priority defaults to `normal`. See [Test Priorities](../web-testing/planner.md#test-priorities) for what each level means.
|
|
59
|
+
|
|
60
|
+
### `#` Scenario heading
|
|
61
|
+
|
|
62
|
+
A single `#` heading inside a test block is the scenario description. Write it as a business outcome, not a click path.
|
|
63
|
+
|
|
64
|
+
### `## Requirements`
|
|
65
|
+
|
|
66
|
+
A per-test start URL, written as a plain line (no bullet) right under the heading. It overrides the suite Prerequisite URL for this test. The Planner writes it for every test; when authoring plans by hand, give each test its own Requirements URL.
|
|
67
|
+
|
|
68
|
+
### `## Steps`
|
|
69
|
+
|
|
70
|
+
A bulleted list (`* `) of planned actions in plain language. The Tester treats these as guidance, not a strict script, and may adapt them to what it sees on the page. To span a step across lines, indent continuation lines with 2 spaces.
|
|
71
|
+
|
|
72
|
+
The Testomat.io classical format inlines `*Expected*:` inside each step. Explorbot splits actions and outcomes into separate `## Steps` and `## Expected` sections.
|
|
73
|
+
|
|
74
|
+
### `## Expected`
|
|
75
|
+
|
|
76
|
+
A bulleted list (`* `) of expected outcomes. Each outcome should describe a verifiable change: a data change, a state change, or a UI change with a side effect. See the Planner's [outcome-strength guidance](../web-testing/planner.md#built-in-styles) for what counts.
|
|
77
|
+
|
|
78
|
+
The Tester passes a test only when it has verified every expected outcome.
|
|
79
|
+
|
|
80
|
+
## Reusing saved plans
|
|
81
|
+
|
|
82
|
+
Saved plans are reusable. Run `npx explorbot explore --configure="new:25%"` to load the matching plan, re-run a subset of its tests, and let the planner generate a few new ones to fill the budget. Filter old picks by priority or style, and order them by priority, file index, or random shuffle. See the [`--configure` reference in commands.md](../reference/commands.md#explore) for the full key list and examples.
|
|
83
|
+
|
|
84
|
+
## See Also
|
|
85
|
+
|
|
86
|
+
- [Automated Tests](../web-testing/automated-tests.md) — the runnable Playwright or CodeceptJS files Explorbot writes after executing a plan
|
|
87
|
+
- [Planner](../web-testing/planner.md) — how plans are generated
|
|
88
|
+
- [Planning Styles](./planning-styles.md) — how planning styles shape scenarios
|
|
89
|
+
- [Commands](../reference/commands.md) — `/plan`, `/explore`, `npx explorbot plan`
|
|
90
|
+
- [Rerun](../web-testing/rerun.md) — re-executing generated tests
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "explorbot",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.4",
|
|
4
4
|
"description": "CLI app built with React Ink, CodeceptJS, and Playwright",
|
|
5
5
|
"license": "Elastic-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
},
|
|
20
20
|
"files": [
|
|
21
21
|
"dist/",
|
|
22
|
+
"docs/",
|
|
22
23
|
"src/**/*.ts",
|
|
23
24
|
"src/**/*.tsx",
|
|
24
25
|
"bin/**/*.ts",
|
|
@@ -119,7 +120,7 @@
|
|
|
119
120
|
"ora-classic": "^5.4.2",
|
|
120
121
|
"parse5": "^8.0.0",
|
|
121
122
|
"pixelmatch": "^7.2.0",
|
|
122
|
-
"playwright": "^1.
|
|
123
|
+
"playwright": "^1.62",
|
|
123
124
|
"pngjs": "^7.0.0",
|
|
124
125
|
"react": "^19.1.1",
|
|
125
126
|
"sambanova-ai-provider": "^1.2.2",
|
|
@@ -105,9 +105,29 @@ Checks that page source does NOT contain expected text.
|
|
|
105
105
|
I.dontSeeInSource('error-class');
|
|
106
106
|
</example>
|
|
107
107
|
|
|
108
|
+
### Asserting the state of a control
|
|
109
|
+
|
|
110
|
+
State means disabled, checked, readonly, required, selected, expanded. A presence assertion says
|
|
111
|
+
nothing about it, so assert it with an attribute selector:
|
|
112
|
+
|
|
113
|
+
<example>
|
|
114
|
+
I.seeElement('button[aria-label="Submit"][disabled]');
|
|
115
|
+
I.dontSeeElement('button[aria-label="Submit"][disabled]');
|
|
116
|
+
I.seeElement('input[name="accept"][checked]');
|
|
117
|
+
I.seeElement('button[aria-label="Details"][aria-expanded="true"]');
|
|
118
|
+
</example>
|
|
119
|
+
|
|
120
|
+
Use I.seeElement for the state you expect and I.dontSeeElement for the state you expect to be
|
|
121
|
+
absent — that pair expresses both directions.
|
|
122
|
+
|
|
123
|
+
I.seeAttributesOnElements(<locator>, { disabled: true }) also exists, but only takes a state that
|
|
124
|
+
must be PRESENT and does not resolve reliably against a role/text locator. Prefer the selector form above.
|
|
125
|
+
|
|
108
126
|
<verification_rules>
|
|
109
127
|
Be strict in assertions to avoid false positives.
|
|
110
128
|
Prefer I.seeElement() with ARIA locators - most reliable.
|
|
129
|
+
For a claim about a control's state, assert it with an attribute selector — presence of the element is not evidence of its state.
|
|
130
|
+
If no assertion above can express the claim, say so instead of proposing an assertion that checks something weaker.
|
|
111
131
|
I.see() and I.dontSee() MUST include context parameter.
|
|
112
132
|
For input field values, ALWAYS use I.seeInField() — never check value via CSS attribute selectors or I.seeInSource.
|
|
113
133
|
Prefer text locators (label, name, placeholder) for form fields: I.seeInField('Search', 'value') over I.seeInField('input[name="search"]', 'value').
|
package/src/action-result.ts
CHANGED
|
@@ -30,6 +30,7 @@ interface ActionResultData extends WebPageState {
|
|
|
30
30
|
iframeSnapshots?: Array<{ src: string; html: string; id?: string }>;
|
|
31
31
|
ariaSnapshot?: string | null;
|
|
32
32
|
ariaSnapshotFile?: string;
|
|
33
|
+
focusedElement?: FocusedElement | null;
|
|
33
34
|
iframeURL?: string;
|
|
34
35
|
links?: Link[];
|
|
35
36
|
}
|
|
@@ -74,6 +75,7 @@ export class ActionResult implements ActionResultData {
|
|
|
74
75
|
private snapshotCache = new TTLCache<string>();
|
|
75
76
|
readonly logFile: string | undefined = undefined;
|
|
76
77
|
readonly ariaSnapshotFile: string | undefined = undefined;
|
|
78
|
+
readonly focusedElement: FocusedElement | null = null;
|
|
77
79
|
private _ariaSnapshot: string | null | undefined = undefined;
|
|
78
80
|
private _lastExtractedHtml: string | undefined = undefined;
|
|
79
81
|
notes: string[] = [];
|
|
@@ -107,6 +109,9 @@ export class ActionResult implements ActionResultData {
|
|
|
107
109
|
if (data.ariaSnapshotFile !== undefined) {
|
|
108
110
|
this.ariaSnapshotFile = data.ariaSnapshotFile;
|
|
109
111
|
}
|
|
112
|
+
if (data.focusedElement !== undefined) {
|
|
113
|
+
this.focusedElement = data.focusedElement;
|
|
114
|
+
}
|
|
110
115
|
|
|
111
116
|
// Store HTML in a private property if provided
|
|
112
117
|
if (data.html !== undefined) {
|
|
@@ -643,3 +648,9 @@ export class Diff {
|
|
|
643
648
|
this._ariaChangeCount = ariaDiff.count;
|
|
644
649
|
}
|
|
645
650
|
}
|
|
651
|
+
|
|
652
|
+
export interface FocusedElement {
|
|
653
|
+
role: string;
|
|
654
|
+
name: string;
|
|
655
|
+
value?: string;
|
|
656
|
+
}
|