@_deep4wee/agent-lens 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +331 -227
- package/dist/cli.js +1068 -911
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +1277 -1033
- package/dist/cli.mjs.map +1 -1
- package/dist/dsl-BIjVN1M0.d.mts +204 -0
- package/dist/dsl-BIjVN1M0.d.ts +204 -0
- package/dist/index.d.mts +131 -178
- package/dist/index.d.ts +131 -178
- package/dist/index.js +1506 -8
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1479 -3
- package/dist/index.mjs.map +1 -1
- package/dist/plugins/a11y-tree/index.d.mts +11 -0
- package/dist/plugins/a11y-tree/index.d.ts +11 -0
- package/dist/plugins/a11y-tree/index.js +190 -0
- package/dist/plugins/a11y-tree/index.js.map +1 -0
- package/dist/plugins/a11y-tree/index.mjs +155 -0
- package/dist/plugins/a11y-tree/index.mjs.map +1 -0
- package/dist/plugins/desktop-webview2/index.d.mts +14 -0
- package/dist/plugins/desktop-webview2/index.d.ts +14 -0
- package/dist/plugins/desktop-webview2/index.js +258 -0
- package/dist/plugins/desktop-webview2/index.js.map +1 -0
- package/dist/plugins/desktop-webview2/index.mjs +221 -0
- package/dist/plugins/desktop-webview2/index.mjs.map +1 -0
- package/dist/plugins/live-controller/index.d.mts +35 -0
- package/dist/plugins/live-controller/index.d.ts +35 -0
- package/dist/plugins/live-controller/index.js +303 -0
- package/dist/plugins/live-controller/index.js.map +1 -0
- package/dist/plugins/live-controller/index.mjs +261 -0
- package/dist/plugins/live-controller/index.mjs.map +1 -0
- package/dist/plugins/mock-ipc/index.d.mts +30 -0
- package/dist/plugins/mock-ipc/index.d.ts +30 -0
- package/dist/plugins/mock-ipc/index.js +210 -0
- package/dist/plugins/mock-ipc/index.js.map +1 -0
- package/dist/plugins/mock-ipc/index.mjs +181 -0
- package/dist/plugins/mock-ipc/index.mjs.map +1 -0
- package/dist/plugins/visual-diff/index.d.mts +30 -0
- package/dist/plugins/visual-diff/index.d.ts +30 -0
- package/dist/plugins/visual-diff/index.js +163 -0
- package/dist/plugins/visual-diff/index.js.map +1 -0
- package/dist/plugins/visual-diff/index.mjs +127 -0
- package/dist/plugins/visual-diff/index.mjs.map +1 -0
- package/docs/plugins.md +415 -0
- package/package.json +40 -2
- package/skills/agent-lens/SKILL.md +130 -43
- package/skills/agent-lens/examples/07-live-controller-interactive-loop.md +108 -0
- package/skills/agent-lens/examples/08-accessibility-semantic-inspection.md +80 -0
- package/skills/agent-lens/examples/09-visual-regression-and-pixel-diffing.md +66 -0
- package/skills/agent-lens/examples/10-authoring-custom-agent-plugins.md +85 -0
- package/skills/agent-lens/references/plugin-development.md +165 -0
|
@@ -1,36 +1,72 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: agent-lens
|
|
3
|
-
description: Visual UI self-verification
|
|
3
|
+
description: Visual UI self-verification platform for AI agents. Take responsive multi-viewport snapshots of live URLs, drive interactive real-time browser sessions, inspect semantic accessibility trees, detect visual regressions with pixel diffing, or run scripted interaction scenarios in Chromium or native desktop WebView2.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# 👁️ AgentLens
|
|
6
|
+
# 👁️ AgentLens (Microkernel Agentic UI Platform)
|
|
7
7
|
|
|
8
|
-
**AgentLens**
|
|
8
|
+
**AgentLens** gives AI coding agents **eyes** and **hands** to test and verify user interfaces before reporting back to humans.
|
|
9
9
|
|
|
10
|
-
Instead of writing frontend code and blindly guessing if it looks right
|
|
10
|
+
Instead of writing frontend code and blindly guessing if it looks right or works properly, AgentLens allows you to:
|
|
11
|
+
1. Capture multi-viewport screenshots of your running app with one-shot `snap`.
|
|
12
|
+
2. Inspect entire scrollable pages with `--full`.
|
|
13
|
+
3. Drive interactive step-by-step browser sessions via persistent `live` commands (`click`, `type`, `snap`) without cold-start browser restarts.
|
|
14
|
+
4. "Read" the exact semantic hierarchy and interactive states using the `a11y-tree` plugin (essential for text-only LLMs or accessibility audits).
|
|
15
|
+
5. Detect pixel-level visual regressions using the `visual-diff` plugin.
|
|
16
|
+
6. Intercept silent JavaScript runtime errors, unhandled promise rejections, and missing assets.
|
|
17
|
+
7. Author your own 1-file plugins on-the-fly in `.agent-lens/plugins/` to solve novel project constraints.
|
|
11
18
|
|
|
12
19
|
---
|
|
13
20
|
|
|
14
|
-
## ⚡
|
|
21
|
+
## ⚡ 3 Ways for an Agent to Interact
|
|
15
22
|
|
|
16
|
-
### 1. Instant Verification (`snap`)
|
|
17
|
-
Ideal for 90% of tasks when you just modified a page, component, or layout:
|
|
23
|
+
### 1. Instant One-Shot Verification (`snap`)
|
|
24
|
+
Ideal for 90% of tasks when you just modified a page, component, or responsive layout:
|
|
18
25
|
```bash
|
|
19
|
-
# Verify
|
|
26
|
+
# Verify live dev server across desktop & mobile viewports:
|
|
20
27
|
npx agent-lens snap --url=http://localhost:5173
|
|
21
28
|
|
|
29
|
+
# Capture entire scrollable page height:
|
|
30
|
+
npx agent-lens snap --url=http://localhost:5173 --full
|
|
31
|
+
|
|
22
32
|
# Auto-start dev server in a monorepo (e.g. Frontend/), snap, and auto-terminate:
|
|
23
33
|
npx agent-lens snap --start="npm run dev" --start-cwd=./Frontend
|
|
24
34
|
|
|
25
|
-
# Zero-config smart snap (auto-detects active server or static build):
|
|
26
|
-
npx agent-lens snap
|
|
27
|
-
|
|
28
35
|
# Focus strictly on one component:
|
|
29
36
|
npx agent-lens snap --url=http://localhost:5173/settings --selector=".pricing-card"
|
|
37
|
+
|
|
38
|
+
# Extract semantic accessibility tree during snap:
|
|
39
|
+
npx agent-lens snap --url=http://localhost:5173 --plugin=a11y-tree
|
|
30
40
|
```
|
|
31
41
|
|
|
32
|
-
|
|
33
|
-
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
### 2. Interactive Live Controller Loop (`live`)
|
|
45
|
+
Ideal for multi-step flows, debugging interactions, or iterative visual tuning without restarting the browser on each action:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
# Step 1: Start background session (launches browser on CDP port 9223)
|
|
49
|
+
npx agent-lens live start --url=http://localhost:5173
|
|
50
|
+
|
|
51
|
+
# Step 2: Click via Vision physical coordinates or CSS selector
|
|
52
|
+
npx agent-lens live click 450 180
|
|
53
|
+
npx agent-lens live click "button.open-modal"
|
|
54
|
+
|
|
55
|
+
# Step 3: Fill inputs
|
|
56
|
+
npx agent-lens live type "input[name='email']" "agent@example.com"
|
|
57
|
+
|
|
58
|
+
# Step 4: Capture current screen or full scrollable page
|
|
59
|
+
npx agent-lens live snap step_02 --full
|
|
60
|
+
|
|
61
|
+
# Step 5: Stop session when done
|
|
62
|
+
npx agent-lens live stop
|
|
63
|
+
```
|
|
64
|
+
> 💡 *Every live action automatically updates `artifacts/live/current.png` in ~50ms, allowing instant visual feedback.*
|
|
65
|
+
|
|
66
|
+
---
|
|
67
|
+
|
|
68
|
+
### 3. Scripted Scenarios (`scenarios/*.scenario.ts`)
|
|
69
|
+
For reproducible test suites, state mocking, animations, and regression diffing:
|
|
34
70
|
```bash
|
|
35
71
|
npx agent-lens --scenario=checkout-flow --url=http://localhost:5173
|
|
36
72
|
```
|
|
@@ -39,24 +75,58 @@ npx agent-lens --scenario=checkout-flow --url=http://localhost:5173
|
|
|
39
75
|
|
|
40
76
|
## 🧭 The Agent Workflow
|
|
41
77
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
78
|
+
```mermaid
|
|
79
|
+
flowchart TD
|
|
80
|
+
Edit["1. Edit Code (UI / CSS / Logic)"] --> Choose["2. Choose Verification Mode"]
|
|
81
|
+
Choose -->|Quick check| Snap["npx agent-lens snap --url=..."]
|
|
82
|
+
Choose -->|Step-by-step flow| Live["npx agent-lens live start -> click -> type -> stop"]
|
|
83
|
+
Choose -->|Regression / Scenarios| Scenario["npx agent-lens --scenario=..."]
|
|
84
|
+
|
|
85
|
+
Snap --> Inspect["3. Inspect Output"]
|
|
86
|
+
Live --> Inspect
|
|
87
|
+
Scenario --> Inspect
|
|
88
|
+
|
|
89
|
+
Inspect --> CheckLog["Check artifacts/latest/report.md for console errors"]
|
|
90
|
+
CheckLog --> VisionCheck{"Do you have Vision tools?"}
|
|
91
|
+
VisionCheck -->|Yes| ViewImg["View artifacts/latest/*.png or artifacts/live/current.png"]
|
|
92
|
+
VisionCheck -->|No| ReadA11y["Read artifacts/latest/a11y-tree.md (Roles, Names, Focus)"]
|
|
93
|
+
|
|
94
|
+
ViewImg --> Iterate["4. Fix errors or regressions & iterate"]
|
|
95
|
+
ReadA11y --> Iterate
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
1. **Check Logs First**: Always inspect `artifacts/latest/report.md`. If there are any **Console Errors**, fix the JavaScript / React exceptions first.
|
|
99
|
+
2. **If You Have Vision (`view_image`)**: Inspect the latest generated PNGs:
|
|
100
|
+
- `artifacts/latest/01_quick_snap_desktop.png`
|
|
101
|
+
- `artifacts/latest/02_quick_snap_mobile.png`
|
|
102
|
+
- Look for clipped text, unexpected wrapping, broken flex/grid columns, or overlay bugs.
|
|
103
|
+
3. **If You Are Text-Only**: Run with `--plugin=a11y-tree` and inspect:
|
|
104
|
+
- `artifacts/latest/a11y-tree.md`
|
|
105
|
+
- Verify that buttons, form inputs, headings, and dialogs are rendered with expected text and states.
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
## 🔌 Built-In Plugins & Extensibility
|
|
110
|
+
|
|
111
|
+
AgentLens is built around a lightweight **Microkernel architecture**. The core engine is decoupled from drivers, mocks, and tooling plugins:
|
|
112
|
+
|
|
113
|
+
| Plugin | Primary Purpose | How to Activate |
|
|
114
|
+
| :--- | :--- | :--- |
|
|
115
|
+
| `live-controller` | Background CDP session for coordinate clicks, drag-and-drop, interactive CLI loop | CLI `live` command or `--plugin=live-controller` |
|
|
116
|
+
| `a11y-tree` | Semantic accessibility tree extraction into clean Markdown | `--plugin=a11y-tree` or in scenario `plugins: ['a11y-tree']` |
|
|
117
|
+
| `visual-diff` | Pixel-by-pixel regression diffing with `pixelmatch` | `--plugin=visual-diff` or in scenario `plugins: ['visual-diff']` |
|
|
118
|
+
| `desktop-webview2` | Windows native `.exe` testing via CDP remote port | Auto-activated on `--mode=desktop` or `--exe=path/to/app.exe` |
|
|
119
|
+
| `mock-ipc` | Desktop IPC bridge mocking (`window.__mockIpc`) | Auto-activated if `scenarios/mocks.ts` exists |
|
|
120
|
+
|
|
121
|
+
### 🛠️ Writing Custom Plugins On-The-Fly
|
|
122
|
+
If you encounter a project constraint that cannot be handled by default options (e.g. custom authentication headers, pre-populating `localStorage`, WebSocket mocking, or database seeding), **write a 1-file plugin**:
|
|
123
|
+
|
|
124
|
+
1. Create `.agent-lens/plugins/<name>.ts`.
|
|
125
|
+
2. Export `definePlugin({ name: '<name>', ... })` as `default`.
|
|
126
|
+
3. Use lifecycle hooks (`onContextCreated`, `extendContext`, `onAfterRun`, `teardown`).
|
|
127
|
+
4. Execute via `npx agent-lens snap --plugin=<name>`.
|
|
128
|
+
|
|
129
|
+
*Full instructions & examples:* [Plugin Development Reference](references/plugin-development.md) and [docs/plugins.md](file:///E:/github/agent-lens/docs/plugins.md).
|
|
60
130
|
|
|
61
131
|
---
|
|
62
132
|
|
|
@@ -77,7 +147,7 @@ await ctx.click('[data-testid="save-button"]');
|
|
|
77
147
|
```
|
|
78
148
|
|
|
79
149
|
### 2. Base Mocks in `scenarios/mocks.ts` (Prevent Root Crashes)
|
|
80
|
-
When your app mounts,
|
|
150
|
+
When your app mounts, root components (Navbar, AuthContext, Layout) often query base endpoints (e.g. `GET_USER`, `GET_SETTINGS`, `GET_ACCOUNTS`). If your scenario only mocks one sub-action, unmocked root actions may return empty and crash the app with `Cannot read properties of null (reading 'length')`.
|
|
81
151
|
- Place common base mocks in `scenarios/mocks.ts`.
|
|
82
152
|
- AgentLens automatically merges `scenarios/mocks.ts` with your scenario's specific `mockIpc: [...]` overrides!
|
|
83
153
|
|
|
@@ -104,6 +174,7 @@ npx agent-lens [command] [options]
|
|
|
104
174
|
|
|
105
175
|
### Commands:
|
|
106
176
|
- `snap`: Instant one-shot snapshot & console check of a URL or static build.
|
|
177
|
+
- `live`: Interactive session (`start`, `click`, `type`, `snap`, `stop`).
|
|
107
178
|
- `init`: Generate starter template in `scenarios/template.scenario.ts` and `scenarios/mocks.ts`.
|
|
108
179
|
- *(default)*: Runs matching scenarios from `scenarios/`.
|
|
109
180
|
|
|
@@ -114,9 +185,11 @@ npx agent-lens [command] [options]
|
|
|
114
185
|
| `--start="<cmd>"` | Auto-launch dev server / backend before test | *(none)* |
|
|
115
186
|
| `--start-cwd=<path>` | Directory to run `--start` in (e.g. `--start-cwd=./Frontend`) | *Auto-detected* |
|
|
116
187
|
| `--clean-artifacts` | Purge older test runs in `artifacts/` | `false` |
|
|
188
|
+
| `--full` | Capture full scrollable page height instead of viewport | `false` |
|
|
117
189
|
| `--selector=<css>` | Focus and resize-to-fit a specific component | *(none)* |
|
|
118
190
|
| `--viewports=<list>` | Viewports to capture (`desktop,mobile,tablet` or `1200x800`) | `desktop,mobile` |
|
|
119
191
|
| `--wait=<ms>` | Wait time after page load before taking snapshots | `1000` |
|
|
192
|
+
| `--plugin=<list>` | Comma-separated plugins to load (e.g. `--plugin=a11y-tree,visual-diff`) | *(none)* |
|
|
120
193
|
| `--scenario=<id>` | Name or prefix of scenario file to run | *(none)* |
|
|
121
194
|
| `--all` | Run all discovered scenarios | `false` |
|
|
122
195
|
| `--mode=preview\|desktop` | Engine: `preview` (Web / Live URL) or `desktop` (native .exe) | `preview` |
|
|
@@ -134,16 +207,16 @@ npx agent-lens [command] [options]
|
|
|
134
207
|
|
|
135
208
|
```typescript
|
|
136
209
|
import { defineVisualTest, VIEWPORT_PRESETS, type TestContext } from 'agent-lens';
|
|
137
|
-
```
|
|
138
210
|
|
|
139
|
-
### Scenario Definition Structure:
|
|
140
|
-
```typescript
|
|
141
211
|
export default defineVisualTest({
|
|
142
212
|
id: 'my-feature-check',
|
|
143
213
|
title: 'Feature Verification',
|
|
144
|
-
route: '/dashboard',
|
|
214
|
+
route: '/dashboard',
|
|
145
215
|
viewports: [VIEWPORT_PRESETS.DEFAULT, VIEWPORT_PRESETS.MIN_SUPPORTED],
|
|
146
216
|
|
|
217
|
+
// Load plugins for this scenario:
|
|
218
|
+
plugins: ['a11y-tree', 'visual-diff'],
|
|
219
|
+
|
|
147
220
|
// HTTP REST / GraphQL Network Mocks (Vite / Next.js / Web SPA)
|
|
148
221
|
mockRoutes: [
|
|
149
222
|
{ url: '**/api/v1/user', body: { id: 1, name: 'Agent', role: 'admin' } },
|
|
@@ -165,12 +238,11 @@ export default defineVisualTest({
|
|
|
165
238
|
teardown: async () => {}
|
|
166
239
|
});
|
|
167
240
|
```
|
|
168
|
-
|
|
169
241
|
|
|
170
242
|
### Available `ctx` Methods:
|
|
171
243
|
|
|
172
244
|
#### 📸 Capturing
|
|
173
|
-
- `await ctx.capture('01_name', options?)`: Capture
|
|
245
|
+
- `await ctx.capture('01_name', options?)`: Capture snapshot (supports `{ fullPage: true, selector: '...' }`).
|
|
174
246
|
- `await ctx.captureBurst('02_anim', { durationMs: 300, intervalMs: 50, selector? })`: Capture animation frames.
|
|
175
247
|
|
|
176
248
|
#### 📐 Viewport Control
|
|
@@ -200,19 +272,34 @@ export default defineVisualTest({
|
|
|
200
272
|
- `ctx.getConsoleErrors()`: Array of caught errors with stack traces.
|
|
201
273
|
- `ctx.getConsoleWarnings()`: Array of caught warnings.
|
|
202
274
|
|
|
203
|
-
#### 🌐 Network Route & Mock IPC
|
|
204
|
-
- `await ctx.setMockRoute('**/api/users', payload, options?)`: Dynamically intercepts HTTP/REST API endpoints
|
|
275
|
+
#### 🌐 Network Route & Mock IPC
|
|
276
|
+
- `await ctx.setMockRoute('**/api/users', payload, options?)`: Dynamically intercepts HTTP/REST API endpoints.
|
|
205
277
|
- `await ctx.setMockIpc('ACTION_NAME', payload, { type: 'SUCCESS' | 'ERROR', delayMs?: number })`: Dynamically alters mock data for hybrid IPC bridges.
|
|
206
|
-
|
|
278
|
+
|
|
279
|
+
#### 🔌 Plugin Extensions (When Plugins Are Loaded)
|
|
280
|
+
- **live-controller**:
|
|
281
|
+
- `await ctx.clickCoords(x, y, options?)`
|
|
282
|
+
- `await ctx.dragAndDrop(fromX, fromY, toX, toY, steps?)`
|
|
283
|
+
- `await ctx.scrollPercent(percent)`
|
|
284
|
+
- `await ctx.snapLive(options?)`
|
|
285
|
+
- **a11y-tree**:
|
|
286
|
+
- `await ctx.dumpAccessibilityTree({ selector?: string, compact?: boolean })`
|
|
287
|
+
- **visual-diff**:
|
|
288
|
+
- `await ctx.compareSnapshots(currentPath, baselinePath, options?)`
|
|
289
|
+
- `await ctx.captureAndCompare(name, baselinePath, captureOptions?, diffOptions?)`
|
|
207
290
|
|
|
208
291
|
---
|
|
209
292
|
|
|
210
|
-
## 📚
|
|
293
|
+
## 📚 Complete Walkthroughs & Examples
|
|
211
294
|
|
|
212
|
-
Check the dedicated example guides in `examples
|
|
295
|
+
Check the dedicated example guides in `examples/`:
|
|
213
296
|
1. [Instant Verification (`snap`)](examples/01-instant-verification-snap.md) — One-shot snapshotting without writing test files.
|
|
214
297
|
2. [Live Dev Server Workflow](examples/02-dev-server-live-testing.md) — Testing active Vite/Next.js servers with `--start` or `--url`.
|
|
215
298
|
3. [Component Isolation & Burst Animations](examples/03-component-isolation-and-animations.md) — Inspecting isolated components and CSS transitions.
|
|
216
299
|
4. [Native Desktop App Testing](examples/04-desktop-native-testing.md) — Testing compiled `.exe` binaries with CDP and crash diagnostics.
|
|
217
300
|
5. [Clean Teardown & Sandboxing](examples/05-clean-teardown-and-sandboxing.md) — Guaranteeing zero leftover test data using `teardown()` and `--clean`.
|
|
218
301
|
6. [State Testing with Mock IPC](examples/06-state-testing-with-mock-ipc.md) — Testing empty states, errors, and data tables.
|
|
302
|
+
7. [Live Controller Interactive Loop](examples/07-live-controller-interactive-loop.md) — Low-latency real-time control via CLI commands.
|
|
303
|
+
8. [Semantic Accessibility Tree Inspection](examples/08-accessibility-semantic-inspection.md) — Extracting UI hierarchies for text LLMs.
|
|
304
|
+
9. [Visual Regression & Pixel Diffing](examples/09-visual-regression-and-pixel-diffing.md) — Automated pixelmatch difference masks.
|
|
305
|
+
10. [Authoring Custom Agent Plugins](examples/10-authoring-custom-agent-plugins.md) — Writing 1-file plugins on-the-fly.
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# 🚀 Example 07: Live Controller Interactive Loop
|
|
2
|
+
|
|
3
|
+
The **Live Controller** plugin turns AgentLens into an interactive command-line browser remote. Instead of restarting the browser on every single change, an agent can keep a background session alive, execute actions step-by-step with sub-50ms latency, and inspect the state in real-time.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 🎯 Use Case
|
|
8
|
+
|
|
9
|
+
You are debugging a multi-step user flow (e.g. multi-page signup, complex drawer, or modal wizard) and want to:
|
|
10
|
+
1. Open the page once.
|
|
11
|
+
2. Click specific buttons or coordinates.
|
|
12
|
+
3. Fill in form values.
|
|
13
|
+
4. Capture full-page screenshots (`--full`).
|
|
14
|
+
5. Close the session when done.
|
|
15
|
+
|
|
16
|
+
---
|
|
17
|
+
|
|
18
|
+
## 🛠️ Step-by-Step CLI Walkthrough
|
|
19
|
+
|
|
20
|
+
### 1. Start the Live Background Session
|
|
21
|
+
```bash
|
|
22
|
+
npx agent-lens live start --url=http://localhost:5173
|
|
23
|
+
```
|
|
24
|
+
*Output:*
|
|
25
|
+
```text
|
|
26
|
+
🚀 [Live] Starting background browser for http://localhost:5173 on CDP port 9223...
|
|
27
|
+
📸 [LiveController] Snapshot saved: artifacts/live/current.png
|
|
28
|
+
✅ [Live] Session active! You can now send live commands:
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
The browser is now running in the background. The initial screen is saved to `artifacts/live/current.png`.
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
### 2. Click Elements (Coordinates or CSS Selectors)
|
|
36
|
+
|
|
37
|
+
#### Vision-Based Coordinate Click (for Multimodal LLMs)
|
|
38
|
+
If your vision model identified a button at physical coordinates `(450, 210)`:
|
|
39
|
+
```bash
|
|
40
|
+
npx agent-lens live click 450 210
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
#### Semantic Selector Click
|
|
44
|
+
```bash
|
|
45
|
+
npx agent-lens live click "button[type='submit']"
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
AgentLens immediately executes the click and updates `artifacts/live/current.png`.
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
### 3. Type into Form Fields
|
|
53
|
+
```bash
|
|
54
|
+
npx agent-lens live type "input[name='search']" "AgentLens Microkernel"
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
### 4. Capture Full Scrollable Page (`--full`)
|
|
60
|
+
To capture the entire scrollable height of the page (beyond the 1200x800 viewport):
|
|
61
|
+
```bash
|
|
62
|
+
npx agent-lens live snap dashboard_full --full
|
|
63
|
+
```
|
|
64
|
+
*Output:*
|
|
65
|
+
```text
|
|
66
|
+
📸 [LiveController] Snapshot saved: artifacts/live/dashboard_full.png (Full Page)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
### 5. Terminate the Session
|
|
72
|
+
When your test flow is complete, stop the live browser:
|
|
73
|
+
```bash
|
|
74
|
+
npx agent-lens live stop
|
|
75
|
+
```
|
|
76
|
+
*Output:*
|
|
77
|
+
```text
|
|
78
|
+
🛑 [Live] Session stopped and browser closed.
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
## 💡 Using `live-controller` Inside Scenarios
|
|
84
|
+
|
|
85
|
+
You can also use Live Controller methods directly inside scripted scenarios:
|
|
86
|
+
|
|
87
|
+
```typescript
|
|
88
|
+
import { defineVisualTest } from 'agent-lens';
|
|
89
|
+
import liveControllerPlugin from '@_deep4wee/agent-lens/plugins/live-controller';
|
|
90
|
+
|
|
91
|
+
export default defineVisualTest({
|
|
92
|
+
id: 'canvas-drawing-test',
|
|
93
|
+
plugins: [liveControllerPlugin],
|
|
94
|
+
run: async (ctx) => {
|
|
95
|
+
// 1. Precise coordinate click
|
|
96
|
+
await (ctx as any).clickCoords(350, 420);
|
|
97
|
+
|
|
98
|
+
// 2. Drag-and-drop simulation
|
|
99
|
+
await (ctx as any).dragAndDrop(100, 100, 300, 300, 15);
|
|
100
|
+
|
|
101
|
+
// 3. Smooth percentage scroll
|
|
102
|
+
await (ctx as any).scrollPercent(50);
|
|
103
|
+
|
|
104
|
+
// 4. Live snapshot with full page capture
|
|
105
|
+
await (ctx as any).snapLive({ name: 'canvas_final', fullPage: true });
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
```
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# ♿ Example 08: Semantic Accessibility Tree Inspection
|
|
2
|
+
|
|
3
|
+
Not every AI agent operates with multimodal vision tools enabled, and even vision models can miss subtle details like whether an element is marked `disabled`, `required`, or properly focused.
|
|
4
|
+
|
|
5
|
+
The **`a11y-tree`** plugin extracts the live semantic accessibility tree of the page and outputs a clean, indented Markdown outline.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## 🎯 Use Case
|
|
10
|
+
|
|
11
|
+
1. **Text-Only LLMs**: The agent lacks vision/image tools but needs to know what is rendered on screen.
|
|
12
|
+
2. **Form Validation**: Checking whether the submit button is `[disabled]` or input is `[required]`.
|
|
13
|
+
3. **Hierarchy Verification**: Verifying heading levels (`h1`, `h2`), navigation landmarks, and dialog focus.
|
|
14
|
+
|
|
15
|
+
---
|
|
16
|
+
|
|
17
|
+
## 🛠️ Step-by-Step Walkthrough
|
|
18
|
+
|
|
19
|
+
### 1. Enabling `a11y-tree` via CLI
|
|
20
|
+
You can enable the plugin directly on quick snap checks:
|
|
21
|
+
```bash
|
|
22
|
+
npx agent-lens snap --url=http://localhost:5173 --plugin=a11y-tree
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
### 2. Inspecting the Generated Tree
|
|
28
|
+
AgentLens outputs:
|
|
29
|
+
1. `artifacts/latest/a11y-tree.md`
|
|
30
|
+
2. An embedded section in `artifacts/latest/report.md`
|
|
31
|
+
|
|
32
|
+
#### Example Output (`a11y-tree.md`):
|
|
33
|
+
```markdown
|
|
34
|
+
- [banner]
|
|
35
|
+
- [heading] "AgentLens Dashboard" (level 1)
|
|
36
|
+
- [navigation]
|
|
37
|
+
- [link] "Overview"
|
|
38
|
+
- [link] "Settings"
|
|
39
|
+
- [main]
|
|
40
|
+
- [heading] "Active Sessions" (level 2)
|
|
41
|
+
- [textbox] "Search scenarios..." (focused)
|
|
42
|
+
- [button] "Run All"
|
|
43
|
+
- [button] "Cancel" (disabled)
|
|
44
|
+
- [list]
|
|
45
|
+
- [listitem] "checkout-flow.scenario.ts"
|
|
46
|
+
- [listitem] "auth-smoke.scenario.ts"
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
### 3. Using `a11y-tree` Inside a Scenario
|
|
52
|
+
|
|
53
|
+
```typescript
|
|
54
|
+
import { defineVisualTest } from 'agent-lens';
|
|
55
|
+
import a11yTreePlugin from '@_deep4wee/agent-lens/plugins/a11y-tree';
|
|
56
|
+
|
|
57
|
+
export default defineVisualTest({
|
|
58
|
+
id: 'a11y-inspection-flow',
|
|
59
|
+
title: 'Accessibility Hierarchy Check',
|
|
60
|
+
plugins: [a11yTreePlugin],
|
|
61
|
+
run: async (ctx) => {
|
|
62
|
+
// Navigate and interact
|
|
63
|
+
await ctx.click('button.open-dialog');
|
|
64
|
+
await ctx.wait(300);
|
|
65
|
+
|
|
66
|
+
// Capture semantic tree of entire page or isolated modal
|
|
67
|
+
const tree = await (ctx as any).dumpAccessibilityTree({
|
|
68
|
+
selector: '.dialog-content',
|
|
69
|
+
compact: true
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
// Custom assertion in scenario
|
|
73
|
+
if (!tree.includes('[button] "Confirm"')) {
|
|
74
|
+
throw new Error('Expected Confirm button in accessibility tree!');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
ctx.log('Accessibility tree verified successfully!');
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
```
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# 🎨 Example 09: Visual Regression & Pixel Diffing
|
|
2
|
+
|
|
3
|
+
The **`visual-diff`** plugin adds automated, pixel-by-pixel regression testing using `pixelmatch` and `pngjs`. It detects unintended visual shifts, color alterations, font changes, and layout regressions down to individual pixels.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 🎯 Use Case
|
|
8
|
+
|
|
9
|
+
You are refactoring CSS styles or upgrading a component library (e.g. Tailwind, Shadcn, MUI) and need to guarantee that the UI did not unintentionally change from a known baseline.
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## 🛠️ Step-by-Step Walkthrough
|
|
14
|
+
|
|
15
|
+
### 1. Establish Baseline Screenshots
|
|
16
|
+
Run your scenario once to generate reference screenshots, and commit them to a baseline directory (e.g. `fixtures/baselines/`):
|
|
17
|
+
```bash
|
|
18
|
+
mkdir -p fixtures/baselines
|
|
19
|
+
cp artifacts/latest/01_homepage_default.png fixtures/baselines/homepage_baseline.png
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
### 2. Write Scenario with Visual Diffing
|
|
25
|
+
|
|
26
|
+
```typescript
|
|
27
|
+
import path from 'path';
|
|
28
|
+
import { defineVisualTest } from 'agent-lens';
|
|
29
|
+
import visualDiffPlugin from '@_deep4wee/agent-lens/plugins/visual-diff';
|
|
30
|
+
|
|
31
|
+
export default defineVisualTest({
|
|
32
|
+
id: 'homepage-regression',
|
|
33
|
+
title: 'Homepage Visual Regression Check',
|
|
34
|
+
plugins: [visualDiffPlugin],
|
|
35
|
+
run: async (ctx) => {
|
|
36
|
+
const baselinePath = path.resolve(process.cwd(), 'fixtures/baselines/homepage_baseline.png');
|
|
37
|
+
|
|
38
|
+
// Capture current view and compare against baseline in a single step:
|
|
39
|
+
const diff = await (ctx as any).captureAndCompare(
|
|
40
|
+
'01_homepage_check',
|
|
41
|
+
baselinePath,
|
|
42
|
+
{ fullPage: false },
|
|
43
|
+
{ threshold: 0.1 } // Sensitivity threshold (0 to 1, default 0.1)
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
ctx.log(`Diff result: ${diff.diffPixels} pixels (${diff.diffPercent}%)`);
|
|
47
|
+
|
|
48
|
+
// Strict assertion: Fail if more than 0.5% of pixels changed
|
|
49
|
+
if (diff.diffPercent > 0.5) {
|
|
50
|
+
throw new Error(`Visual regression detected! Diff is ${diff.diffPercent}% (${diff.diffPixels} px).`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
### 3. Inspecting the Report
|
|
59
|
+
|
|
60
|
+
When the scenario runs, `visual-diff` generates:
|
|
61
|
+
- `artifacts/<timestamp>/01_homepage_check_diff.png` (visual difference mask in bright red)
|
|
62
|
+
- A comparison table in `artifacts/latest/report.md`:
|
|
63
|
+
|
|
64
|
+
| Test Step | Baseline | Current | Diff Image | Changed Pixels | Status |
|
|
65
|
+
| :--- | :--- | :--- | :--- | :-: | :-: |
|
|
66
|
+
| **01_homepage_check** | [Baseline](fixtures/baselines/homepage_baseline.png) | [Current](01_homepage_check.png) | [Diff](01_homepage_check_diff.png) | 48 px | 🟡 Minor Shift (0.04%) |
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# 🛠️ Example 10: Authoring Custom Agent Plugins
|
|
2
|
+
|
|
3
|
+
When testing real-world applications, AI coding agents frequently hit barriers that standard testing tools cannot handle out of the box — such as bypassing mock single-sign-on (SSO), pre-populating browser IndexedDB databases, or simulating unstable network throttles.
|
|
4
|
+
|
|
5
|
+
This example shows how an agent can solve this autonomously by creating a 1-file plugin.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## 🎯 Use Case
|
|
10
|
+
|
|
11
|
+
Your application checks for an active tenant ID in `window.__TENANT_CONFIG__` and crashes if it is missing:
|
|
12
|
+
```javascript
|
|
13
|
+
// App code (main.tsx)
|
|
14
|
+
const tenant = window.__TENANT_CONFIG__.activeId; // Crashes if undefined!
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Instead of modifying production code or struggling with complex setup scripts, the agent authors a custom plugin to inject this state before React hydrates.
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## 🛠️ Step-by-Step Walkthrough
|
|
22
|
+
|
|
23
|
+
### 1. Agent Creates the 1-File Plugin
|
|
24
|
+
Create `.agent-lens/plugins/tenant-seeder.ts`:
|
|
25
|
+
|
|
26
|
+
```typescript
|
|
27
|
+
import { definePlugin } from 'agent-lens';
|
|
28
|
+
|
|
29
|
+
export default definePlugin({
|
|
30
|
+
name: 'tenant-seeder',
|
|
31
|
+
version: '1.0.0',
|
|
32
|
+
|
|
33
|
+
onContextCreated: async (context) => {
|
|
34
|
+
// Inject global tenant configuration before page scripts execute
|
|
35
|
+
await context.addInitScript(() => {
|
|
36
|
+
(window as any).__TENANT_CONFIG__ = {
|
|
37
|
+
activeId: 'tenant-enterprise-99',
|
|
38
|
+
name: 'Acme Global Corp',
|
|
39
|
+
tier: 'ENTERPRISE',
|
|
40
|
+
features: {
|
|
41
|
+
analyticsDashboard: true,
|
|
42
|
+
auditLogs: true
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
});
|
|
46
|
+
},
|
|
47
|
+
|
|
48
|
+
onAfterRun: (reportData) => {
|
|
49
|
+
if (!reportData.customSections) return;
|
|
50
|
+
reportData.customSections.push({
|
|
51
|
+
title: '🏢 Tenant Seeder Status',
|
|
52
|
+
content: 'Successfully injected enterprise tenant mock: `tenant-enterprise-99`.'
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
---
|
|
59
|
+
|
|
60
|
+
### 2. Run with Quick Snap or Scenarios
|
|
61
|
+
|
|
62
|
+
#### Quick Snap
|
|
63
|
+
```bash
|
|
64
|
+
npx agent-lens snap --url=http://localhost:5173 --plugin=tenant-seeder
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
#### Scripted Scenario
|
|
68
|
+
```typescript
|
|
69
|
+
import { defineVisualTest } from 'agent-lens';
|
|
70
|
+
|
|
71
|
+
export default defineVisualTest({
|
|
72
|
+
id: 'enterprise-dashboard',
|
|
73
|
+
plugins: ['tenant-seeder'], // Auto-resolved from .agent-lens/plugins/tenant-seeder.ts!
|
|
74
|
+
run: async (ctx) => {
|
|
75
|
+
await ctx.capture('01_enterprise_dashboard');
|
|
76
|
+
const headerText = await ctx.readText('.tenant-banner');
|
|
77
|
+
ctx.log(`Banner verified: ${headerText}`);
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
---
|
|
83
|
+
|
|
84
|
+
### 3. Verification & Cleanup
|
|
85
|
+
The agent reviews `artifacts/latest/report.md`. If the custom plugin was only needed for temporary testing, the agent can delete `.agent-lens/plugins/tenant-seeder.ts` or keep it committed in the repository for future test runs.
|