barebrowse 0.2.2 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/barebrowse/SKILL.md +107 -0
- package/CHANGELOG.md +57 -0
- package/CLAUDE.md +4 -2
- package/README.md +52 -5
- package/barebrowse.context.md +27 -8
- package/cli.js +289 -48
- package/docs/00-context/assumptions.md +38 -0
- package/docs/{blueprint.md → 00-context/system-state.md} +30 -5
- package/docs/00-context/vision.md +52 -0
- package/docs/01-product/prd.md +284 -0
- package/docs/03-logs/bug-log.md +16 -0
- package/docs/03-logs/decisions-log.md +32 -0
- package/docs/03-logs/implementation-log.md +54 -0
- package/docs/03-logs/insights.md +35 -0
- package/docs/03-logs/validation-log.md +123 -0
- package/docs/04-process/definition-of-done.md +31 -0
- package/docs/04-process/dev-workflow.md +68 -0
- package/docs/{testing.md → 04-process/testing.md} +21 -2
- package/docs/README.md +55 -0
- package/docs/archive/poc-plan.md +230 -0
- package/package.json +1 -1
- package/src/aria.js +1 -1
- package/src/daemon.js +321 -0
- package/src/session-client.js +70 -0
|
@@ -6,14 +6,14 @@
|
|
|
6
6
|
node --test test/unit/*.test.js test/integration/*.test.js
|
|
7
7
|
```
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
64 tests, 6 files, ~60s on a typical machine. No test framework -- uses Node's built-in `node:test` runner.
|
|
10
10
|
|
|
11
11
|
## Test pyramid
|
|
12
12
|
|
|
13
13
|
```
|
|
14
14
|
/ E2E \ 15 tests — real websites (Google, Wikipedia, GitHub, etc.)
|
|
15
15
|
/----------\
|
|
16
|
-
/ Integration \
|
|
16
|
+
/ Integration \ 21 tests — browse/connect pipeline + CLI session lifecycle
|
|
17
17
|
/----------------\
|
|
18
18
|
/ Unit \ 28 tests — pruning, cookie extraction, CDP client, browser launch
|
|
19
19
|
/--------------------\
|
|
@@ -98,6 +98,25 @@ Tests the full `browse()` and `connect()` pipeline end-to-end against real pages
|
|
|
98
98
|
| 10 | connect() | supports multiple navigations in one session | Multiple goto() calls on same page |
|
|
99
99
|
| 11 | connect() | snapshot accepts prune: false for raw output | snapshot(false) preserves full tree |
|
|
100
100
|
|
|
101
|
+
### `test/integration/cli.test.js` -- 10 tests
|
|
102
|
+
|
|
103
|
+
Tests the full CLI session lifecycle: daemon spawn, command dispatch over HTTP, and cleanup. Uses a temp directory so tests don't pollute the project.
|
|
104
|
+
|
|
105
|
+
| # | Test | What it validates |
|
|
106
|
+
|---|------|-------------------|
|
|
107
|
+
| 1 | open starts a daemon and creates session.json | `barebrowse open about:blank` spawns daemon, writes session.json with port+pid |
|
|
108
|
+
| 2 | status shows running session | `barebrowse status` reports pid, port, start time |
|
|
109
|
+
| 3 | snapshot creates a .yml file | `barebrowse snapshot` writes .barebrowse/page-*.yml |
|
|
110
|
+
| 4 | goto navigates and snapshot shows new page content | `barebrowse goto example.com` + snapshot contains "Example Domain" + refs |
|
|
111
|
+
| 5 | click sends click command | `barebrowse click <ref>` returns "ok" |
|
|
112
|
+
| 6 | eval executes JS and returns result | `barebrowse eval 1+1` returns "2" |
|
|
113
|
+
| 7 | console-logs creates a .json file | After eval with console.log, `console-logs` writes JSON |
|
|
114
|
+
| 8 | network-log creates a .json file | `network-log` writes JSON with request entries |
|
|
115
|
+
| 9 | close shuts down the daemon | `barebrowse close` removes session.json, daemon exits |
|
|
116
|
+
| 10 | status after close shows no session | `barebrowse status` exits non-zero when no session |
|
|
117
|
+
|
|
118
|
+
Note: Tests run sequentially within the suite (each depends on the session opened in test 1). The `after()` hook ensures daemon cleanup even if tests fail.
|
|
119
|
+
|
|
101
120
|
---
|
|
102
121
|
|
|
103
122
|
## E2E tests (15 tests)
|
package/docs/README.md
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# barebrowse -- Documentation
|
|
2
|
+
|
|
3
|
+
## Navigation
|
|
4
|
+
|
|
5
|
+
### 00-context/ -- Why and what exists
|
|
6
|
+
|
|
7
|
+
| File | What's in it |
|
|
8
|
+
|------|-------------|
|
|
9
|
+
| [vision.md](00-context/vision.md) | What barebrowse is, what it's not, the core insight, success criteria |
|
|
10
|
+
| [assumptions.md](00-context/assumptions.md) | Hard constraints, assumptions, known limitations, risks |
|
|
11
|
+
| [system-state.md](00-context/system-state.md) | Current architecture, full pipeline, module table, capabilities, tested sites |
|
|
12
|
+
|
|
13
|
+
### 01-product/ -- What the product must do
|
|
14
|
+
|
|
15
|
+
| File | What's in it |
|
|
16
|
+
|------|-------------|
|
|
17
|
+
| [prd.md](01-product/prd.md) | Product requirements, API design, three modes, pruning strategy, future features |
|
|
18
|
+
|
|
19
|
+
### 02-features/ -- How features are designed
|
|
20
|
+
|
|
21
|
+
*Feature-specific docs go here as the project grows.*
|
|
22
|
+
|
|
23
|
+
### 03-logs/ -- What changed over time
|
|
24
|
+
|
|
25
|
+
| File | What's in it |
|
|
26
|
+
|------|-------------|
|
|
27
|
+
| [decisions-log.md](03-logs/decisions-log.md) | Settled design decisions with rationale (don't re-debate these) |
|
|
28
|
+
| [implementation-log.md](03-logs/implementation-log.md) | What changed per version (summary of CHANGELOG) |
|
|
29
|
+
| [bug-log.md](03-logs/bug-log.md) | Bugs: symptom, root cause, fix, regression test |
|
|
30
|
+
| [validation-log.md](03-logs/validation-log.md) | Test suite results, site validation matrix, token reduction measurements |
|
|
31
|
+
| [insights.md](03-logs/insights.md) | Lessons learned, repos studied, technical patterns |
|
|
32
|
+
|
|
33
|
+
### 04-process/ -- How to work with this system
|
|
34
|
+
|
|
35
|
+
| File | What's in it |
|
|
36
|
+
|------|-------------|
|
|
37
|
+
| [dev-workflow.md](04-process/dev-workflow.md) | Dev rules, dependency hierarchy, running tests, environment setup |
|
|
38
|
+
| [definition-of-done.md](04-process/definition-of-done.md) | Checklist: when is a feature/fix actually done |
|
|
39
|
+
| [testing.md](04-process/testing.md) | Test pyramid, all 64 tests documented, writing new tests, CI strategy |
|
|
40
|
+
|
|
41
|
+
### archive/ -- Historical docs
|
|
42
|
+
|
|
43
|
+
| File | Why archived |
|
|
44
|
+
|------|-------------|
|
|
45
|
+
| [poc-plan.md](archive/poc-plan.md) | All 4 POC phases completed. Useful bits migrated to system-state.md and testing.md. |
|
|
46
|
+
|
|
47
|
+
## Also at project root
|
|
48
|
+
|
|
49
|
+
| File | Purpose |
|
|
50
|
+
|------|---------|
|
|
51
|
+
| `README.md` | Public-facing project overview |
|
|
52
|
+
| `barebrowse.context.md` | LLM-consumable integration guide (full API, gotchas, wiring) |
|
|
53
|
+
| `.claude/skills/barebrowse/SKILL.md` | CLI command reference + Claude Code skill definition |
|
|
54
|
+
| `CHANGELOG.md` | Detailed version-by-version changelog |
|
|
55
|
+
| `CLAUDE.md` | AI agent instructions for this project |
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
# barebrowse — POC Plan
|
|
2
|
+
|
|
3
|
+
**Date:** 2026-02-22
|
|
4
|
+
**Goal:** Prove that an autonomous agent can get an authenticated, pruned ARIA snapshot of any web page via CDP — no Playwright, no bundled browser.
|
|
5
|
+
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Repo Structure
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
barebrowse/
|
|
12
|
+
├── src/
|
|
13
|
+
│ ├── index.js # Public API: browse(), connect()
|
|
14
|
+
│ ├── chromium.js # Find/launch/connect to Chromium browsers
|
|
15
|
+
│ ├── cdp.js # Vanilla WebSocket CDP client
|
|
16
|
+
│ ├── aria.js # Accessibility.getFullAXTree → structured tree
|
|
17
|
+
│ ├── auth.js # Cookie extraction + CDP Network.setCookie injection
|
|
18
|
+
│ ├── prune.js # ARIA tree pruning (ported from mcprune)
|
|
19
|
+
│ ├── interact.js # Click, type, scroll via CDP Input domain
|
|
20
|
+
│ ├── consent.js # Auto-dismiss cookie consent dialogs
|
|
21
|
+
│ └── stealth.js # Anti-detection patches via Runtime.evaluate (Phase 4)
|
|
22
|
+
├── test/
|
|
23
|
+
│ ├── integration/
|
|
24
|
+
│ │ ├── browse.test.js # End-to-end: URL → pruned snapshot
|
|
25
|
+
│ │ ├── auth.test.js # Cookie injection → authenticated page
|
|
26
|
+
│ │ └── interact.test.js # Click/type on a live page
|
|
27
|
+
│ └── unit/
|
|
28
|
+
│ ├── cdp.test.js # CDP client message handling
|
|
29
|
+
│ ├── aria.test.js # ARIA tree formatting
|
|
30
|
+
│ └── prune.test.js # Pruning logic on sample trees
|
|
31
|
+
├── docs/
|
|
32
|
+
│ ├── prd.md # Product requirements (comprehensive)
|
|
33
|
+
│ └── poc-plan.md # This file
|
|
34
|
+
├── package.json
|
|
35
|
+
└── CLAUDE.md
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
**No build step.** Vanilla JS, ES modules, runs directly with Node.js >= 22.
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## Phases
|
|
43
|
+
|
|
44
|
+
### Phase 1 — CDP + ARIA Foundation
|
|
45
|
+
|
|
46
|
+
**Prove:** Get an ARIA tree from any page via CDP, no Playwright.
|
|
47
|
+
|
|
48
|
+
**Files:**
|
|
49
|
+
- `src/chromium.js` — Find installed Chromium browsers on the system (Chrome, Chromium, Brave, Edge). Launch headless with `--headless=new --remote-debugging-port=<port>`. Parse CDP WebSocket URL from stderr output.
|
|
50
|
+
- `src/cdp.js` — Vanilla WebSocket client that speaks CDP. Send JSON commands, receive responses and events. Handle command IDs, promises, event subscriptions. ~100 lines.
|
|
51
|
+
- `src/aria.js` — Call `Accessibility.getFullAXTree` via CDP. Transform the raw CDP response (flat array of AXNodes with parentId references) into a nested tree structure. Format as readable output.
|
|
52
|
+
- `src/index.js` — Wire chromium → cdp → aria into `browse(url)` function. Minimal, just the pipeline.
|
|
53
|
+
|
|
54
|
+
**Test:**
|
|
55
|
+
```bash
|
|
56
|
+
node -e "import { browse } from './src/index.js'; console.log(await browse('https://example.com'))"
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
**DoD:**
|
|
60
|
+
- [x] `chromium.js` finds and launches at least one Chromium browser on Fedora Linux
|
|
61
|
+
- [x] `cdp.js` connects via WebSocket, sends commands, receives responses
|
|
62
|
+
- [x] `aria.js` returns a structured ARIA tree for any public page
|
|
63
|
+
- [x] `browse(url)` works end-to-end with zero external dependencies
|
|
64
|
+
- [x] Headless Chrome process is cleaned up on close
|
|
65
|
+
|
|
66
|
+
### Phase 2 — Auth + Prune
|
|
67
|
+
|
|
68
|
+
**Prove:** Authenticated, pruned ARIA snapshot of a Cloudflare-protected page.
|
|
69
|
+
|
|
70
|
+
**Files:**
|
|
71
|
+
- `src/auth.js` — Extract cookies from user's browser profile (use sweet-cookie or implement minimal extraction from Chrome's Cookies SQLite DB + Linux keyring decryption via `secret-tool`). Inject via CDP `Network.setCookie` before navigation.
|
|
72
|
+
- `src/prune.js` — Port mcprune's pruning logic as a pure function. Input: raw ARIA tree. Output: pruned ARIA tree. Role-based: keep landmarks + interactive elements, drop noise/structural wrappers.
|
|
73
|
+
- Update `src/index.js` — Add cookie injection and pruning to the `browse()` pipeline.
|
|
74
|
+
|
|
75
|
+
**Test:**
|
|
76
|
+
```bash
|
|
77
|
+
# Should return authenticated content, not a login wall or CF challenge
|
|
78
|
+
node -e "import { browse } from './src/index.js'; console.log(await browse('https://some-cf-protected-site.com'))"
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
**DoD:**
|
|
82
|
+
- [x] `auth.js` extracts cookies from Firefox profile on Linux (also supports Chromium when installed)
|
|
83
|
+
- [x] Cookies injected via CDP before navigation
|
|
84
|
+
- [ ] CF-protected page returns real content, not challenge page (needs active session to test)
|
|
85
|
+
- [x] `prune.js` reduces ARIA tree by 47%+ on HN (minimal site — heavier sites will see 70%+)
|
|
86
|
+
- [x] Pruned output preserves all interactive elements and landmarks
|
|
87
|
+
- [x] `browse(url)` returns pruned, authenticated snapshot by default
|
|
88
|
+
|
|
89
|
+
### Phase 3 — Headed Mode + Interaction
|
|
90
|
+
|
|
91
|
+
**Prove:** Connect to user's running browser and interact with a logged-in page.
|
|
92
|
+
|
|
93
|
+
**Files:**
|
|
94
|
+
- Update `src/chromium.js` — Add `connect()` mode: connect to an already-running browser's debug port instead of launching a new one. Detect running browsers with debug ports.
|
|
95
|
+
- `src/interact.js` — Click (`Input.dispatchMouseEvent`), type (`Input.dispatchKeyEvent`), scroll. Resolve ARIA node IDs to DOM coordinates for click targets.
|
|
96
|
+
- Update `src/index.js` — Add `connect()` export for long-lived sessions. Add `mode: 'headed'` option.
|
|
97
|
+
|
|
98
|
+
**Prerequisite:** User must launch their browser with `--remote-debugging-port=9222` flag.
|
|
99
|
+
|
|
100
|
+
**Test:**
|
|
101
|
+
```bash
|
|
102
|
+
# User has Chrome open with debug port, logged into GitHub
|
|
103
|
+
node -e "
|
|
104
|
+
import { connect } from './src/index.js';
|
|
105
|
+
const page = await connect({ mode: 'headed' });
|
|
106
|
+
await page.goto('https://github.com/notifications');
|
|
107
|
+
console.log(await page.snapshot());
|
|
108
|
+
"
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
**DoD:**
|
|
112
|
+
- [x] `connect()` attaches to a running Chromium browser via CDP
|
|
113
|
+
- [x] Same ARIA + prune pipeline works on headed browser
|
|
114
|
+
- [x] `click()` and `type()` send real input events via CDP
|
|
115
|
+
- [x] `press()` sends special keys (Enter, Tab, Escape, arrows) — triggers form submit
|
|
116
|
+
- [x] `scrollIntoView` before click ensures off-screen elements are reachable
|
|
117
|
+
- [x] `type({ clear: true })` replaces pre-filled input content
|
|
118
|
+
- [x] `waitForNavigation()` waits for page load after link clicks
|
|
119
|
+
- [x] Interactions tested against real sites: Wikipedia, GitHub, Google, Hacker News, DuckDuckGo, YouTube
|
|
120
|
+
- [x] Browser stays open after barebrowse disconnects
|
|
121
|
+
- [x] Cookie injection via `page.injectCookies()` for headed mode (Firefox → Chromium)
|
|
122
|
+
- [x] Permission prompts suppressed via launch flags + CDP `Browser.setPermission`
|
|
123
|
+
- [x] Cookie consent dialogs auto-dismissed across 16+ sites in 7 languages
|
|
124
|
+
- [x] YouTube end-to-end: Firefox cookies → search → click → video playback in headed mode
|
|
125
|
+
|
|
126
|
+
### Phase 4 — Hybrid + bareagent Integration
|
|
127
|
+
|
|
128
|
+
**Prove:** Agent autonomously browses the web using barebrowse tools.
|
|
129
|
+
|
|
130
|
+
**Files:**
|
|
131
|
+
- Update `src/chromium.js` — Add `mode: 'hybrid'`. Try headless first. If navigation returns a CF challenge or 403, automatically retry in headed mode.
|
|
132
|
+
- `src/stealth.js` — Basic anti-detection: patch `navigator.webdriver`, `navigator.plugins`, `window.chrome`. Applied via `Runtime.evaluate` on new page.
|
|
133
|
+
- Update `src/index.js` — Final API surface: `browse()`, `connect()`.
|
|
134
|
+
|
|
135
|
+
**Test:**
|
|
136
|
+
```js
|
|
137
|
+
import { Loop } from 'bare-agent';
|
|
138
|
+
import { browse } from './src/index.js';
|
|
139
|
+
|
|
140
|
+
const tools = [
|
|
141
|
+
{ name: 'browse', execute: ({ url }) => browse(url) },
|
|
142
|
+
];
|
|
143
|
+
|
|
144
|
+
const loop = new Loop({ provider });
|
|
145
|
+
await loop.run([
|
|
146
|
+
{ role: 'user', content: 'Go to hacker news and tell me the top 3 stories' }
|
|
147
|
+
], tools);
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
**DoD:**
|
|
151
|
+
- [ ] Hybrid mode automatically falls back when headless is blocked
|
|
152
|
+
- [ ] Stealth patches reduce headless detection on common sites
|
|
153
|
+
- [ ] bareagent can use `browse()` as a tool in its think/act/observe loop
|
|
154
|
+
- [ ] Agent successfully completes a multi-page research task autonomously
|
|
155
|
+
|
|
156
|
+
---
|
|
157
|
+
|
|
158
|
+
## Definition of Done — Full POC
|
|
159
|
+
|
|
160
|
+
The POC is complete when ALL of these are true:
|
|
161
|
+
|
|
162
|
+
1. **`browse(url)` works end-to-end** — URL in, pruned ARIA snapshot out, authenticated as the user
|
|
163
|
+
2. **Zero heavy deps** — no Playwright, no Puppeteer. Only deps: `ws` (WebSocket client, if Node's built-in isn't sufficient) and optionally `sweet-cookie`
|
|
164
|
+
3. **Three modes work** — headless (default), headed (connect to running browser), hybrid (auto-fallback)
|
|
165
|
+
4. **Works on Fedora Linux** — finds Chrome/Chromium/Brave, launches headless, connects headed
|
|
166
|
+
5. **Token-efficient output** — pruned ARIA tree is 70%+ smaller than raw tree
|
|
167
|
+
6. **Clean process management** — headless browser spawned and killed cleanly, no orphan processes
|
|
168
|
+
7. **Under 1,000 lines total** for core src/ (excluding tests)
|
|
169
|
+
8. **Documented** — PRD captures all decisions, this file captures all phases
|
|
170
|
+
|
|
171
|
+
## What the POC is NOT
|
|
172
|
+
|
|
173
|
+
- Not production-ready. No error recovery, no retry logic, no edge case handling beyond happy path.
|
|
174
|
+
- Not cross-platform tested. Linux first (Fedora). macOS/Windows later.
|
|
175
|
+
- Not an MCP server. That's a future wrapper.
|
|
176
|
+
- Not a published npm package. Local development only.
|
|
177
|
+
|
|
178
|
+
---
|
|
179
|
+
|
|
180
|
+
## Running Tests
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
# All tests (47+ tests)
|
|
184
|
+
node --test test/unit/*.test.js test/integration/*.test.js
|
|
185
|
+
|
|
186
|
+
# Unit tests only (fast, no network)
|
|
187
|
+
node --test test/unit/prune.test.js # 16 tests — pruning logic
|
|
188
|
+
node --test test/unit/auth.test.js # 7 tests — cookie extraction (2 fail when Chromium locked)
|
|
189
|
+
node --test test/unit/cdp.test.js # 5 tests — CDP client + browser launch
|
|
190
|
+
|
|
191
|
+
# Integration tests (needs network + Chromium)
|
|
192
|
+
node --test test/integration/browse.test.js # 11 tests — end-to-end pipeline
|
|
193
|
+
node --test test/integration/interact.test.js # 15 tests — interactions on real sites
|
|
194
|
+
|
|
195
|
+
# Quick smoke test
|
|
196
|
+
node -e "import { browse } from './src/index.js'; console.log(await browse('https://example.com'))"
|
|
197
|
+
|
|
198
|
+
# Headed mode demos (requires: chromium-browser --remote-debugging-port=9222)
|
|
199
|
+
node examples/headed-demo.js # Wikipedia → DuckDuckGo search
|
|
200
|
+
node examples/yt-demo.js # YouTube: Firefox cookies → search → play video
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
---
|
|
204
|
+
|
|
205
|
+
## Repos Studied — What We Borrowed vs Built
|
|
206
|
+
|
|
207
|
+
| steipete repo | What we studied | What we used | Why not more |
|
|
208
|
+
|---|---|---|---|
|
|
209
|
+
| **sweet-cookie** | Cookie extraction (SQLite + keyring) | **Concept only** — wrote `auth.js` ourselves | Not on npm (different package). Our version is simpler, tailored, vanilla JS |
|
|
210
|
+
| **sweetlink** | CDP dual-channel, selector discovery, daemon | **CDP-direct concept only** | Daemon + WebSocket bridge + in-page runtime = bloat. CDP direct is 100 lines vs ~2,000 |
|
|
211
|
+
| **canvas** | Stealth/anti-detection patterns | **Noted for Phase 4** `stealth.js` | Not needed yet — headless + real cookies handles most cases |
|
|
212
|
+
| **mcprune (own)** | ARIA pruning pipeline | **Full port** — `prune.js` (472 lines) | Proven code, adapted node format from Playwright YAML to CDP tree objects |
|
|
213
|
+
|
|
214
|
+
### What to explore in later phases
|
|
215
|
+
|
|
216
|
+
- **Selector discovery** (sweetlink) — crawl ARIA tree, score interactive elements, rank action targets. Phase 3/4.
|
|
217
|
+
- **Stealth patches** (canvas) — `navigator.webdriver`, plugins, chrome object spoofing. Phase 4.
|
|
218
|
+
- **In-page JS execution** (sweetlink) — `Runtime.evaluate` for complex interactions. Phase 3.
|
|
219
|
+
- **Screenshot + visual grounding** — `Page.captureScreenshot` for multimodal agents. Post-POC.
|
|
220
|
+
|
|
221
|
+
---
|
|
222
|
+
|
|
223
|
+
## Dev Rules (from AGENT_RULES.md)
|
|
224
|
+
|
|
225
|
+
- **Vanilla JS only.** No TypeScript, no build step, no transpilation.
|
|
226
|
+
- **Dependency hierarchy:** vanilla → stdlib → external. Write it yourself if <50 lines.
|
|
227
|
+
- **Simple > clever.** Readable code a junior can follow.
|
|
228
|
+
- **POC first.** Validate logic before designing. Never ship the POC — rewrite it.
|
|
229
|
+
- **Test behavior, not implementation.** Integration tests over unit tests.
|
|
230
|
+
- **No speculative code.** Every line must have a purpose.
|
package/package.json
CHANGED
package/src/aria.js
CHANGED
|
@@ -20,7 +20,7 @@ export function formatTree(node, depth = 0) {
|
|
|
20
20
|
|
|
21
21
|
// Skip ignored nodes but still process their children
|
|
22
22
|
if (node.ignored) {
|
|
23
|
-
return node.children.map((c) => formatTree(c, depth)).join('');
|
|
23
|
+
return node.children.map((c) => formatTree(c, depth)).filter(Boolean).join('\n');
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
// Skip low-level rendering nodes that are noise for agents
|
package/src/daemon.js
ADDED
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* daemon.js -- Background HTTP server holding a connect() session.
|
|
3
|
+
*
|
|
4
|
+
* startDaemon() — spawn a detached child process running the daemon
|
|
5
|
+
* runDaemon() — the actual HTTP server (called via --daemon-internal)
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { createServer } from 'node:http';
|
|
9
|
+
import { spawn } from 'node:child_process';
|
|
10
|
+
import { writeFileSync, mkdirSync, existsSync, readFileSync, unlinkSync } from 'node:fs';
|
|
11
|
+
import { join, resolve } from 'node:path';
|
|
12
|
+
import { connect } from './index.js';
|
|
13
|
+
|
|
14
|
+
const SESSION_FILE = 'session.json';
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Spawn a detached child process that runs the daemon.
|
|
18
|
+
* Parent polls for session.json, then exits.
|
|
19
|
+
*/
|
|
20
|
+
export async function startDaemon(opts, outputDir, initialUrl) {
|
|
21
|
+
const absDir = resolve(outputDir);
|
|
22
|
+
mkdirSync(absDir, { recursive: true });
|
|
23
|
+
|
|
24
|
+
// Clean stale session
|
|
25
|
+
const sessionPath = join(absDir, SESSION_FILE);
|
|
26
|
+
if (existsSync(sessionPath)) unlinkSync(sessionPath);
|
|
27
|
+
|
|
28
|
+
// Build child args
|
|
29
|
+
const args = [join(import.meta.dirname, '..', 'cli.js'), '--daemon-internal'];
|
|
30
|
+
args.push('--output-dir', absDir);
|
|
31
|
+
if (initialUrl) args.push('--url', initialUrl);
|
|
32
|
+
if (opts.mode) args.push('--mode', opts.mode);
|
|
33
|
+
if (opts.port) args.push('--port', String(opts.port));
|
|
34
|
+
if (opts.cookies === false) args.push('--no-cookies');
|
|
35
|
+
if (opts.browser) args.push('--browser', opts.browser);
|
|
36
|
+
if (opts.timeout) args.push('--timeout', String(opts.timeout));
|
|
37
|
+
if (opts.pruneMode) args.push('--prune-mode', opts.pruneMode);
|
|
38
|
+
if (opts.consent === false) args.push('--no-consent');
|
|
39
|
+
|
|
40
|
+
const child = spawn(process.execPath, args, {
|
|
41
|
+
detached: true,
|
|
42
|
+
stdio: 'ignore',
|
|
43
|
+
env: { ...process.env },
|
|
44
|
+
});
|
|
45
|
+
child.unref();
|
|
46
|
+
|
|
47
|
+
// Poll for session.json (50ms interval, 15s timeout)
|
|
48
|
+
const deadline = Date.now() + 15000;
|
|
49
|
+
while (Date.now() < deadline) {
|
|
50
|
+
if (existsSync(sessionPath)) {
|
|
51
|
+
try {
|
|
52
|
+
const data = JSON.parse(readFileSync(sessionPath, 'utf8'));
|
|
53
|
+
if (data.port && data.pid) return data;
|
|
54
|
+
} catch { /* partial write, retry */ }
|
|
55
|
+
}
|
|
56
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
57
|
+
}
|
|
58
|
+
throw new Error('Daemon failed to start within 15s');
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Run the daemon HTTP server. Called by cli.js --daemon-internal.
|
|
63
|
+
* Holds a connect() session and serves commands over HTTP.
|
|
64
|
+
*/
|
|
65
|
+
export async function runDaemon(opts, outputDir, initialUrl) {
|
|
66
|
+
const absDir = resolve(outputDir);
|
|
67
|
+
mkdirSync(absDir, { recursive: true });
|
|
68
|
+
|
|
69
|
+
// Connect to browser
|
|
70
|
+
const page = await connect({
|
|
71
|
+
mode: opts.mode || 'headless',
|
|
72
|
+
port: opts.port ? Number(opts.port) : undefined,
|
|
73
|
+
consent: opts.consent,
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
// Console log capture
|
|
77
|
+
const consoleLogs = [];
|
|
78
|
+
await page.cdp.send('Runtime.enable');
|
|
79
|
+
page.cdp.on('Runtime.consoleAPICalled', (params) => {
|
|
80
|
+
consoleLogs.push({
|
|
81
|
+
type: params.type,
|
|
82
|
+
timestamp: new Date().toISOString(),
|
|
83
|
+
args: params.args.map((a) => a.value ?? a.description ?? a.type),
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
// Network log capture (Network.enable already called by connect)
|
|
88
|
+
const networkLogs = [];
|
|
89
|
+
const pendingRequests = new Map();
|
|
90
|
+
|
|
91
|
+
page.cdp.on('Network.requestWillBeSent', (params) => {
|
|
92
|
+
pendingRequests.set(params.requestId, {
|
|
93
|
+
url: params.request.url,
|
|
94
|
+
method: params.request.method,
|
|
95
|
+
timestamp: new Date().toISOString(),
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
page.cdp.on('Network.responseReceived', (params) => {
|
|
100
|
+
const req = pendingRequests.get(params.requestId);
|
|
101
|
+
if (req) {
|
|
102
|
+
networkLogs.push({
|
|
103
|
+
...req,
|
|
104
|
+
status: params.response.status,
|
|
105
|
+
statusText: params.response.statusText,
|
|
106
|
+
mimeType: params.response.mimeType,
|
|
107
|
+
});
|
|
108
|
+
pendingRequests.delete(params.requestId);
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
page.cdp.on('Network.loadingFailed', (params) => {
|
|
113
|
+
const req = pendingRequests.get(params.requestId);
|
|
114
|
+
if (req) {
|
|
115
|
+
networkLogs.push({
|
|
116
|
+
...req,
|
|
117
|
+
status: 0,
|
|
118
|
+
error: params.errorText,
|
|
119
|
+
});
|
|
120
|
+
pendingRequests.delete(params.requestId);
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
// Navigate to initial URL if provided
|
|
125
|
+
if (initialUrl) {
|
|
126
|
+
if (opts.cookies !== false) {
|
|
127
|
+
try { await page.injectCookies(initialUrl, { browser: opts.browser }); } catch { /* no cookies */ }
|
|
128
|
+
}
|
|
129
|
+
await page.goto(initialUrl, opts.timeout ? Number(opts.timeout) : 30000);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Default prune mode
|
|
133
|
+
const defaultPruneMode = opts.pruneMode || 'act';
|
|
134
|
+
|
|
135
|
+
// Command handlers
|
|
136
|
+
const handlers = {
|
|
137
|
+
async goto({ url, timeout }) {
|
|
138
|
+
await page.goto(url, timeout || 30000);
|
|
139
|
+
return { ok: true };
|
|
140
|
+
},
|
|
141
|
+
|
|
142
|
+
async snapshot({ mode }) {
|
|
143
|
+
const pruneMode = mode || defaultPruneMode;
|
|
144
|
+
const text = await page.snapshot({ mode: pruneMode });
|
|
145
|
+
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
146
|
+
const file = join(absDir, `page-${ts}.yml`);
|
|
147
|
+
writeFileSync(file, text);
|
|
148
|
+
return { ok: true, file };
|
|
149
|
+
},
|
|
150
|
+
|
|
151
|
+
async screenshot({ format }) {
|
|
152
|
+
const data = await page.screenshot({ format: format || 'png' });
|
|
153
|
+
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
154
|
+
const ext = format || 'png';
|
|
155
|
+
const file = join(absDir, `screenshot-${ts}.${ext}`);
|
|
156
|
+
writeFileSync(file, Buffer.from(data, 'base64'));
|
|
157
|
+
return { ok: true, file };
|
|
158
|
+
},
|
|
159
|
+
|
|
160
|
+
async click({ ref }) {
|
|
161
|
+
await page.click(String(ref));
|
|
162
|
+
return { ok: true };
|
|
163
|
+
},
|
|
164
|
+
|
|
165
|
+
async type({ ref, text, clear }) {
|
|
166
|
+
await page.type(String(ref), text, clear ? { clear: true } : undefined);
|
|
167
|
+
return { ok: true };
|
|
168
|
+
},
|
|
169
|
+
|
|
170
|
+
async fill({ ref, text }) {
|
|
171
|
+
await page.type(String(ref), text, { clear: true });
|
|
172
|
+
return { ok: true };
|
|
173
|
+
},
|
|
174
|
+
|
|
175
|
+
async press({ key }) {
|
|
176
|
+
await page.press(key);
|
|
177
|
+
return { ok: true };
|
|
178
|
+
},
|
|
179
|
+
|
|
180
|
+
async scroll({ deltaY }) {
|
|
181
|
+
await page.scroll(Number(deltaY));
|
|
182
|
+
return { ok: true };
|
|
183
|
+
},
|
|
184
|
+
|
|
185
|
+
async hover({ ref }) {
|
|
186
|
+
await page.hover(String(ref));
|
|
187
|
+
return { ok: true };
|
|
188
|
+
},
|
|
189
|
+
|
|
190
|
+
async select({ ref, value }) {
|
|
191
|
+
await page.select(String(ref), value);
|
|
192
|
+
return { ok: true };
|
|
193
|
+
},
|
|
194
|
+
|
|
195
|
+
async eval({ expression }) {
|
|
196
|
+
const result = await page.cdp.send('Runtime.evaluate', {
|
|
197
|
+
expression,
|
|
198
|
+
returnByValue: true,
|
|
199
|
+
awaitPromise: true,
|
|
200
|
+
});
|
|
201
|
+
if (result.exceptionDetails) {
|
|
202
|
+
return { ok: false, error: result.exceptionDetails.text || 'eval error' };
|
|
203
|
+
}
|
|
204
|
+
return { ok: true, value: result.result.value };
|
|
205
|
+
},
|
|
206
|
+
|
|
207
|
+
async 'wait-idle'({ timeout }) {
|
|
208
|
+
await page.waitForNetworkIdle({ timeout: timeout || 30000 });
|
|
209
|
+
return { ok: true };
|
|
210
|
+
},
|
|
211
|
+
|
|
212
|
+
async 'wait-nav'({ timeout }) {
|
|
213
|
+
await page.waitForNavigation(timeout || 30000);
|
|
214
|
+
return { ok: true };
|
|
215
|
+
},
|
|
216
|
+
|
|
217
|
+
async 'console-logs'({ level, clear }) {
|
|
218
|
+
let logs = consoleLogs;
|
|
219
|
+
if (level) logs = logs.filter((l) => l.type === level);
|
|
220
|
+
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
221
|
+
const file = join(absDir, `console-${ts}.json`);
|
|
222
|
+
writeFileSync(file, JSON.stringify(logs, null, 2));
|
|
223
|
+
if (clear) consoleLogs.length = 0;
|
|
224
|
+
return { ok: true, file, count: logs.length };
|
|
225
|
+
},
|
|
226
|
+
|
|
227
|
+
async 'network-log'({ failed }) {
|
|
228
|
+
let logs = networkLogs;
|
|
229
|
+
if (failed) logs = logs.filter((l) => l.status === 0 || l.status >= 400);
|
|
230
|
+
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
231
|
+
const file = join(absDir, `network-${ts}.json`);
|
|
232
|
+
writeFileSync(file, JSON.stringify(logs, null, 2));
|
|
233
|
+
return { ok: true, file, count: logs.length };
|
|
234
|
+
},
|
|
235
|
+
|
|
236
|
+
async close() {
|
|
237
|
+
await page.close();
|
|
238
|
+
// Clean up session file
|
|
239
|
+
const sessionPath = join(absDir, SESSION_FILE);
|
|
240
|
+
if (existsSync(sessionPath)) unlinkSync(sessionPath);
|
|
241
|
+
// Respond before exiting
|
|
242
|
+
return { ok: true };
|
|
243
|
+
},
|
|
244
|
+
|
|
245
|
+
async status() {
|
|
246
|
+
return { ok: true, pid: process.pid, uptime: process.uptime() };
|
|
247
|
+
},
|
|
248
|
+
};
|
|
249
|
+
|
|
250
|
+
// Start HTTP server on random port
|
|
251
|
+
const server = createServer(async (req, res) => {
|
|
252
|
+
if (req.method === 'GET' && req.url === '/status') {
|
|
253
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
254
|
+
res.end(JSON.stringify({ ok: true, pid: process.pid }));
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
if (req.method !== 'POST' || req.url !== '/command') {
|
|
259
|
+
res.writeHead(404);
|
|
260
|
+
res.end('Not found');
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
let body = '';
|
|
265
|
+
for await (const chunk of req) body += chunk;
|
|
266
|
+
|
|
267
|
+
let parsed;
|
|
268
|
+
try {
|
|
269
|
+
parsed = JSON.parse(body);
|
|
270
|
+
} catch {
|
|
271
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
272
|
+
res.end(JSON.stringify({ ok: false, error: 'Invalid JSON' }));
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const { command, args } = parsed;
|
|
277
|
+
const handler = handlers[command];
|
|
278
|
+
if (!handler) {
|
|
279
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
280
|
+
res.end(JSON.stringify({ ok: false, error: `Unknown command: ${command}` }));
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
try {
|
|
285
|
+
const result = await handler(args || {});
|
|
286
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
287
|
+
res.end(JSON.stringify(result));
|
|
288
|
+
|
|
289
|
+
// Exit after close command
|
|
290
|
+
if (command === 'close') {
|
|
291
|
+
server.close();
|
|
292
|
+
process.exit(0);
|
|
293
|
+
}
|
|
294
|
+
} catch (err) {
|
|
295
|
+
res.writeHead(500, { 'Content-Type': 'application/json' });
|
|
296
|
+
res.end(JSON.stringify({ ok: false, error: err.message }));
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
await new Promise((resolve) => {
|
|
301
|
+
server.listen(0, '127.0.0.1', () => resolve());
|
|
302
|
+
});
|
|
303
|
+
|
|
304
|
+
const port = server.address().port;
|
|
305
|
+
|
|
306
|
+
// Write session.json so parent/clients can find us
|
|
307
|
+
const sessionPath = join(absDir, SESSION_FILE);
|
|
308
|
+
writeFileSync(sessionPath, JSON.stringify({
|
|
309
|
+
port,
|
|
310
|
+
pid: process.pid,
|
|
311
|
+
startedAt: new Date().toISOString(),
|
|
312
|
+
}));
|
|
313
|
+
|
|
314
|
+
// Handle SIGTERM gracefully
|
|
315
|
+
process.on('SIGTERM', async () => {
|
|
316
|
+
try { await page.close(); } catch { /* already closed */ }
|
|
317
|
+
if (existsSync(sessionPath)) unlinkSync(sessionPath);
|
|
318
|
+
server.close();
|
|
319
|
+
process.exit(0);
|
|
320
|
+
});
|
|
321
|
+
}
|