@glittering0717/ai-browser 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +155 -20
- package/bin/ai-browser.cjs +1 -1
- package/examples/AGENT_INSTRUCTIONS.md +22 -2
- package/llms-install.md +14 -4
- package/package.json +9 -5
- package/scripts/build_dmg.cjs +220 -0
- package/scripts/marketing/data_snapshot.sh +69 -0
- package/scripts/render_icon.cjs +118 -0
- package/src/main/axExtractor.js +103 -36
- package/src/main/error_log.js +245 -0
- package/src/main/index.js +487 -16
- package/src/main/mcp_server.js +39 -1
- package/src/main/mcp_tools.js +64 -10
- package/src/main/network_monitor.js +86 -23
- package/src/main/page_manager.js +1040 -115
- package/src/main/ws_server.js +127 -16
- package/src/preload/actions.cjs +61 -2
- package/src/preload/bridge.cjs +31 -1
- package/src/preload/extractor.cjs +520 -25
- package/src/renderer/error_banner.html +90 -0
- package/src/renderer/error_page.html +125 -0
- package/src/renderer/tab_bar.html +116 -56
- package/src/shared/act_params.cjs +53 -0
- package/src/shared/auth_wall.cjs +162 -0
- package/src/shared/config.js +8 -2
- package/src/shared/deadline.js +6 -0
- package/src/shared/deep_active.cjs +31 -0
- package/src/shared/error_banner.cjs +43 -0
- package/src/shared/find_ref.cjs +44 -0
- package/src/shared/frame_probe.cjs +112 -0
- package/src/shared/hit_point.cjs +80 -0
- package/src/shared/input_verify.js +47 -0
- package/src/shared/key_dispatch.cjs +37 -0
- package/src/shared/nav_wait.cjs +47 -0
- package/src/shared/protocol.js +15 -0
- package/src/shared/ref_guard.cjs +133 -0
- package/src/shared/screenshot_opts.cjs +130 -0
- package/src/shared/tree_diff.cjs +105 -0
- package/src/shared/version.js +1 -1
- package/tools/coverage_probe.cjs +239 -0
package/README.md
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
<h1 align="center">AI Browser</h1>
|
|
2
2
|
|
|
3
|
+
<p align="center">
|
|
4
|
+
<b>中文读者请看这里:<a href="README-CN.md">README-CN.md(3 分钟速读 + 中文安装指引)</a></b>
|
|
5
|
+
</p>
|
|
6
|
+
|
|
3
7
|
<p align="center">
|
|
4
8
|
<b>A real browser whose only operator is your agent.</b><br>
|
|
5
9
|
It reads pages as a semantic tree instead of screenshots, acts through trusted browser input —
|
|
@@ -11,18 +15,24 @@
|
|
|
11
15
|
<img alt="MIT" src="https://img.shields.io/badge/license-MIT-blue.svg">
|
|
12
16
|
<img alt="Electron" src="https://img.shields.io/badge/built%20with-Electron%2033-9cf">
|
|
13
17
|
<img alt="Platforms" src="https://img.shields.io/badge/platforms-macOS%20%E2%80%A2%20Linux-blueviolet">
|
|
14
|
-
<img alt="Tests" src="https://img.shields.io/badge/tests-
|
|
18
|
+
<img alt="Tests" src="https://img.shields.io/badge/tests-658%20unit%20%2B%20447%20e2e-2ea043">
|
|
15
19
|
<img alt="MCP" src="https://img.shields.io/badge/spec-MCP%20(stdio)-f5b23b">
|
|
16
20
|
</p>
|
|
17
21
|
|
|
18
22
|
<p align="center">
|
|
19
|
-
<em>
|
|
20
|
-
<a href="docs/marketing/demo.gif"><img src="docs/marketing/demo.gif" alt="AI Browser demo:
|
|
21
|
-
Reproduce: <code>npm start</code> then <code>node examples/
|
|
23
|
+
<em>A noisy real-world feed (300+ accessibility nodes) becomes a typed structure — then one <code>ui.act</code> switches the feed and the tree changes with it. Every call is a raw JSON-RPC message over one WebSocket: no SDK, no coordinates.</em><br>
|
|
24
|
+
<a href="docs/marketing/demo.gif"><img src="docs/marketing/demo.gif" alt="AI Browser demo: navigate a crowded feed page, get the whole page as one semantic tree, extract typed items, and act on an element id — all over raw WebSocket JSON-RPC" width="860"></a><br>
|
|
25
|
+
Reproduce: <code>npm start</code> then <code>node examples/feed-tree.mjs</code>
|
|
26
|
+
</p>
|
|
27
|
+
|
|
28
|
+
<p align="center">
|
|
29
|
+
<em>Human + agent, same browser, same socket. The agent drives, then simply stops and watches; a person clicks an article in the real window, the agent notices the new tab on its next read, and continues on the page the human chose. No lock, no hand-off API.</em><br>
|
|
30
|
+
<a href="docs/marketing/demo2.gif"><img src="docs/marketing/demo2.gif" alt="Human takeover demo: the agent lists a feed and pauses; a human clicks an article in the real browser window; the agent detects the new tab and resumes scrolling on the chosen page" width="860"></a><br>
|
|
31
|
+
Try it yourself: <code>npm start</code> then <code>node examples/human-takeover.mjs</code> — click anything when it says <em>YOUR TURN</em>.
|
|
22
32
|
</p>
|
|
23
33
|
|
|
24
34
|
**What it is.** An Electron app that exposes its tabs over two equivalent interfaces — an MCP server
|
|
25
|
-
(
|
|
35
|
+
(20 `browse_*` tools) and a raw WebSocket JSON-RPC API (`ws://localhost:9223`, methods `ui.*`). Your
|
|
26
36
|
agent navigates, reads a structured semantic tree (`role` / `label` / `value` / `states` / `actions`),
|
|
27
37
|
and clicks or types by `data-ai-id`. There is no screenshot step and no OCR: `src/` contains no
|
|
28
38
|
capture call at all.
|
|
@@ -38,11 +48,36 @@ is a conversation between the agent and the user, not a browser API.
|
|
|
38
48
|
|
|
39
49
|
## Install
|
|
40
50
|
|
|
41
|
-
|
|
51
|
+
**Install the prebuilt package unless you have a reason not to.** Option A is one file you
|
|
52
|
+
double-click: no Node.js, no dependency tree, no build step. Source install (Option C) pulls a
|
|
53
|
+
~276 MB `node_modules` plus a ~100 MB Electron engine download on top — only use it when no
|
|
54
|
+
package fits your platform, the package misbehaves, or you specifically need to run from source.
|
|
55
|
+
|
|
56
|
+
### Option A — macOS install package (recommended)
|
|
42
57
|
|
|
43
|
-
|
|
58
|
+
Grab `AI-Browser-<version>-arm64.dmg` from
|
|
59
|
+
[GitHub Releases](https://github.com/Glittering/ai-browser/releases) (~110 MB to download,
|
|
60
|
+
~245 MB on disk — most of it is the bundled Chromium engine, which *is* the product). Open the
|
|
61
|
+
dmg, drag **AI Browser** to `/Applications`, launch it. Done.
|
|
44
62
|
|
|
45
|
-
|
|
63
|
+
Two things to know before you download it:
|
|
64
|
+
|
|
65
|
+
- **It is not notarized.** macOS Gatekeeper will refuse to open it on the first try. Right-click
|
|
66
|
+
the app → **Open**, confirm once, and it runs normally afterwards. (A paid Apple Developer
|
|
67
|
+
account is what removes this step; until then, this is the honest cost.)
|
|
68
|
+
- **Only macOS `arm64` is built.** Windows and Linux have no package yet, because neither has
|
|
69
|
+
been verified on this project — see [Compatibility](#compatibility).
|
|
70
|
+
|
|
71
|
+
Agents still connect through the MCP server (Option B below), and the two channels cooperate:
|
|
72
|
+
when the packaged app is already running, the MCP server finds it on port 9223 and **reuses it
|
|
73
|
+
instead of spawning a second copy of Electron** — one browser, one login session, whichever way
|
|
74
|
+
you start it. So "package for the engine + `npx` for the MCP bridge" is the normal agent setup
|
|
75
|
+
too.
|
|
76
|
+
|
|
77
|
+
### Option B — npx (the agent channel)
|
|
78
|
+
|
|
79
|
+
The package is published as [`@glittering0717/ai-browser`](https://www.npmjs.com/package/@glittering0717/ai-browser);
|
|
80
|
+
no clone needed:
|
|
46
81
|
|
|
47
82
|
```bash
|
|
48
83
|
npx -y @glittering0717/ai-browser --help
|
|
@@ -54,16 +89,21 @@ To open the visible browser window directly:
|
|
|
54
89
|
npx -y @glittering0717/ai-browser start
|
|
55
90
|
```
|
|
56
91
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
92
|
+
Requires **Node ≥ 18** and a desktop session (this is a GUI app). If a launch needs its own
|
|
93
|
+
engine, the first run prepares it (~100 MB, downloaded once). npm ≥ 11 blocks dependency install
|
|
94
|
+
scripts by default — the launcher detects that and runs Electron's own downloader itself, so
|
|
95
|
+
nothing extra is needed. If the download from GitHub releases is slow or blocked (common on
|
|
96
|
+
mainland-China networks), point it at a mirror:
|
|
61
97
|
|
|
62
98
|
```bash
|
|
63
99
|
ELECTRON_MIRROR=https://npmmirror.com/mirrors/electron/ npx -y @glittering0717/ai-browser start
|
|
64
100
|
```
|
|
65
101
|
|
|
66
|
-
### Option
|
|
102
|
+
### Option C — from source (fallback)
|
|
103
|
+
|
|
104
|
+
Use this when there is no package for your platform, the package gives you trouble, or you need
|
|
105
|
+
to modify the code. It is the heaviest of the three: `npm install` pulls the full dependency
|
|
106
|
+
tree (~276 MB), then the Electron engine (~100 MB) on top.
|
|
67
107
|
|
|
68
108
|
```bash
|
|
69
109
|
git clone https://github.com/Glittering/ai-browser.git
|
|
@@ -83,8 +123,8 @@ Or make it sticky by creating a `.npmrc` in the project root:
|
|
|
83
123
|
electron_mirror=https://npmmirror.com/mirrors/electron/
|
|
84
124
|
```
|
|
85
125
|
|
|
86
|
-
Confirm the binary actually landed before blaming anything else — a missing binary still leaves
|
|
87
|
-
with an "install that succeeded":
|
|
126
|
+
Confirm the binary actually landed before blaming anything else — a missing binary still leaves
|
|
127
|
+
you with an "install that succeeded":
|
|
88
128
|
|
|
89
129
|
```bash
|
|
90
130
|
ls node_modules/electron/dist
|
|
@@ -99,6 +139,19 @@ npm start
|
|
|
99
139
|
A 1280×800 window opens and the terminal prints `AI Browser WS server listening on ws://localhost:9223`.
|
|
100
140
|
(The first tab opens on the default start page; ignore it and navigate away.)
|
|
101
141
|
|
|
142
|
+
To build the package yourself:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
npm install
|
|
146
|
+
node scripts/build_dmg.cjs # current arch
|
|
147
|
+
node scripts/build_dmg.cjs --arch arm64 # or x64
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Output lands in `dist/`. The script uses `@electron/packager` plus macOS's built-in `hdiutil`
|
|
151
|
+
(no external binary downloads); the reasoning — including why `electron-builder` refuses to build
|
|
152
|
+
this project — is documented in
|
|
153
|
+
[`docs/DEPENDENCY_AND_DISTRIBUTION_AUDIT.md`](docs/DEPENDENCY_AND_DISTRIBUTION_AUDIT.md).
|
|
154
|
+
|
|
102
155
|
---
|
|
103
156
|
|
|
104
157
|
## Connect your agent
|
|
@@ -123,8 +176,9 @@ can instead point `command: "node"` at `/absolute/path/to/ai-browser/src/main/mc
|
|
|
123
176
|
**You do not need to start the browser yourself.** On the first `browse_*` call the MCP server probes
|
|
124
177
|
port 9223; if nothing is listening it spawns Electron detached and waits up to 30 seconds for the
|
|
125
178
|
WebSocket server (the very first call may additionally spend time preparing the engine). If you would
|
|
126
|
-
rather keep one long-lived window shared by every client,
|
|
127
|
-
(or `npm start` from source)
|
|
179
|
+
rather keep one long-lived window shared by every client, launch the **AI Browser** app you installed
|
|
180
|
+
in Option A (or run `npx -y @glittering0717/ai-browser start`, or `npm start` from source) — a second
|
|
181
|
+
launch just focuses the existing window.
|
|
128
182
|
|
|
129
183
|
`examples/AGENT_INSTRUCTIONS.md` has a copy-paste prompt you can drop into a `CLAUDE.md`, Cursor Rule,
|
|
130
184
|
or any system prompt; it teaches the tool loop, the token-saving params, and when to ask a human.
|
|
@@ -357,7 +411,7 @@ MCP is a thin stdio adapter. Anything MCP can do is reachable directly over JSON
|
|
|
357
411
|
| `ui.subscribe` / `ui.unsubscribe` | Event subscription; `params.events`, `"*"` for all | — (WS only — see below) |
|
|
358
412
|
| `ui.downloads` | This tab's downloads with on-disk paths (`state`, `received_bytes`, `dir`) | `browse_downloads` |
|
|
359
413
|
| `ui.get_focused` | The focused element subtree | — |
|
|
360
|
-
| `ui.peek` | Hover a folded/hover-only affordance and return
|
|
414
|
+
| `ui.peek` | Hover a folded/hover-only affordance and return what appeared/disappeared (`revealed` / `hidden`), without committing. Ids are real `data-ai-id`s you can hand straight to `ui.act`; `revertMs: 0` keeps the mouse there so you can click what it found | — |
|
|
361
415
|
| `ui.network_body` | Response body by URL substring (`url_pattern`) | — (superseded by `browse_network`) |
|
|
362
416
|
| `ui.quit` | Graceful shutdown | `browse_quit` |
|
|
363
417
|
|
|
@@ -439,7 +493,9 @@ perturbs the thing it is measuring.
|
|
|
439
493
|
**Writing.** Actions are **trusted browser input**, not synthesized JS. The main process drives the
|
|
440
494
|
page over the Chrome DevTools Protocol — `Input.dispatchMouseEvent` for clicks, and per-character
|
|
441
495
|
`Input.dispatchKeyEvent` (`rawKeyDown → char → keyUp`) for typing — then validates that the click
|
|
442
|
-
actually landed via `elementFromPoint
|
|
496
|
+
actually landed via `elementFromPoint` (piercing shadow roots: the plain call returns the *host* for
|
|
497
|
+
anything inside a web component, which silently downgraded every such click to a synthetic one).
|
|
498
|
+
Because the events arrive as real keyboard and mouse input,
|
|
443
499
|
Draft.js, ProseMirror, CKEditor, Quill, and plain `<textarea>` all work through the same path without
|
|
444
500
|
per-framework branches.
|
|
445
501
|
|
|
@@ -462,7 +518,9 @@ Runnable scripts live in [`examples/`](examples/) and connect straight to `ws://
|
|
|
462
518
|
| [`AGENT_INSTRUCTIONS.md`](examples/AGENT_INSTRUCTIONS.md) | Copy-paste instructions for your agent, incl. the human hand-off protocol. |
|
|
463
519
|
| [`raw-ws.mjs`](examples/raw-ws.mjs) | Smallest possible raw WebSocket client. |
|
|
464
520
|
| [`read-page.mjs`](examples/read-page.mjs) | Load one page, print its interactive elements. |
|
|
521
|
+
| [`feed-tree.mjs`](examples/feed-tree.mjs) | Crowded feed → typed structure → act, all over raw JSON-RPC (top demo). |
|
|
465
522
|
| [`with-human.mjs`](examples/with-human.mjs) | Full flow for when a human has to step in. |
|
|
523
|
+
| [`human-takeover.mjs`](examples/human-takeover.mjs) | Agent pauses, you click, agent resumes on your page (second demo). |
|
|
466
524
|
|
|
467
525
|
```bash
|
|
468
526
|
node examples/read-page.mjs
|
|
@@ -551,6 +609,65 @@ not "logged in" in another:
|
|
|
551
609
|
AI_BROWSER_USER_DATA=~/.ai-browser-work npm start
|
|
552
610
|
```
|
|
553
611
|
|
|
612
|
+
**A red bar appeared at the bottom of the window.** That is a main-process exception — a bug in
|
|
613
|
+
AI Browser itself, not in the page you were driving. It is deliberately *not* a modal dialog: a
|
|
614
|
+
native `dialog.showMessageBox` is an `NSAlert` modal loop on macOS and blocks the whole main process
|
|
615
|
+
(measured: WebSocket calls got 4ms with no dialog vs **8 seconds of no response** with one), which
|
|
616
|
+
turned a self-healing JS exception into "every RPC timed out". The bar shows the message and the log
|
|
617
|
+
path, the process keeps running, and you can dismiss it with `×`.
|
|
618
|
+
|
|
619
|
+
Every exception is appended synchronously to `~/.ai-browser/logs/error.jsonl` (2MB × 4 rolling), with
|
|
620
|
+
the last 20 WebSocket calls attached so you can see *what was being done* when it broke. Read it over
|
|
621
|
+
the API with `ui.errors` / `browse_errors`, or just `tail` the file. A record of `boot` without a
|
|
622
|
+
matching `shutdown` means the process died hard last time.
|
|
623
|
+
|
|
624
|
+
If the exception happened before any window existed, the bar has nowhere to go, so it falls back to a
|
|
625
|
+
native box once (logged as `error_dialog_fallback`) rather than leaving you with a silently dead app.
|
|
626
|
+
Both paths are switchable: `AI_BROWSER_ERROR_BANNER=0` disables the bar, `AI_BROWSER_ERROR_DIALOG=1`
|
|
627
|
+
forces the modal box — the latter blocks the main process, so it is only for staring at a startup
|
|
628
|
+
crash on purpose.
|
|
629
|
+
|
|
630
|
+
### The element is clearly on the page but not in the tree
|
|
631
|
+
|
|
632
|
+
Before you write a site-specific script, work down this list. The reflex — "pin it with a
|
|
633
|
+
`querySelector` chain" — is what produced 142 site-specific scripts and 731 hardcoded selectors in
|
|
634
|
+
this project's own history, and almost none of them turned out to be necessary.
|
|
635
|
+
|
|
636
|
+
| # | Check | How | If it is the cause |
|
|
637
|
+
|---|---|---|---|
|
|
638
|
+
| 1 | Is it a **read-layer** gap? | `node tools/coverage_probe.cjs https://the.site/` | the tree is missing clickable elements → report it; the fix belongs in `src/preload/extractor.cjs`, not in your script |
|
|
639
|
+
| 2 | Is it an **action-layer** gap? | the node is in the tree, `ui.act` fails or does nothing | the element is readable but not reachable (iframe/OOPIF, canvas, trusted input, un-mounted input) → that is a browser-side fix too |
|
|
640
|
+
| 3 | Is it **timing**? | re-read the tree 2–3s later, or `wait_until` | the SPA simply had not rendered when you read it |
|
|
641
|
+
| 4 | Does it only exist **while hovered**? | `ui.peek { target, revertMs: 0 }` | the affordance is hover-rendered. `peek` hovers it and returns what appeared — pass `revertMs: 0` to keep it open, then `ui.act click` the id you got back |
|
|
642
|
+
| 5 | Is it **unnamed**? | `ui.peek`, or click it and read what shows up | an icon-only button the site gave no `aria-label`/`title`/`alt` cannot be named from nothing. But the panel it opens usually is named — so open it and read that instead of guessing which icon is which |
|
|
643
|
+
| 6 | Only now: **pin it yourself** | a selector in *your* script | keep it in your script. Do not encode it as "how this site works" — the next site will not match |
|
|
644
|
+
|
|
645
|
+
`coverage_probe` counts what a human would call click targets — elements the browser itself renders
|
|
646
|
+
with a hand cursor (`cursor:pointer`), where the pointer starts (not every descendant that inherits
|
|
647
|
+
it), and reports how many of them carry a `data-ai-id`:
|
|
648
|
+
|
|
649
|
+
```
|
|
650
|
+
$ node tools/coverage_probe.cjs https://www.csdn.net/
|
|
651
|
+
可点目标(手型边界): 257
|
|
652
|
+
├ 自己带 data-ai-id(直接可操作): 257 → 100%
|
|
653
|
+
├ 被单个交互后代盖满且它在树里(壳子,点内层等价): 0
|
|
654
|
+
├ 祖先歧义(同一祖先下多个手型后代,点祖先指不准): 0
|
|
655
|
+
└ 整条祖先链无 id(真看不见): 0
|
|
656
|
+
可用率(in_tree + by_desc + anc_only): 100 % | 真缺口: 0
|
|
657
|
+
```
|
|
658
|
+
|
|
659
|
+
It exits non-zero below 80% so you can drop it into CI. Measured on 2026-10-17 (Baidu / Bilibili /
|
|
660
|
+
CSDN): **82% / 97% / 88%** before the click-boundary fix, **100% / 100% / 100%** after, at ~40ms
|
|
661
|
+
per `ui.get_tree`.
|
|
662
|
+
|
|
663
|
+
Two things the numbers do **not** cover, so you do not chase them: an element the page gives no
|
|
664
|
+
readable name (image-only links with no `alt` — 53% of Bilibili's clickable nodes) can never get a
|
|
665
|
+
label without inventing one, and a canvas has no DOM to read at all. Those are genuine boundaries,
|
|
666
|
+
not bugs. The full write-up, including why the first measurement of this was wrong by 3× (inherited
|
|
667
|
+
`cursor` inflates the denominator), is in
|
|
668
|
+
[`docs/AGENT_OPERABILITY_FEEDBACK.md`](docs/AGENT_OPERABILITY_FEEDBACK.md) §6.
|
|
669
|
+
|
|
670
|
+
|
|
554
671
|
---
|
|
555
672
|
|
|
556
673
|
## Test & dev
|
|
@@ -565,9 +682,11 @@ npm run value # value truncation contract + read-only get_value — 15 ch
|
|
|
565
682
|
npm run canvas # canvas draw-call capture — 13 checks
|
|
566
683
|
npm run modeldiff # canvas app data-model diff (the "unchanged" false green) — 12 checks
|
|
567
684
|
npm run audit # every advertised action must exist; wait/downloads/permissions/quit — 35 checks
|
|
685
|
+
npm run nav # address bar + load-failure error page (the "white screen" report) — 19 checks
|
|
568
686
|
npm run focus # no focus stealing + full capability while backgrounded — 32 checks
|
|
569
687
|
npm run interact # trusted drag / key press / wheel (pointer gestures) — 31 checks
|
|
570
688
|
npm run view # viewport repaint + snapshot/diff verification — 28 checks
|
|
689
|
+
npm run e2e:all # 一次跑完上面全部套件(每个独立端口)+ 可信汇总 — 428 checks
|
|
571
690
|
npm run realsites # navigate/read/act against a matrix of real sites
|
|
572
691
|
```
|
|
573
692
|
|
|
@@ -575,6 +694,16 @@ All e2e suites boot a genuine Electron against offline fixtures and exit non-zer
|
|
|
575
694
|
Together they cover what jsdom structurally cannot: preload integrity, `ui.evaluate` under a strict
|
|
576
695
|
CSP, CDP click hit-testing, multi-tab lifecycle, event fan-out, and the `evaluate` guard rails.
|
|
577
696
|
|
|
697
|
+
**Each suite proves it is talking to the browser it just started.** Before any assertion runs it
|
|
698
|
+
calls `ui.server_info` and compares the returned `pid` against the child process it spawned; a
|
|
699
|
+
mismatch aborts with exit 4 instead of quietly producing results about somebody else's instance.
|
|
700
|
+
That gap was measured, not assumed: with an impostor WebSocket server holding the port, a suite
|
|
701
|
+
happily sent `ui.new_tab` / `ui.evaluate` / `ui.get_tree` to it and never noticed. Repro:
|
|
702
|
+
`REPRO_PORT=9799 node e2e/_repro_identity.cjs`.
|
|
703
|
+
|
|
704
|
+
`npm run e2e:all` also gives every suite its own port, because they share `/tmp/ai-browser-e2e-<port>`
|
|
705
|
+
as the profile and therefore Electron's single-instance lock too.
|
|
706
|
+
|
|
578
707
|
They all need the WS port free — stop a running `npm start` first, or point them elsewhere:
|
|
579
708
|
|
|
580
709
|
```bash
|
|
@@ -635,7 +764,13 @@ Two things it changed in the product:
|
|
|
635
764
|
the page through `ui.evaluate`.
|
|
636
765
|
|
|
637
766
|
**`npm run network`** — that a POST can be captured with its full request body, that headers are
|
|
638
|
-
redacted by default, and that bodies paginate instead of being silently truncated.
|
|
767
|
+
redacted by default, and that bodies paginate instead of being silently truncated. It also pins the
|
|
768
|
+
eviction rule: the retained byte budget is **global across tabs**, so a background tab quietly
|
|
769
|
+
filling it used to evict the body the agent was about to read on the *active* tab — the request
|
|
770
|
+
really happened and the body really was captured, it just got pushed out by someone else's noise.
|
|
771
|
+
Eviction now sacrifices non-active tabs first (including the background body being stored), and
|
|
772
|
+
`ui.network_configure` reports `active_tab` / `body_bytes` so a future `evicted` can be attributed
|
|
773
|
+
instead of guessed at.
|
|
639
774
|
|
|
640
775
|
**`npm run modeldiff`** — the suite for the one place `ui.diff` used to lie. A canvas-rendered app's
|
|
641
776
|
DOM is a still photograph: dragging a node or editing a widget repaints `<canvas>` and moves
|
package/bin/ai-browser.cjs
CHANGED
|
@@ -19,7 +19,7 @@ const HELP = `ai-browser ${pkg.version}
|
|
|
19
19
|
|
|
20
20
|
Usage:
|
|
21
21
|
ai-browser [start] Launch the visible AI Browser window
|
|
22
|
-
ai-browser mcp Run the MCP stdio server (
|
|
22
|
+
ai-browser mcp Run the MCP stdio server (20 browse_* tools)
|
|
23
23
|
ai-browser --version Print version
|
|
24
24
|
ai-browser --help Show this help
|
|
25
25
|
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
## 中文版(copy-paste)
|
|
16
16
|
|
|
17
17
|
```text
|
|
18
|
-
我可以用 AI Browser 浏览网页。它提供 MCP 服务(
|
|
18
|
+
我可以用 AI Browser 浏览网页。它提供 MCP 服务(20 个 browse_* 工具),第一次调用时会自动
|
|
19
19
|
拉起一个真实可见的浏览器窗口,用完调用 browse_quit 释放。页面以结构化语义树返回,不是截图。
|
|
20
20
|
|
|
21
21
|
标准循环:
|
|
@@ -30,6 +30,8 @@
|
|
|
30
30
|
text_contains / url_contains
|
|
31
31
|
5. browse_evaluate { js } 在页面上下文跑 JS 表达式(上限 5000 字符,
|
|
32
32
|
禁用 process. / require / child_process)
|
|
33
|
+
6. browse_errors 出事了先看这里:主进程异常 / 崩溃 / 导航失败 /
|
|
34
|
+
RPC 报错的最近记录(含调用上下文与日志路径)
|
|
33
35
|
|
|
34
36
|
省 token(browse_get_tree 参数):
|
|
35
37
|
- { "subset": "interactive" } 只保留可交互节点,砍掉纯布局分支 —— 默认就该带上
|
|
@@ -39,6 +41,14 @@
|
|
|
39
41
|
验证写操作(success 不等于做成了):browse_diff 先 snapshot 基线,写操作之后再 diff,
|
|
40
42
|
verdict 显式给出 changed / unchanged。
|
|
41
43
|
|
|
44
|
+
找不到元素 / 点了没反应时的排查顺序(**别一上来就写站点专用选择器**):
|
|
45
|
+
1. 重读 browse_get_tree,或先 browse_wait —— SPA 常常只是还没渲染完
|
|
46
|
+
2. 节点在树里但操作失败 → 是**动作层**问题(跨进程 iframe、canvas、需要受信输入、
|
|
47
|
+
上传控件未挂载),写选择器解决不了
|
|
48
|
+
3. 树里没有、页面上明明有 → 是**读层**缺口,属于浏览器该修的东西;先跑
|
|
49
|
+
tools/coverage_probe.cjs <url> 量化,再考虑绕过
|
|
50
|
+
4. 以上都不成立,才在自己脚本里用选择器兜底;别把"某个站点的选择器"记成通用知识
|
|
51
|
+
|
|
42
52
|
其他:browse_scroll 滚动文档(画布类页面用 browse_act 的 wheel);browse_list_tabs /
|
|
43
53
|
browse_new_tab / browse_close_tab / browse_set_active_tab 管标签页;browse_network 拉取网络
|
|
44
54
|
请求(含 POST body,免订阅);browse_capabilities 一次探明页面该走哪条路(结构化数据 /
|
|
@@ -57,7 +67,7 @@ browse_downloads 查下载文件落盘路径;browse_repaint 在窗口画面陈
|
|
|
57
67
|
## English (copy-paste)
|
|
58
68
|
|
|
59
69
|
```text
|
|
60
|
-
I can browse the web with AI Browser. It exposes an MCP server (
|
|
70
|
+
I can browse the web with AI Browser. It exposes an MCP server (20 browse_* tools), auto-starts a
|
|
61
71
|
real, visible browser window on my first call, and I release it with browse_quit when done. Pages come
|
|
62
72
|
back as a structured semantic tree — no screenshots, no OCR.
|
|
63
73
|
|
|
@@ -82,6 +92,16 @@ Saving tokens (browse_get_tree params):
|
|
|
82
92
|
Verifying writes (success is not evidence): browse_diff snapshots a baseline, then diffs after the
|
|
83
93
|
write and returns an explicit verdict of changed / unchanged.
|
|
84
94
|
|
|
95
|
+
When an element cannot be found, or a click does nothing — work down this list, do **not** reach for a
|
|
96
|
+
site-specific selector first:
|
|
97
|
+
1. re-read with browse_get_tree, or browse_wait — an SPA usually just has not rendered yet
|
|
98
|
+
2. the node is in the tree but the action fails → an **action-layer** gap (cross-process iframe,
|
|
99
|
+
canvas, trusted input, unmounted upload input); a selector will not fix it
|
|
100
|
+
3. the node is missing though the page clearly renders it → a **read-layer** gap, i.e. something the
|
|
101
|
+
browser should fix; quantify it with `tools/coverage_probe.cjs <url>` before working around it
|
|
102
|
+
4. only then pin it with a selector inside your own script — and never record "the selector for site X"
|
|
103
|
+
as general knowledge
|
|
104
|
+
|
|
85
105
|
Also: browse_scroll (document scrolling; use the wheel action on canvas apps); browse_list_tabs /
|
|
86
106
|
browse_new_tab / browse_close_tab / browse_set_active_tab for tabs; browse_network for captured
|
|
87
107
|
requests including POST bodies (no subscription); browse_capabilities to pick a route on an
|
package/llms-install.md
CHANGED
|
@@ -11,18 +11,28 @@ semantic tree (roles / labels / values / actions, stable `data-ai-id` per elemen
|
|
|
11
11
|
not screenshots. Input is trusted CDP keyboard/mouse events. A human can use the
|
|
12
12
|
window's mouse and keyboard at any time (login, QR scan, CAPTCHA, payment).
|
|
13
13
|
|
|
14
|
-
Two equivalent interfaces: **MCP stdio** (
|
|
14
|
+
Two equivalent interfaces: **MCP stdio** (20 `browse_*` tools) and a raw WebSocket
|
|
15
15
|
JSON-RPC API at `ws://localhost:9223` (`ui.*` methods).
|
|
16
16
|
|
|
17
17
|
## Requirements
|
|
18
18
|
|
|
19
19
|
- Node.js **≥ 18**
|
|
20
20
|
- A desktop session (GUI app): macOS or Linux. Windows is experimental/untested.
|
|
21
|
-
- ~
|
|
21
|
+
- ~110 MB download for the Electron engine (prepared on first launch / `npm install`, from GitHub
|
|
22
|
+
releases); it expands to ~245 MB on disk. Do not judge the install by the download size alone.
|
|
23
|
+
|
|
24
|
+
> **Installation priority.** The prebuilt macOS `.dmg` (README "Option A") is the recommended
|
|
25
|
+
> install for humans: one file, no Node.js, no dependency tree. The npx path below is for agents
|
|
26
|
+
> and for machines that already have Node — it is also how the MCP server gets registered, so it
|
|
27
|
+
> stays necessary even when the packaged app is installed. The two cooperate: if the packaged app
|
|
28
|
+
> is already running, the MCP server reuses it on port 9223 instead of downloading a second
|
|
29
|
+
> Electron. Source install is the fallback — only when no package fits, the package misbehaves,
|
|
30
|
+
> or the code itself needs changes; it pulls ~276 MB of dependencies plus the engine.
|
|
22
31
|
|
|
23
32
|
## Install
|
|
24
33
|
|
|
25
|
-
|
|
34
|
+
For agents (this is also the MCP bridge) — npm package
|
|
35
|
+
[`@glittering0717/ai-browser`](https://www.npmjs.com/package/@glittering0717/ai-browser),
|
|
26
36
|
no clone needed:
|
|
27
37
|
|
|
28
38
|
```bash
|
|
@@ -80,7 +90,7 @@ call can take longer if the engine download hasn't happened yet — the progress
|
|
|
80
90
|
A long-lived shared window is also fine — run the `start` command manually; a second launch only
|
|
81
91
|
focuses the existing window (single instance lock).
|
|
82
92
|
|
|
83
|
-
## The
|
|
93
|
+
## The 20 tools
|
|
84
94
|
|
|
85
95
|
- `browse_navigate` — load a URL in the active or given tab.
|
|
86
96
|
- `browse_get_tree` — semantic tree + context (modals, toasts, required-field
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@glittering0717/ai-browser",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"description": "AI-native browser — renders pixels for humans, structured semantic-tree API for agents. No screenshots, no OCR, no coordinate guessing.",
|
|
5
5
|
"main": "src/main/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
"bin",
|
|
11
11
|
"scripts",
|
|
12
12
|
"src",
|
|
13
|
+
"tools/coverage_probe.cjs",
|
|
13
14
|
"examples/AGENT_INSTRUCTIONS.md",
|
|
14
15
|
"examples/mcp-config.json",
|
|
15
16
|
"llms-install.md",
|
|
@@ -60,22 +61,25 @@
|
|
|
60
61
|
"canvas": "node e2e/canvas_ws.cjs",
|
|
61
62
|
"focus": "node e2e/focus_ws.cjs",
|
|
62
63
|
"interact": "node e2e/interact_ws.cjs",
|
|
64
|
+
"upload": "node e2e/upload_ws.cjs",
|
|
63
65
|
"view": "node e2e/view_ws.cjs",
|
|
64
66
|
"realsites": "node e2e/e2e_realsites.cjs",
|
|
65
67
|
"realsites:meta": "vitest run tests/test_realsites_sites.js",
|
|
66
68
|
"mcp": "node src/main/mcp_server.js",
|
|
67
69
|
"modeldiff": "node e2e/modeldiff_ws.cjs",
|
|
68
|
-
"
|
|
70
|
+
"nav": "node e2e/nav_ws.cjs",
|
|
71
|
+
"audit": "node e2e/audit_ws.cjs",
|
|
72
|
+
"e2e:all": "node e2e/run_all.cjs",
|
|
73
|
+
"coverage": "node tools/coverage_probe.cjs"
|
|
69
74
|
},
|
|
70
75
|
"dependencies": {
|
|
71
76
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
72
77
|
"electron": "^33.4.11",
|
|
73
|
-
"jsdom": "^29.1.1",
|
|
74
78
|
"ws": "^8.0.0"
|
|
75
79
|
},
|
|
76
80
|
"devDependencies": {
|
|
77
|
-
"@
|
|
78
|
-
"
|
|
81
|
+
"@electron/packager": "^20.3.0",
|
|
82
|
+
"jsdom": "^29.1.1",
|
|
79
83
|
"vitest": "^2.0.0"
|
|
80
84
|
}
|
|
81
85
|
}
|