@glittering0717/ai-browser 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +136 -19
- package/bin/ai-browser.cjs +1 -1
- package/examples/AGENT_INSTRUCTIONS.md +22 -2
- package/llms-install.md +14 -4
- package/package.json +7 -5
- package/scripts/build_dmg.cjs +220 -0
- package/scripts/marketing/data_snapshot.sh +69 -0
- package/scripts/render_icon.cjs +118 -0
- package/src/main/axExtractor.js +103 -36
- package/src/main/error_log.js +245 -0
- package/src/main/index.js +276 -7
- package/src/main/mcp_server.js +39 -1
- package/src/main/mcp_tools.js +64 -10
- package/src/main/page_manager.js +936 -128
- package/src/main/ws_server.js +73 -12
- package/src/preload/actions.cjs +61 -2
- package/src/preload/bridge.cjs +31 -1
- package/src/preload/extractor.cjs +520 -25
- package/src/renderer/error_banner.html +90 -0
- package/src/shared/act_params.cjs +53 -0
- package/src/shared/auth_wall.cjs +162 -0
- package/src/shared/deadline.js +6 -0
- package/src/shared/deep_active.cjs +31 -0
- package/src/shared/error_banner.cjs +43 -0
- package/src/shared/find_ref.cjs +44 -0
- package/src/shared/frame_probe.cjs +112 -0
- package/src/shared/hit_point.cjs +80 -0
- package/src/shared/input_verify.js +47 -0
- package/src/shared/key_dispatch.cjs +37 -0
- package/src/shared/nav_wait.cjs +47 -0
- package/src/shared/protocol.js +8 -0
- package/src/shared/ref_guard.cjs +133 -0
- package/src/shared/screenshot_opts.cjs +130 -0
- package/src/shared/tree_diff.cjs +105 -0
- package/src/shared/version.js +1 -1
- package/tools/coverage_probe.cjs +239 -0
package/README.md
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
<h1 align="center">AI Browser</h1>
|
|
2
2
|
|
|
3
|
+
<p align="center">
|
|
4
|
+
<b>中文读者请看这里:<a href="README-CN.md">README-CN.md(3 分钟速读 + 中文安装指引)</a></b>
|
|
5
|
+
</p>
|
|
6
|
+
|
|
3
7
|
<p align="center">
|
|
4
8
|
<b>A real browser whose only operator is your agent.</b><br>
|
|
5
9
|
It reads pages as a semantic tree instead of screenshots, acts through trusted browser input —
|
|
@@ -11,18 +15,24 @@
|
|
|
11
15
|
<img alt="MIT" src="https://img.shields.io/badge/license-MIT-blue.svg">
|
|
12
16
|
<img alt="Electron" src="https://img.shields.io/badge/built%20with-Electron%2033-9cf">
|
|
13
17
|
<img alt="Platforms" src="https://img.shields.io/badge/platforms-macOS%20%E2%80%A2%20Linux-blueviolet">
|
|
14
|
-
<img alt="Tests" src="https://img.shields.io/badge/tests-
|
|
18
|
+
<img alt="Tests" src="https://img.shields.io/badge/tests-658%20unit%20%2B%20447%20e2e-2ea043">
|
|
15
19
|
<img alt="MCP" src="https://img.shields.io/badge/spec-MCP%20(stdio)-f5b23b">
|
|
16
20
|
</p>
|
|
17
21
|
|
|
18
22
|
<p align="center">
|
|
19
|
-
<em>
|
|
20
|
-
<a href="docs/marketing/demo.gif"><img src="docs/marketing/demo.gif" alt="AI Browser demo:
|
|
21
|
-
Reproduce: <code>npm start</code> then <code>node examples/
|
|
23
|
+
<em>A noisy real-world feed (300+ accessibility nodes) becomes a typed structure — then one <code>ui.act</code> switches the feed and the tree changes with it. Every call is a raw JSON-RPC message over one WebSocket: no SDK, no coordinates.</em><br>
|
|
24
|
+
<a href="docs/marketing/demo.gif"><img src="docs/marketing/demo.gif" alt="AI Browser demo: navigate a crowded feed page, get the whole page as one semantic tree, extract typed items, and act on an element id — all over raw WebSocket JSON-RPC" width="860"></a><br>
|
|
25
|
+
Reproduce: <code>npm start</code> then <code>node examples/feed-tree.mjs</code>
|
|
26
|
+
</p>
|
|
27
|
+
|
|
28
|
+
<p align="center">
|
|
29
|
+
<em>Human + agent, same browser, same socket. The agent drives, then simply stops and watches; a person clicks an article in the real window, the agent notices the new tab on its next read, and continues on the page the human chose. No lock, no hand-off API.</em><br>
|
|
30
|
+
<a href="docs/marketing/demo2.gif"><img src="docs/marketing/demo2.gif" alt="Human takeover demo: the agent lists a feed and pauses; a human clicks an article in the real browser window; the agent detects the new tab and resumes scrolling on the chosen page" width="860"></a><br>
|
|
31
|
+
Try it yourself: <code>npm start</code> then <code>node examples/human-takeover.mjs</code> — click anything when it says <em>YOUR TURN</em>.
|
|
22
32
|
</p>
|
|
23
33
|
|
|
24
34
|
**What it is.** An Electron app that exposes its tabs over two equivalent interfaces — an MCP server
|
|
25
|
-
(
|
|
35
|
+
(20 `browse_*` tools) and a raw WebSocket JSON-RPC API (`ws://localhost:9223`, methods `ui.*`). Your
|
|
26
36
|
agent navigates, reads a structured semantic tree (`role` / `label` / `value` / `states` / `actions`),
|
|
27
37
|
and clicks or types by `data-ai-id`. There is no screenshot step and no OCR: `src/` contains no
|
|
28
38
|
capture call at all.
|
|
@@ -38,11 +48,36 @@ is a conversation between the agent and the user, not a browser API.
|
|
|
38
48
|
|
|
39
49
|
## Install
|
|
40
50
|
|
|
41
|
-
|
|
51
|
+
**Install the prebuilt package unless you have a reason not to.** Option A is one file you
|
|
52
|
+
double-click: no Node.js, no dependency tree, no build step. Source install (Option C) pulls a
|
|
53
|
+
~276 MB `node_modules` plus a ~100 MB Electron engine download on top — only use it when no
|
|
54
|
+
package fits your platform, the package misbehaves, or you specifically need to run from source.
|
|
55
|
+
|
|
56
|
+
### Option A — macOS install package (recommended)
|
|
42
57
|
|
|
43
|
-
|
|
58
|
+
Grab `AI-Browser-<version>-arm64.dmg` from
|
|
59
|
+
[GitHub Releases](https://github.com/Glittering/ai-browser/releases) (~110 MB to download,
|
|
60
|
+
~245 MB on disk — most of it is the bundled Chromium engine, which *is* the product). Open the
|
|
61
|
+
dmg, drag **AI Browser** to `/Applications`, launch it. Done.
|
|
44
62
|
|
|
45
|
-
|
|
63
|
+
Two things to know before you download it:
|
|
64
|
+
|
|
65
|
+
- **It is not notarized.** macOS Gatekeeper will refuse to open it on the first try. Right-click
|
|
66
|
+
the app → **Open**, confirm once, and it runs normally afterwards. (A paid Apple Developer
|
|
67
|
+
account is what removes this step; until then, this is the honest cost.)
|
|
68
|
+
- **Only macOS `arm64` is built.** Windows and Linux have no package yet, because neither has
|
|
69
|
+
been verified on this project — see [Compatibility](#compatibility).
|
|
70
|
+
|
|
71
|
+
Agents still connect through the MCP server (Option B below), and the two channels cooperate:
|
|
72
|
+
when the packaged app is already running, the MCP server finds it on port 9223 and **reuses it
|
|
73
|
+
instead of spawning a second copy of Electron** — one browser, one login session, whichever way
|
|
74
|
+
you start it. So "package for the engine + `npx` for the MCP bridge" is the normal agent setup
|
|
75
|
+
too.
|
|
76
|
+
|
|
77
|
+
### Option B — npx (the agent channel)
|
|
78
|
+
|
|
79
|
+
The package is published as [`@glittering0717/ai-browser`](https://www.npmjs.com/package/@glittering0717/ai-browser);
|
|
80
|
+
no clone needed:
|
|
46
81
|
|
|
47
82
|
```bash
|
|
48
83
|
npx -y @glittering0717/ai-browser --help
|
|
@@ -54,16 +89,21 @@ To open the visible browser window directly:
|
|
|
54
89
|
npx -y @glittering0717/ai-browser start
|
|
55
90
|
```
|
|
56
91
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
92
|
+
Requires **Node ≥ 18** and a desktop session (this is a GUI app). If a launch needs its own
|
|
93
|
+
engine, the first run prepares it (~100 MB, downloaded once). npm ≥ 11 blocks dependency install
|
|
94
|
+
scripts by default — the launcher detects that and runs Electron's own downloader itself, so
|
|
95
|
+
nothing extra is needed. If the download from GitHub releases is slow or blocked (common on
|
|
96
|
+
mainland-China networks), point it at a mirror:
|
|
61
97
|
|
|
62
98
|
```bash
|
|
63
99
|
ELECTRON_MIRROR=https://npmmirror.com/mirrors/electron/ npx -y @glittering0717/ai-browser start
|
|
64
100
|
```
|
|
65
101
|
|
|
66
|
-
### Option
|
|
102
|
+
### Option C — from source (fallback)
|
|
103
|
+
|
|
104
|
+
Use this when there is no package for your platform, the package gives you trouble, or you need
|
|
105
|
+
to modify the code. It is the heaviest of the three: `npm install` pulls the full dependency
|
|
106
|
+
tree (~276 MB), then the Electron engine (~100 MB) on top.
|
|
67
107
|
|
|
68
108
|
```bash
|
|
69
109
|
git clone https://github.com/Glittering/ai-browser.git
|
|
@@ -83,8 +123,8 @@ Or make it sticky by creating a `.npmrc` in the project root:
|
|
|
83
123
|
electron_mirror=https://npmmirror.com/mirrors/electron/
|
|
84
124
|
```
|
|
85
125
|
|
|
86
|
-
Confirm the binary actually landed before blaming anything else — a missing binary still leaves
|
|
87
|
-
with an "install that succeeded":
|
|
126
|
+
Confirm the binary actually landed before blaming anything else — a missing binary still leaves
|
|
127
|
+
you with an "install that succeeded":
|
|
88
128
|
|
|
89
129
|
```bash
|
|
90
130
|
ls node_modules/electron/dist
|
|
@@ -99,6 +139,19 @@ npm start
|
|
|
99
139
|
A 1280×800 window opens and the terminal prints `AI Browser WS server listening on ws://localhost:9223`.
|
|
100
140
|
(The first tab opens on the default start page; ignore it and navigate away.)
|
|
101
141
|
|
|
142
|
+
To build the package yourself:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
npm install
|
|
146
|
+
node scripts/build_dmg.cjs # current arch
|
|
147
|
+
node scripts/build_dmg.cjs --arch arm64 # or x64
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Output lands in `dist/`. The script uses `@electron/packager` plus macOS's built-in `hdiutil`
|
|
151
|
+
(no external binary downloads); the reasoning — including why `electron-builder` refuses to build
|
|
152
|
+
this project — is documented in
|
|
153
|
+
[`docs/DEPENDENCY_AND_DISTRIBUTION_AUDIT.md`](docs/DEPENDENCY_AND_DISTRIBUTION_AUDIT.md).
|
|
154
|
+
|
|
102
155
|
---
|
|
103
156
|
|
|
104
157
|
## Connect your agent
|
|
@@ -123,8 +176,9 @@ can instead point `command: "node"` at `/absolute/path/to/ai-browser/src/main/mc
|
|
|
123
176
|
**You do not need to start the browser yourself.** On the first `browse_*` call the MCP server probes
|
|
124
177
|
port 9223; if nothing is listening it spawns Electron detached and waits up to 30 seconds for the
|
|
125
178
|
WebSocket server (the very first call may additionally spend time preparing the engine). If you would
|
|
126
|
-
rather keep one long-lived window shared by every client,
|
|
127
|
-
(or `npm start` from source)
|
|
179
|
+
rather keep one long-lived window shared by every client, launch the **AI Browser** app you installed
|
|
180
|
+
in Option A (or run `npx -y @glittering0717/ai-browser start`, or `npm start` from source) — a second
|
|
181
|
+
launch just focuses the existing window.
|
|
128
182
|
|
|
129
183
|
`examples/AGENT_INSTRUCTIONS.md` has a copy-paste prompt you can drop into a `CLAUDE.md`, Cursor Rule,
|
|
130
184
|
or any system prompt; it teaches the tool loop, the token-saving params, and when to ask a human.
|
|
@@ -357,7 +411,7 @@ MCP is a thin stdio adapter. Anything MCP can do is reachable directly over JSON
|
|
|
357
411
|
| `ui.subscribe` / `ui.unsubscribe` | Event subscription; `params.events`, `"*"` for all | — (WS only — see below) |
|
|
358
412
|
| `ui.downloads` | This tab's downloads with on-disk paths (`state`, `received_bytes`, `dir`) | `browse_downloads` |
|
|
359
413
|
| `ui.get_focused` | The focused element subtree | — |
|
|
360
|
-
| `ui.peek` | Hover a folded/hover-only affordance and return
|
|
414
|
+
| `ui.peek` | Hover a folded/hover-only affordance and return what appeared/disappeared (`revealed` / `hidden`), without committing. Ids are real `data-ai-id`s you can hand straight to `ui.act`; `revertMs: 0` keeps the mouse there so you can click what it found | — |
|
|
361
415
|
| `ui.network_body` | Response body by URL substring (`url_pattern`) | — (superseded by `browse_network`) |
|
|
362
416
|
| `ui.quit` | Graceful shutdown | `browse_quit` |
|
|
363
417
|
|
|
@@ -439,7 +493,9 @@ perturbs the thing it is measuring.
|
|
|
439
493
|
**Writing.** Actions are **trusted browser input**, not synthesized JS. The main process drives the
|
|
440
494
|
page over the Chrome DevTools Protocol — `Input.dispatchMouseEvent` for clicks, and per-character
|
|
441
495
|
`Input.dispatchKeyEvent` (`rawKeyDown → char → keyUp`) for typing — then validates that the click
|
|
442
|
-
actually landed via `elementFromPoint
|
|
496
|
+
actually landed via `elementFromPoint` (piercing shadow roots: the plain call returns the *host* for
|
|
497
|
+
anything inside a web component, which silently downgraded every such click to a synthetic one).
|
|
498
|
+
Because the events arrive as real keyboard and mouse input,
|
|
443
499
|
Draft.js, ProseMirror, CKEditor, Quill, and plain `<textarea>` all work through the same path without
|
|
444
500
|
per-framework branches.
|
|
445
501
|
|
|
@@ -462,7 +518,9 @@ Runnable scripts live in [`examples/`](examples/) and connect straight to `ws://
|
|
|
462
518
|
| [`AGENT_INSTRUCTIONS.md`](examples/AGENT_INSTRUCTIONS.md) | Copy-paste instructions for your agent, incl. the human hand-off protocol. |
|
|
463
519
|
| [`raw-ws.mjs`](examples/raw-ws.mjs) | Smallest possible raw WebSocket client. |
|
|
464
520
|
| [`read-page.mjs`](examples/read-page.mjs) | Load one page, print its interactive elements. |
|
|
521
|
+
| [`feed-tree.mjs`](examples/feed-tree.mjs) | Crowded feed → typed structure → act, all over raw JSON-RPC (top demo). |
|
|
465
522
|
| [`with-human.mjs`](examples/with-human.mjs) | Full flow for when a human has to step in. |
|
|
523
|
+
| [`human-takeover.mjs`](examples/human-takeover.mjs) | Agent pauses, you click, agent resumes on your page (second demo). |
|
|
466
524
|
|
|
467
525
|
```bash
|
|
468
526
|
node examples/read-page.mjs
|
|
@@ -551,6 +609,65 @@ not "logged in" in another:
|
|
|
551
609
|
AI_BROWSER_USER_DATA=~/.ai-browser-work npm start
|
|
552
610
|
```
|
|
553
611
|
|
|
612
|
+
**A red bar appeared at the bottom of the window.** That is a main-process exception — a bug in
|
|
613
|
+
AI Browser itself, not in the page you were driving. It is deliberately *not* a modal dialog: a
|
|
614
|
+
native `dialog.showMessageBox` is an `NSAlert` modal loop on macOS and blocks the whole main process
|
|
615
|
+
(measured: WebSocket calls got 4ms with no dialog vs **8 seconds of no response** with one), which
|
|
616
|
+
turned a self-healing JS exception into "every RPC timed out". The bar shows the message and the log
|
|
617
|
+
path, the process keeps running, and you can dismiss it with `×`.
|
|
618
|
+
|
|
619
|
+
Every exception is appended synchronously to `~/.ai-browser/logs/error.jsonl` (2MB × 4 rolling), with
|
|
620
|
+
the last 20 WebSocket calls attached so you can see *what was being done* when it broke. Read it over
|
|
621
|
+
the API with `ui.errors` / `browse_errors`, or just `tail` the file. A record of `boot` without a
|
|
622
|
+
matching `shutdown` means the process died hard last time.
|
|
623
|
+
|
|
624
|
+
If the exception happened before any window existed, the bar has nowhere to go, so it falls back to a
|
|
625
|
+
native box once (logged as `error_dialog_fallback`) rather than leaving you with a silently dead app.
|
|
626
|
+
Both paths are switchable: `AI_BROWSER_ERROR_BANNER=0` disables the bar, `AI_BROWSER_ERROR_DIALOG=1`
|
|
627
|
+
forces the modal box — the latter blocks the main process, so it is only for staring at a startup
|
|
628
|
+
crash on purpose.
|
|
629
|
+
|
|
630
|
+
### The element is clearly on the page but not in the tree
|
|
631
|
+
|
|
632
|
+
Before you write a site-specific script, work down this list. The reflex — "pin it with a
|
|
633
|
+
`querySelector` chain" — is what produced 142 site-specific scripts and 731 hardcoded selectors in
|
|
634
|
+
this project's own history, and almost none of them turned out to be necessary.
|
|
635
|
+
|
|
636
|
+
| # | Check | How | If it is the cause |
|
|
637
|
+
|---|---|---|---|
|
|
638
|
+
| 1 | Is it a **read-layer** gap? | `node tools/coverage_probe.cjs https://the.site/` | the tree is missing clickable elements → report it; the fix belongs in `src/preload/extractor.cjs`, not in your script |
|
|
639
|
+
| 2 | Is it an **action-layer** gap? | the node is in the tree, `ui.act` fails or does nothing | the element is readable but not reachable (iframe/OOPIF, canvas, trusted input, un-mounted input) → that is a browser-side fix too |
|
|
640
|
+
| 3 | Is it **timing**? | re-read the tree 2–3s later, or `wait_until` | the SPA simply had not rendered when you read it |
|
|
641
|
+
| 4 | Does it only exist **while hovered**? | `ui.peek { target, revertMs: 0 }` | the affordance is hover-rendered. `peek` hovers it and returns what appeared — pass `revertMs: 0` to keep it open, then `ui.act click` the id you got back |
|
|
642
|
+
| 5 | Is it **unnamed**? | `ui.peek`, or click it and read what shows up | an icon-only button the site gave no `aria-label`/`title`/`alt` cannot be named from nothing. But the panel it opens usually is named — so open it and read that instead of guessing which icon is which |
|
|
643
|
+
| 6 | Only now: **pin it yourself** | a selector in *your* script | keep it in your script. Do not encode it as "how this site works" — the next site will not match |
|
|
644
|
+
|
|
645
|
+
`coverage_probe` counts what a human would call click targets — elements the browser itself renders
|
|
646
|
+
with a hand cursor (`cursor:pointer`), where the pointer starts (not every descendant that inherits
|
|
647
|
+
it), and reports how many of them carry a `data-ai-id`:
|
|
648
|
+
|
|
649
|
+
```
|
|
650
|
+
$ node tools/coverage_probe.cjs https://www.csdn.net/
|
|
651
|
+
可点目标(手型边界): 257
|
|
652
|
+
├ 自己带 data-ai-id(直接可操作): 257 → 100%
|
|
653
|
+
├ 被单个交互后代盖满且它在树里(壳子,点内层等价): 0
|
|
654
|
+
├ 祖先歧义(同一祖先下多个手型后代,点祖先指不准): 0
|
|
655
|
+
└ 整条祖先链无 id(真看不见): 0
|
|
656
|
+
可用率(in_tree + by_desc + anc_only): 100 % | 真缺口: 0
|
|
657
|
+
```
|
|
658
|
+
|
|
659
|
+
It exits non-zero below 80% so you can drop it into CI. Measured on 2026-10-17 (Baidu / Bilibili /
|
|
660
|
+
CSDN): **82% / 97% / 88%** before the click-boundary fix, **100% / 100% / 100%** after, at ~40ms
|
|
661
|
+
per `ui.get_tree`.
|
|
662
|
+
|
|
663
|
+
Two things the numbers do **not** cover, so you do not chase them: an element the page gives no
|
|
664
|
+
readable name (image-only links with no `alt` — 53% of Bilibili's clickable nodes) can never get a
|
|
665
|
+
label without inventing one, and a canvas has no DOM to read at all. Those are genuine boundaries,
|
|
666
|
+
not bugs. The full write-up, including why the first measurement of this was wrong by 3× (inherited
|
|
667
|
+
`cursor` inflates the denominator), is in
|
|
668
|
+
[`docs/AGENT_OPERABILITY_FEEDBACK.md`](docs/AGENT_OPERABILITY_FEEDBACK.md) §6.
|
|
669
|
+
|
|
670
|
+
|
|
554
671
|
---
|
|
555
672
|
|
|
556
673
|
## Test & dev
|
package/bin/ai-browser.cjs
CHANGED
|
@@ -19,7 +19,7 @@ const HELP = `ai-browser ${pkg.version}
|
|
|
19
19
|
|
|
20
20
|
Usage:
|
|
21
21
|
ai-browser [start] Launch the visible AI Browser window
|
|
22
|
-
ai-browser mcp Run the MCP stdio server (
|
|
22
|
+
ai-browser mcp Run the MCP stdio server (20 browse_* tools)
|
|
23
23
|
ai-browser --version Print version
|
|
24
24
|
ai-browser --help Show this help
|
|
25
25
|
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
## 中文版(copy-paste)
|
|
16
16
|
|
|
17
17
|
```text
|
|
18
|
-
我可以用 AI Browser 浏览网页。它提供 MCP 服务(
|
|
18
|
+
我可以用 AI Browser 浏览网页。它提供 MCP 服务(20 个 browse_* 工具),第一次调用时会自动
|
|
19
19
|
拉起一个真实可见的浏览器窗口,用完调用 browse_quit 释放。页面以结构化语义树返回,不是截图。
|
|
20
20
|
|
|
21
21
|
标准循环:
|
|
@@ -30,6 +30,8 @@
|
|
|
30
30
|
text_contains / url_contains
|
|
31
31
|
5. browse_evaluate { js } 在页面上下文跑 JS 表达式(上限 5000 字符,
|
|
32
32
|
禁用 process. / require / child_process)
|
|
33
|
+
6. browse_errors 出事了先看这里:主进程异常 / 崩溃 / 导航失败 /
|
|
34
|
+
RPC 报错的最近记录(含调用上下文与日志路径)
|
|
33
35
|
|
|
34
36
|
省 token(browse_get_tree 参数):
|
|
35
37
|
- { "subset": "interactive" } 只保留可交互节点,砍掉纯布局分支 —— 默认就该带上
|
|
@@ -39,6 +41,14 @@
|
|
|
39
41
|
验证写操作(success 不等于做成了):browse_diff 先 snapshot 基线,写操作之后再 diff,
|
|
40
42
|
verdict 显式给出 changed / unchanged。
|
|
41
43
|
|
|
44
|
+
找不到元素 / 点了没反应时的排查顺序(**别一上来就写站点专用选择器**):
|
|
45
|
+
1. 重读 browse_get_tree,或先 browse_wait —— SPA 常常只是还没渲染完
|
|
46
|
+
2. 节点在树里但操作失败 → 是**动作层**问题(跨进程 iframe、canvas、需要受信输入、
|
|
47
|
+
上传控件未挂载),写选择器解决不了
|
|
48
|
+
3. 树里没有、页面上明明有 → 是**读层**缺口,属于浏览器该修的东西;先跑
|
|
49
|
+
tools/coverage_probe.cjs <url> 量化,再考虑绕过
|
|
50
|
+
4. 以上都不成立,才在自己脚本里用选择器兜底;别把"某个站点的选择器"记成通用知识
|
|
51
|
+
|
|
42
52
|
其他:browse_scroll 滚动文档(画布类页面用 browse_act 的 wheel);browse_list_tabs /
|
|
43
53
|
browse_new_tab / browse_close_tab / browse_set_active_tab 管标签页;browse_network 拉取网络
|
|
44
54
|
请求(含 POST body,免订阅);browse_capabilities 一次探明页面该走哪条路(结构化数据 /
|
|
@@ -57,7 +67,7 @@ browse_downloads 查下载文件落盘路径;browse_repaint 在窗口画面陈
|
|
|
57
67
|
## English (copy-paste)
|
|
58
68
|
|
|
59
69
|
```text
|
|
60
|
-
I can browse the web with AI Browser. It exposes an MCP server (
|
|
70
|
+
I can browse the web with AI Browser. It exposes an MCP server (20 browse_* tools), auto-starts a
|
|
61
71
|
real, visible browser window on my first call, and I release it with browse_quit when done. Pages come
|
|
62
72
|
back as a structured semantic tree — no screenshots, no OCR.
|
|
63
73
|
|
|
@@ -82,6 +92,16 @@ Saving tokens (browse_get_tree params):
|
|
|
82
92
|
Verifying writes (success is not evidence): browse_diff snapshots a baseline, then diffs after the
|
|
83
93
|
write and returns an explicit verdict of changed / unchanged.
|
|
84
94
|
|
|
95
|
+
When an element cannot be found, or a click does nothing — work down this list, do **not** reach for a
|
|
96
|
+
site-specific selector first:
|
|
97
|
+
1. re-read with browse_get_tree, or browse_wait — an SPA usually just has not rendered yet
|
|
98
|
+
2. the node is in the tree but the action fails → an **action-layer** gap (cross-process iframe,
|
|
99
|
+
canvas, trusted input, unmounted upload input); a selector will not fix it
|
|
100
|
+
3. the node is missing though the page clearly renders it → a **read-layer** gap, i.e. something the
|
|
101
|
+
browser should fix; quantify it with `tools/coverage_probe.cjs <url>` before working around it
|
|
102
|
+
4. only then pin it with a selector inside your own script — and never record "the selector for site X"
|
|
103
|
+
as general knowledge
|
|
104
|
+
|
|
85
105
|
Also: browse_scroll (document scrolling; use the wheel action on canvas apps); browse_list_tabs /
|
|
86
106
|
browse_new_tab / browse_close_tab / browse_set_active_tab for tabs; browse_network for captured
|
|
87
107
|
requests including POST bodies (no subscription); browse_capabilities to pick a route on an
|
package/llms-install.md
CHANGED
|
@@ -11,18 +11,28 @@ semantic tree (roles / labels / values / actions, stable `data-ai-id` per elemen
|
|
|
11
11
|
not screenshots. Input is trusted CDP keyboard/mouse events. A human can use the
|
|
12
12
|
window's mouse and keyboard at any time (login, QR scan, CAPTCHA, payment).
|
|
13
13
|
|
|
14
|
-
Two equivalent interfaces: **MCP stdio** (
|
|
14
|
+
Two equivalent interfaces: **MCP stdio** (20 `browse_*` tools) and a raw WebSocket
|
|
15
15
|
JSON-RPC API at `ws://localhost:9223` (`ui.*` methods).
|
|
16
16
|
|
|
17
17
|
## Requirements
|
|
18
18
|
|
|
19
19
|
- Node.js **≥ 18**
|
|
20
20
|
- A desktop session (GUI app): macOS or Linux. Windows is experimental/untested.
|
|
21
|
-
- ~
|
|
21
|
+
- ~110 MB download for the Electron engine (prepared on first launch / `npm install`, from GitHub
|
|
22
|
+
releases); it expands to ~245 MB on disk. Do not judge the install by the download size alone.
|
|
23
|
+
|
|
24
|
+
> **Installation priority.** The prebuilt macOS `.dmg` (README "Option A") is the recommended
|
|
25
|
+
> install for humans: one file, no Node.js, no dependency tree. The npx path below is for agents
|
|
26
|
+
> and for machines that already have Node — it is also how the MCP server gets registered, so it
|
|
27
|
+
> stays necessary even when the packaged app is installed. The two cooperate: if the packaged app
|
|
28
|
+
> is already running, the MCP server reuses it on port 9223 instead of downloading a second
|
|
29
|
+
> Electron. Source install is the fallback — only when no package fits, the package misbehaves,
|
|
30
|
+
> or the code itself needs changes; it pulls ~276 MB of dependencies plus the engine.
|
|
22
31
|
|
|
23
32
|
## Install
|
|
24
33
|
|
|
25
|
-
|
|
34
|
+
For agents (this is also the MCP bridge) — npm package
|
|
35
|
+
[`@glittering0717/ai-browser`](https://www.npmjs.com/package/@glittering0717/ai-browser),
|
|
26
36
|
no clone needed:
|
|
27
37
|
|
|
28
38
|
```bash
|
|
@@ -80,7 +90,7 @@ call can take longer if the engine download hasn't happened yet — the progress
|
|
|
80
90
|
A long-lived shared window is also fine — run the `start` command manually; a second launch only
|
|
81
91
|
focuses the existing window (single instance lock).
|
|
82
92
|
|
|
83
|
-
## The
|
|
93
|
+
## The 20 tools
|
|
84
94
|
|
|
85
95
|
- `browse_navigate` — load a URL in the active or given tab.
|
|
86
96
|
- `browse_get_tree` — semantic tree + context (modals, toasts, required-field
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@glittering0717/ai-browser",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"description": "AI-native browser — renders pixels for humans, structured semantic-tree API for agents. No screenshots, no OCR, no coordinate guessing.",
|
|
5
5
|
"main": "src/main/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
"bin",
|
|
11
11
|
"scripts",
|
|
12
12
|
"src",
|
|
13
|
+
"tools/coverage_probe.cjs",
|
|
13
14
|
"examples/AGENT_INSTRUCTIONS.md",
|
|
14
15
|
"examples/mcp-config.json",
|
|
15
16
|
"llms-install.md",
|
|
@@ -60,6 +61,7 @@
|
|
|
60
61
|
"canvas": "node e2e/canvas_ws.cjs",
|
|
61
62
|
"focus": "node e2e/focus_ws.cjs",
|
|
62
63
|
"interact": "node e2e/interact_ws.cjs",
|
|
64
|
+
"upload": "node e2e/upload_ws.cjs",
|
|
63
65
|
"view": "node e2e/view_ws.cjs",
|
|
64
66
|
"realsites": "node e2e/e2e_realsites.cjs",
|
|
65
67
|
"realsites:meta": "vitest run tests/test_realsites_sites.js",
|
|
@@ -67,17 +69,17 @@
|
|
|
67
69
|
"modeldiff": "node e2e/modeldiff_ws.cjs",
|
|
68
70
|
"nav": "node e2e/nav_ws.cjs",
|
|
69
71
|
"audit": "node e2e/audit_ws.cjs",
|
|
70
|
-
"e2e:all": "node e2e/run_all.cjs"
|
|
72
|
+
"e2e:all": "node e2e/run_all.cjs",
|
|
73
|
+
"coverage": "node tools/coverage_probe.cjs"
|
|
71
74
|
},
|
|
72
75
|
"dependencies": {
|
|
73
76
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
74
77
|
"electron": "^33.4.11",
|
|
75
|
-
"jsdom": "^29.1.1",
|
|
76
78
|
"ws": "^8.0.0"
|
|
77
79
|
},
|
|
78
80
|
"devDependencies": {
|
|
79
|
-
"@
|
|
80
|
-
"
|
|
81
|
+
"@electron/packager": "^20.3.0",
|
|
82
|
+
"jsdom": "^29.1.1",
|
|
81
83
|
"vitest": "^2.0.0"
|
|
82
84
|
}
|
|
83
85
|
}
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// scripts/build_dmg.cjs — 把 AI Browser 打成 macOS .app,再压成 .dmg。
|
|
3
|
+
//
|
|
4
|
+
// # 为什么用 @electron/packager + hdiutil,而不是 electron-builder
|
|
5
|
+
//
|
|
6
|
+
// electron-builder 有一条硬校验(app-builder-lib/out/util/packageMetadata.js):
|
|
7
|
+
// const deps = ["electron", "electron-prebuilt", "electron-rebuild"];
|
|
8
|
+
// for (const name of deps) if (name in dependencies) errors.push(...)
|
|
9
|
+
// `electron` 出现在 dependencies 里就直接报错,**没有任何环境变量能绕过**
|
|
10
|
+
// (ALLOW_ELECTRON_BUILDER_AS_PRODUCTION_DEPENDENCY 只管 electron-builder 自己)。
|
|
11
|
+
//
|
|
12
|
+
// 这条校验的假设是"Electron 应用不该把 electron 当运行时依赖分发"。而本项目的
|
|
13
|
+
// npm 渠道恰恰是这么设计的:`npx -y @glittering0717/ai-browser mcp` 要把 electron
|
|
14
|
+
// 装进 node_modules 再拉起来。所以两个渠道对 package.json 的要求是冲突的。
|
|
15
|
+
//
|
|
16
|
+
// 眼前的产物只是一个**未签名、不自动更新**的 dmg,electron-builder 的核心卖点
|
|
17
|
+
// (签名/公证/自动更新/NSIS/Squirrel)一个都用不上,却要为此引入架构冲突和
|
|
18
|
+
// 一堆从 GitHub 下载的二进制。@electron/packager 只做"打包"这一件事,
|
|
19
|
+
// hdiutil 是 macOS 自带,不需要下载任何外部工具。
|
|
20
|
+
//
|
|
21
|
+
// 如果将来真要签名+公证+自动更新,正确的做法是改用 electron-builder 官方推荐的
|
|
22
|
+
// 双 package.json 布局(根目录放开发依赖,app/ 子目录放应用清单),而不是在这里
|
|
23
|
+
// 加 hack。
|
|
24
|
+
//
|
|
25
|
+
// # 用法
|
|
26
|
+
// node scripts/build_dmg.cjs # 打当前架构
|
|
27
|
+
// node scripts/build_dmg.cjs --arch arm64 # 指定架构
|
|
28
|
+
// node scripts/build_dmg.cjs --skip-dmg # 只出 .app,不压 dmg
|
|
29
|
+
'use strict';
|
|
30
|
+
|
|
31
|
+
const { spawnSync } = require('node:child_process');
|
|
32
|
+
const fs = require('node:fs');
|
|
33
|
+
const os = require('node:os');
|
|
34
|
+
const path = require('node:path');
|
|
35
|
+
|
|
36
|
+
const root = path.resolve(__dirname, '..');
|
|
37
|
+
const pkg = require('../package.json');
|
|
38
|
+
|
|
39
|
+
const APP_NAME = 'AI Browser';
|
|
40
|
+
const OUT_DIR = path.join(root, 'dist');
|
|
41
|
+
|
|
42
|
+
function parseArgs(argv) {
|
|
43
|
+
const args = { arch: process.arch, skipDmg: false };
|
|
44
|
+
for (let i = 0; i < argv.length; i++) {
|
|
45
|
+
if (argv[i] === '--arch') args.arch = argv[++i];
|
|
46
|
+
else if (argv[i] === '--skip-dmg') args.skipDmg = true;
|
|
47
|
+
else {
|
|
48
|
+
console.error(`[build] unknown argument: ${argv[i]}`);
|
|
49
|
+
process.exit(2);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return args;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// 黑名单。用黑名单而不是白名单是因为 packager 只有 ignore 这一个过滤手段,
|
|
56
|
+
// 所以每一条都要写清"为什么它不该进安装包"——
|
|
57
|
+
// outputs/ 28MB 截图证据,纯本地产物
|
|
58
|
+
// docs/ 内部文档,含营销资料
|
|
59
|
+
// e2e/ tests/ 测试代码与夹具
|
|
60
|
+
// tools/_oneoff/ 一次性调试脚本(dep_closure / peek_* 等)
|
|
61
|
+
// dist/ 上一次的构建产物
|
|
62
|
+
// .git/ 版本库
|
|
63
|
+
// node_modules/.cache|.bin npm 缓存与可执行文件软链
|
|
64
|
+
// node_modules/electron 243MB 的 Electron 发行版——打包后的可执行文件
|
|
65
|
+
// **就是** Electron,asar 里再放一份纯属浪费;src 里所有
|
|
66
|
+
// `from 'electron'` 都由 Electron 内置模块系统解析,不经过
|
|
67
|
+
// node_modules。(这一点和 electron-builder 的默认行为一致。)
|
|
68
|
+
// build/ 图标母版与中间产物。打包只需要 build/icon.icns 这一个
|
|
69
|
+
// 文件(由下面的 icon 选项显式指定,从源目录读取),
|
|
70
|
+
// 目录里的 svg/png/iconset 都不该随包分发;src 里也没有
|
|
71
|
+
// 任何地方引用 build/。
|
|
72
|
+
const IGNORE = new RegExp(
|
|
73
|
+
'(^|\\/)(\\.git|outputs|docs|e2e|tests|dist|build|node_modules\\/\\.cache|node_modules\\/\\.bin|node_modules\\/electron|tools\\/_oneoff)(\\/|$)'
|
|
74
|
+
);
|
|
75
|
+
|
|
76
|
+
function run(cmd, args, opts = {}) {
|
|
77
|
+
const r = spawnSync(cmd, args, { cwd: root, stdio: 'inherit', ...opts });
|
|
78
|
+
if (r.error) throw new Error(`${cmd} failed to start: ${r.error.message}`);
|
|
79
|
+
if (r.status !== 0) throw new Error(`${cmd} exited with code ${r.status}`);
|
|
80
|
+
return r;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function dirSize(p) {
|
|
84
|
+
let total = 0;
|
|
85
|
+
const stack = [p];
|
|
86
|
+
while (stack.length) {
|
|
87
|
+
const cur = stack.pop();
|
|
88
|
+
let entries;
|
|
89
|
+
try { entries = fs.readdirSync(cur, { withFileTypes: true }); } catch (e) { continue; }
|
|
90
|
+
for (const e of entries) {
|
|
91
|
+
const full = path.join(cur, e.name);
|
|
92
|
+
if (e.isDirectory()) stack.push(full);
|
|
93
|
+
else { try { total += fs.statSync(full).size; } catch (_) {} }
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return total;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function mb(bytes) {
|
|
100
|
+
return (bytes / 1024 / 1024).toFixed(1) + ' MB';
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// @electron/packager 默认走 downloadElectronZip 从 GitHub 拉 Electron zip。
|
|
104
|
+
// 本机到 GitHub 的连接不稳定(间歇性 502 / fetch failed),而
|
|
105
|
+
// ~/Library/Caches/electron/<hash>/ 下通常已经有下载好的 zip。
|
|
106
|
+
// 这里探测缓存:有对应版本的 zip 就用 electronZipDir 指过去(packager 的
|
|
107
|
+
// getElectronZipPath 见到这个选项就直接用本地文件,完全不联网),
|
|
108
|
+
// 没有才回落到在线下载。注意 electronZipDir 要求目录里躺着
|
|
109
|
+
// electron-v<version>-<platform>-<arch>.zip。
|
|
110
|
+
function findElectronZipDir(version, platform, arch) {
|
|
111
|
+
const cacheRoot = path.join(os.homedir(), 'Library', 'Caches', 'electron');
|
|
112
|
+
const zipName = `electron-v${version}-${platform}-${arch}.zip`;
|
|
113
|
+
let entries = [];
|
|
114
|
+
try {
|
|
115
|
+
entries = fs.readdirSync(cacheRoot, { withFileTypes: true });
|
|
116
|
+
} catch (_) {
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
for (const e of entries) {
|
|
120
|
+
if (!e.isDirectory()) continue;
|
|
121
|
+
const dir = path.join(cacheRoot, e.name);
|
|
122
|
+
if (fs.existsSync(path.join(dir, zipName))) return dir;
|
|
123
|
+
}
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function main() {
|
|
128
|
+
const args = parseArgs(process.argv.slice(2));
|
|
129
|
+
// @electron/packager v20 是纯 ESM("type": "module" + exports 只指向 dist/index.js),
|
|
130
|
+
// require() 拿到的是模块命名空间而不是函数,必须用动态 import。
|
|
131
|
+
// 它的导出是命名导出 `packager`,没有 default —— 别写成 mod.default。
|
|
132
|
+
import('@electron/packager')
|
|
133
|
+
.then((mod) => build(mod.packager, args))
|
|
134
|
+
.catch((e) => {
|
|
135
|
+
console.error(`[build] FAILED: ${e.message}`);
|
|
136
|
+
process.exit(1);
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function build(packager, args) {
|
|
141
|
+
fs.rmSync(OUT_DIR, { recursive: true, force: true });
|
|
142
|
+
fs.mkdirSync(OUT_DIR, { recursive: true });
|
|
143
|
+
|
|
144
|
+
console.log(`[build] ${APP_NAME} ${pkg.version} — darwin/${args.arch}`);
|
|
145
|
+
console.log(`[build] ignore: ${IGNORE}`);
|
|
146
|
+
|
|
147
|
+
// electron 的实际版本以 node_modules 里装的那份为准(package.json 里是 ^ 范围)
|
|
148
|
+
const electronVersion = require('electron/package.json').version;
|
|
149
|
+
const zipDir = findElectronZipDir(electronVersion, 'darwin', args.arch);
|
|
150
|
+
if (zipDir) {
|
|
151
|
+
console.log(`[build] electron zip: 使用本地缓存 ${zipDir}(不联网)`);
|
|
152
|
+
} else {
|
|
153
|
+
console.log(`[build] electron zip: 本地无缓存,将从 GitHub 下载 v${electronVersion}`);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
packager({
|
|
157
|
+
dir: root,
|
|
158
|
+
name: APP_NAME,
|
|
159
|
+
platform: 'darwin',
|
|
160
|
+
arch: args.arch,
|
|
161
|
+
out: OUT_DIR,
|
|
162
|
+
overwrite: true,
|
|
163
|
+
ignore: IGNORE,
|
|
164
|
+
electronZipDir: zipDir || undefined,
|
|
165
|
+
// 只打生产依赖:npm prune --production 会把 devDependencies(vitest、jsdom、
|
|
166
|
+
// @electron/packager 自己)从安装包里剔掉。用户侧少一整套测试工具链。
|
|
167
|
+
prune: true,
|
|
168
|
+
asar: true,
|
|
169
|
+
appBundleId: 'com.glittering.ai-browser',
|
|
170
|
+
appCategoryType: 'public.app-categories.productivity',
|
|
171
|
+
// 图标必须显式传 icon 选项。@electron/packager 的 mac.js copyIcon() 在
|
|
172
|
+
// `!this.opts.icon` 时直接 return —— 它**不会**自动去 build/ 里找
|
|
173
|
+
// icon.icns(老版 electron-packager 的行为,v20 已经没有了)。
|
|
174
|
+
// normalizeIconExtension 只在源目录路径上 existsSync,与 IGNORE 无关,
|
|
175
|
+
// 所以 build/ 可以同时留在 ignore 列表里,不进安装包。
|
|
176
|
+
icon: path.join(root, 'build', 'icon.icns'),
|
|
177
|
+
// 图标母版是 build/icon.svg,由 scripts/render_icon.cjs 用项目自带的
|
|
178
|
+
// Chromium 栅格化,再经 sips + iconutil 打成 icns。改图标走这条链,
|
|
179
|
+
// 不要直接改 png。
|
|
180
|
+
})
|
|
181
|
+
.then((paths) => {
|
|
182
|
+
const appPath = paths[0];
|
|
183
|
+
const size = dirSize(appPath);
|
|
184
|
+
console.log(`[build] app bundle: ${appPath}`);
|
|
185
|
+
console.log(`[build] app size: ${mb(size)}`);
|
|
186
|
+
|
|
187
|
+
if (args.skipDmg) {
|
|
188
|
+
console.log('[build] --skip-dmg,不压 dmg');
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const dmg = path.join(OUT_DIR, `AI-Browser-${pkg.version}-${args.arch}.dmg`);
|
|
193
|
+
console.log(`[build] creating dmg: ${dmg}`);
|
|
194
|
+
// hdiutil 是 macOS 自带的,不需要 electron-builder 那套从 GitHub 下载的
|
|
195
|
+
// dmg-license 等二进制——在国内网络下那一步经常卡住。
|
|
196
|
+
// UDZO = zlib 压缩,比 UDBZ(bzip2) 快得多,体积只大一点。
|
|
197
|
+
run('hdiutil', [
|
|
198
|
+
'create',
|
|
199
|
+
'-volname', `${APP_NAME} ${pkg.version}`,
|
|
200
|
+
'-srcfolder', appPath,
|
|
201
|
+
'-ov',
|
|
202
|
+
'-format', 'UDZO',
|
|
203
|
+
dmg,
|
|
204
|
+
]);
|
|
205
|
+
|
|
206
|
+
const dmgSize = fs.statSync(dmg).size;
|
|
207
|
+
console.log('');
|
|
208
|
+
console.log('[build] ===== 结果 =====');
|
|
209
|
+
console.log(`[build] dmg: ${dmg}`);
|
|
210
|
+
console.log(`[build] size: ${mb(dmgSize)}`);
|
|
211
|
+
console.log(`[build] app: ${mb(size)}(解压后占用)`);
|
|
212
|
+
console.log('[build] 未签名、未公证:用户需要"右键 → 打开",见 docs/ 说明');
|
|
213
|
+
})
|
|
214
|
+
.catch((e) => {
|
|
215
|
+
console.error(`[build] FAILED: ${e.message}`);
|
|
216
|
+
process.exit(1);
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
if (require.main === module) main();
|