@glittering0717/ai-browser 1.3.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -5
- package/bin/ai-browser.cjs +32 -4
- package/examples/AGENT_INSTRUCTIONS.md +49 -164
- package/llms-install.md +27 -2
- package/package.json +3 -1
- package/scripts/cli_call.cjs +178 -0
- package/scripts/skill.cjs +133 -0
- package/skills/ai-browser/SKILL.md +55 -0
- package/skills/ai-browser/references/hard-pages.md +52 -0
- package/skills/ai-browser/references/network.md +39 -0
- package/skills/ai-browser/references/tasks.md +44 -0
- package/skills/ai-browser/references/troubleshooting.md +31 -0
- package/skills/content-publish/SKILL.md +61 -0
- package/skills/read-webpage/SKILL.md +52 -0
- package/skills/web-network-monitor/SKILL.md +52 -0
- package/src/main/axExtractor.js +50 -6
- package/src/main/capability_probe.js +93 -5
- package/src/main/index.js +99 -6
- package/src/main/mcp_args.js +122 -0
- package/src/main/mcp_server.js +55 -72
- package/src/main/mcp_tools.js +127 -86
- package/src/main/mcp_ws.js +7 -1
- package/src/main/page_manager.js +481 -71
- package/src/main/window_open_shim_source.js +117 -0
- package/src/preload/bridge.cjs +22 -4
- package/src/preload/extractor.cjs +161 -17
- package/src/shared/doc_coverage.js +264 -0
- package/src/shared/effect_state.js +160 -0
- package/src/shared/frame_probe.cjs +91 -1
- package/src/shared/frame_read_source.js +260 -0
- package/src/shared/opened_tab.js +71 -0
- package/src/shared/ref_epoch.js +99 -0
- package/src/shared/tab_hooks.js +33 -0
- package/src/shared/version.js +1 -1
- package/tools/coverage_probe.cjs +111 -9
package/README.md
CHANGED
|
@@ -205,13 +205,38 @@ small surface, since every tool schema is injected into your agent's context on
|
|
|
205
205
|
| `browse_evaluate` | Run a JS *expression* in page context (5000-char cap, Node identifiers rejected). |
|
|
206
206
|
| `browse_scroll` | Scroll the page, or scroll an element into view. |
|
|
207
207
|
| `browse_wait` | Poll until `button_enabled` / `modal_appeared` / `text_contains` / `url_contains`. |
|
|
208
|
-
| `
|
|
209
|
-
| `browse_new_tab` | Open a new tab, optionally at a URL. |
|
|
210
|
-
| `browse_close_tab` | Close a tab by id. |
|
|
211
|
-
| `browse_set_active_tab` | Switch which tab subsequent calls act on. |
|
|
208
|
+
| `browse_tabs` | Tabs: `list` / `new` / `close` / `switch`. `switch` sets the default target for later calls. |
|
|
212
209
|
| `browse_network` | Chrome-DevTools-style network inspection: `list` (filter by method / URL / status / type), `get` (request + response headers and bodies), `clear`, `configure`. |
|
|
213
210
|
| `browse_quit` | Shut the whole browser down and release the process. |
|
|
214
211
|
|
|
212
|
+
### What every act result already tells you: `effect_state`
|
|
213
|
+
|
|
214
|
+
`browse_act` no longer leaves you guessing whether your input reached the page. Every result
|
|
215
|
+
carries an explicit verdict:
|
|
216
|
+
|
|
217
|
+
| `effect_state` | Meaning | `retry` |
|
|
218
|
+
|---|---|---|
|
|
219
|
+
| `committed` | dispatched and acked by the page | `no` |
|
|
220
|
+
| `none` | provably nothing happened (no such target, bad params, CDP down) | `safe` |
|
|
221
|
+
| `unknown` | cannot prove nothing happened — timeout, paste dispatched but never consumed, half a key sequence | `verify_first` |
|
|
222
|
+
|
|
223
|
+
The default is `unknown`, not `none`: when the browser cannot tell, it says so. Retrying an
|
|
224
|
+
`unknown` blindly is how a form gets submitted twice — read the page first. `unknown` results
|
|
225
|
+
also carry a one-sentence `effect_hint` saying exactly that.
|
|
226
|
+
|
|
227
|
+
When a `target` cannot be found, the result adds `stale_ref` / `ref_epoch` / `tree_epoch` /
|
|
228
|
+
`ref_diagnosis`, because "no such target" hides three very different situations:
|
|
229
|
+
|
|
230
|
+
- `ref_diagnosis: "stale"` — the page was re-read after you took that ref (`ref_epoch` <
|
|
231
|
+
`tree_epoch`). Get a fresh one; retrying the same ref will keep failing.
|
|
232
|
+
- `ref_diagnosis: "never_seen"` — the ref has never been in any tree read so far. Re-reading
|
|
233
|
+
will not help: wrong value (visible text instead of a `data-ai-id`), a typo, or another tab's ref.
|
|
234
|
+
- `ref_diagnosis: "present_in_tree"` — in the current tree but not found in the DOM: iframe,
|
|
235
|
+
shadow root, or a detached node.
|
|
236
|
+
|
|
237
|
+
All three report `effect_state: "none"` (nothing happened) with `retry: "verify_first"` — no side
|
|
238
|
+
effects, but repeating the identical call is pointless.
|
|
239
|
+
|
|
215
240
|
### Prove it happened: `browse_diff`
|
|
216
241
|
|
|
217
242
|
A tool reporting `success` is **not** evidence that anything changed — the browser will happily
|
|
@@ -396,6 +421,22 @@ MCP is a thin stdio adapter. Anything MCP can do is reachable directly over JSON
|
|
|
396
421
|
`ws://localhost:9223` from any language. Source of truth: the method switch in
|
|
397
422
|
[`src/main/ws_server.js`](src/main/ws_server.js).
|
|
398
423
|
|
|
424
|
+
You do not have to write a client to do this. The CLI speaks the same protocol:
|
|
425
|
+
|
|
426
|
+
```bash
|
|
427
|
+
ai-browser status # running? pid / port / tabs
|
|
428
|
+
ai-browser call ui.list_tabs
|
|
429
|
+
ai-browser call ui.get_tree '{"subset":"interactive"}'
|
|
430
|
+
ai-browser call ui.navigate '{"url":"https://example.com"}'
|
|
431
|
+
echo '{"url":"https://example.com"}' | ai-browser call ui.navigate -
|
|
432
|
+
```
|
|
433
|
+
|
|
434
|
+
`call` prints the raw `result` as one line of JSON (pipe it into `jq`), exits `0` on success,
|
|
435
|
+
`1` when the call failed, `2` on a bad command line. `--port`, `--timeout`, `--pretty` are available.
|
|
436
|
+
|
|
437
|
+
So there are three doors to one implementation — MCP stdio (`ai-browser mcp`), JSON-RPC over
|
|
438
|
+
WebSocket, and this CLI. None of them is a separate code path.
|
|
439
|
+
|
|
399
440
|
| Method | Purpose | Maps to |
|
|
400
441
|
|---|---|---|
|
|
401
442
|
| `ui.get_tree` | Semantic tree (+ context). Params: `focusedOnly`, `ax`, `subset`, `mode`, `tab` | `browse_get_tree` |
|
|
@@ -407,7 +448,7 @@ MCP is a thin stdio adapter. Anything MCP can do is reachable directly over JSON
|
|
|
407
448
|
| `ui.evaluate` | Run JS in page context, same guard as MCP | `browse_evaluate` |
|
|
408
449
|
| `ui.scroll` | Scroll page or element (`direction`, `amount`, `target`). **Document scrolling only** — for canvas apps use `ui.act{action:'wheel'}`. | `browse_scroll` |
|
|
409
450
|
| `ui.wait` | Poll until a condition or timeout | `browse_wait` |
|
|
410
|
-
| `ui.new_tab` / `ui.close_tab` / `ui.list_tabs` / `ui.set_active_tab` | Tab management | `
|
|
451
|
+
| `ui.new_tab` / `ui.close_tab` / `ui.list_tabs` / `ui.set_active_tab` | Tab management (the four are merged into `browse_tabs{op}` on the MCP side) | `browse_tabs` |
|
|
411
452
|
| `ui.subscribe` / `ui.unsubscribe` | Event subscription; `params.events`, `"*"` for all | — (WS only — see below) |
|
|
412
453
|
| `ui.downloads` | This tab's downloads with on-disk paths (`state`, `received_bytes`, `dir`) | `browse_downloads` |
|
|
413
454
|
| `ui.get_focused` | The focused element subtree | — |
|
|
@@ -660,6 +701,16 @@ It exits non-zero below 80% so you can drop it into CI. Measured on 2026-10-17 (
|
|
|
660
701
|
CSDN): **82% / 97% / 88%** before the click-boundary fix, **100% / 100% / 100%** after, at ~40ms
|
|
661
702
|
per `ui.get_tree`.
|
|
662
703
|
|
|
704
|
+
> **Those three numbers were measuring the wrong denominator.** Until 2026-10-19 the tool only
|
|
705
|
+
> counted the top document: on a page whose body lives in a cross-process iframe it reported
|
|
706
|
+
> "4 click targets, 100% usable, 0 gaps" while the entire course catalogue was invisible. It now
|
|
707
|
+
> measures every `frame-N` it can reach and **prints how many frames it could not** — an
|
|
708
|
+
> unmeasured area inside the denominator is worse than a wrong measurement. Re-measured
|
|
709
|
+
> 2026-10-19: Baidu 100%, CSDN 99% (8 cross-process frames now counted, 2 residual gaps),
|
|
710
|
+
> and the 超星 course page 82% with the 2 remaining gaps named explicitly. The write-up of this
|
|
711
|
+
> class of mistake — a metric that silently excludes the part that matters — is
|
|
712
|
+
> [`docs/POSTMORTEM_FRAME_VISIBILITY.md`](docs/POSTMORTEM_FRAME_VISIBILITY.md).
|
|
713
|
+
|
|
663
714
|
Two things the numbers do **not** cover, so you do not chase them: an element the page gives no
|
|
664
715
|
readable name (image-only links with no `alt` — 53% of Bilibili's clickable nodes) can never get a
|
|
665
716
|
label without inventing one, and a canvas has no DOM to read at all. Those are genuine boundaries,
|
package/bin/ai-browser.cjs
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// ai-browser launch the visible browser window (foreground)
|
|
5
5
|
// ai-browser start same
|
|
6
6
|
// ai-browser mcp run the MCP stdio server (used by MCP clients)
|
|
7
|
+
// ai-browser skill list / install bundled agent skills
|
|
7
8
|
// ai-browser --version
|
|
8
9
|
// ai-browser --help
|
|
9
10
|
//
|
|
@@ -19,13 +20,22 @@ const HELP = `ai-browser ${pkg.version}
|
|
|
19
20
|
|
|
20
21
|
Usage:
|
|
21
22
|
ai-browser [start] Launch the visible AI Browser window
|
|
22
|
-
ai-browser mcp Run the MCP stdio server (
|
|
23
|
+
ai-browser mcp Run the MCP stdio server (17 browse_* tools)
|
|
24
|
+
ai-browser skill [list|install|print]
|
|
25
|
+
Install the bundled agent skills into your client
|
|
26
|
+
ai-browser call <method> [json|-]
|
|
27
|
+
Call one JSON-RPC method and print the result
|
|
28
|
+
ai-browser status Is the browser running? pid / port / tabs
|
|
23
29
|
ai-browser --version Print version
|
|
24
30
|
ai-browser --help Show this help
|
|
25
31
|
|
|
26
|
-
The browser exposes:
|
|
27
|
-
• MCP server over stdio (
|
|
28
|
-
•
|
|
32
|
+
The browser exposes one implementation through three thin doors:
|
|
33
|
+
• MCP server over stdio → ai-browser mcp (for agent clients)
|
|
34
|
+
• WebSocket JSON-RPC → ws://127.0.0.1:9223 (ui.* methods)
|
|
35
|
+
• This CLI → ai-browser call ui.list_tabs
|
|
36
|
+
|
|
37
|
+
Agent skills ship in this package. Install them once with "ai-browser skill install"
|
|
38
|
+
and every new session picks them up automatically — no manual instruction pasting.
|
|
29
39
|
|
|
30
40
|
Docs: https://github.com/Glittering/ai-browser
|
|
31
41
|
`;
|
|
@@ -55,6 +65,24 @@ async function main() {
|
|
|
55
65
|
await import(pathToFileURL(serverPath).href);
|
|
56
66
|
break;
|
|
57
67
|
}
|
|
68
|
+
case 'skill': {
|
|
69
|
+
// 技能装进客户端的技能目录 —— 装一次,之后每次新会话自动发现。
|
|
70
|
+
const { run } = require('../scripts/skill.cjs');
|
|
71
|
+
process.exit(run(argv.slice(1)));
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
case 'call': {
|
|
75
|
+
// 命令行直接调一条 JSON-RPC。存在的理由:以前想从 shell 问一句"有哪些标签页"
|
|
76
|
+
// 都得先 npm i ws 再写脚本。见 scripts/cli_call.cjs 顶部注释。
|
|
77
|
+
const { runCall } = require('../scripts/cli_call.cjs');
|
|
78
|
+
process.exit(await runCall(argv.slice(1)));
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
case 'status': {
|
|
82
|
+
const { runStatus } = require('../scripts/cli_call.cjs');
|
|
83
|
+
process.exit(await runStatus(argv.slice(1)));
|
|
84
|
+
break;
|
|
85
|
+
}
|
|
58
86
|
case '--version':
|
|
59
87
|
case '-v':
|
|
60
88
|
case 'version':
|
|
@@ -1,186 +1,71 @@
|
|
|
1
|
-
# AI Browser —
|
|
1
|
+
# AI Browser — agent 接入
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
(Claude Code 的 `CLAUDE.md`、Cursor Rules、Codex `AGENTS.md`、或任意 MCP 客户端的自定义指令)。
|
|
5
|
-
不需要改任何东西——浏览器由 agent 自己在第一次调用时拉起。
|
|
3
|
+
## 先别贴这份文件
|
|
6
4
|
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
```bash
|
|
6
|
+
npx -y @glittering0717/ai-browser skill install
|
|
7
|
+
```
|
|
9
8
|
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
装一次,之后**每次新会话客户端会自动发现**这些技能,不需要你在对话里重新教一遍、也不用把
|
|
10
|
+
下面任何内容粘进 system prompt:
|
|
12
11
|
|
|
13
|
-
|
|
12
|
+
| 技能 | 干什么 |
|
|
13
|
+
|---|---|
|
|
14
|
+
| `ai-browser` | 最小循环、卡住时的第一反应、难搞页面的处理 |
|
|
15
|
+
| `content-publish` | 把文章发布到 CSDN / 知乎 / 掘金 |
|
|
16
|
+
| `read-webpage` | 读一篇文章正文并总结 |
|
|
17
|
+
| `web-network-monitor` | 长时间盯某个页面的请求 |
|
|
14
18
|
|
|
15
|
-
|
|
19
|
+
不装技能也能用——20 个 `browse_*` 工具的 schema 本身自带说明。技能只是把"怎么组合"讲清楚。
|
|
16
20
|
|
|
17
|
-
|
|
18
|
-
我可以用 AI Browser 浏览网页。它提供 MCP 服务(20 个 browse_* 工具),第一次调用时会自动
|
|
19
|
-
拉起一个真实可见的浏览器窗口,用完调用 browse_quit 释放。页面以结构化语义树返回,不是截图。
|
|
20
|
-
|
|
21
|
-
标准循环:
|
|
22
|
-
1. browse_navigate { url } 打开页面
|
|
23
|
-
2. browse_get_tree 读页面,返回语义树:每个节点有 role / label / value /
|
|
24
|
-
states / actions
|
|
25
|
-
3. browse_act { action, target } 操作元素,target 填第 2 步节点里的 id(即 data-ai-id)
|
|
26
|
-
action 可选:click / type(光标处追加)/ setContent
|
|
27
|
-
(整体替换)/ clear / focus / hover / scroll_to /
|
|
28
|
-
upload / get_value(只读回读全文)/ drag / press / wheel
|
|
29
|
-
4. browse_wait { condition } 等待条件成立:button_enabled / modal_appeared /
|
|
30
|
-
text_contains / url_contains
|
|
31
|
-
5. browse_evaluate { js } 在页面上下文跑 JS 表达式(上限 5000 字符,
|
|
32
|
-
禁用 process. / require / child_process)
|
|
33
|
-
6. browse_errors 出事了先看这里:主进程异常 / 崩溃 / 导航失败 /
|
|
34
|
-
RPC 报错的最近记录(含调用上下文与日志路径)
|
|
35
|
-
|
|
36
|
-
省 token(browse_get_tree 参数):
|
|
37
|
-
- { "subset": "interactive" } 只保留可交互节点,砍掉纯布局分支 —— 默认就该带上
|
|
38
|
-
- { "ax": true } 走 AX 语义层,链接会带 url
|
|
39
|
-
- { "mode": "diff" } 只看上次读取以来新增 / 消失的可交互节点
|
|
40
|
-
|
|
41
|
-
验证写操作(success 不等于做成了):browse_diff 先 snapshot 基线,写操作之后再 diff,
|
|
42
|
-
verdict 显式给出 changed / unchanged。
|
|
43
|
-
|
|
44
|
-
找不到元素 / 点了没反应时的排查顺序(**别一上来就写站点专用选择器**):
|
|
45
|
-
1. 重读 browse_get_tree,或先 browse_wait —— SPA 常常只是还没渲染完
|
|
46
|
-
2. 节点在树里但操作失败 → 是**动作层**问题(跨进程 iframe、canvas、需要受信输入、
|
|
47
|
-
上传控件未挂载),写选择器解决不了
|
|
48
|
-
3. 树里没有、页面上明明有 → 是**读层**缺口,属于浏览器该修的东西;先跑
|
|
49
|
-
tools/coverage_probe.cjs <url> 量化,再考虑绕过
|
|
50
|
-
4. 以上都不成立,才在自己脚本里用选择器兜底;别把"某个站点的选择器"记成通用知识
|
|
51
|
-
|
|
52
|
-
其他:browse_scroll 滚动文档(画布类页面用 browse_act 的 wheel);browse_list_tabs /
|
|
53
|
-
browse_new_tab / browse_close_tab / browse_set_active_tab 管标签页;browse_network 拉取网络
|
|
54
|
-
请求(含 POST body,免订阅);browse_capabilities 一次探明页面该走哪条路(结构化数据 /
|
|
55
|
-
语义树 / 指针手势);browse_canvas 读 canvas 绘制调用里的文本(不是 OCR);
|
|
56
|
-
browse_downloads 查下载文件落盘路径;browse_repaint 在窗口画面陈旧时强制重绘。
|
|
57
|
-
|
|
58
|
-
需要人帮忙时(登录、扫码、短信验证码、滑块 / 人机验证,以及任何我做不了的操作):
|
|
59
|
-
浏览器窗口是真实可见的,人随时可以用鼠标键盘操作。我应该在对话里直接说一句人话请用户去窗口里
|
|
60
|
-
完成,例如"请在浏览器窗口里扫码登录,完成后回复我'好了'",然后停下来等用户回复;收到"好了"之后
|
|
61
|
-
重新 browse_get_tree 确认状态再继续。不要编造登录凭据,也不要反复重试那些注定失败的点击。
|
|
62
|
-
浏览器侧没有任何"交接"接口,也不需要——人机协作就发生在我们的对话里。
|
|
63
|
-
|
|
64
|
-
收尾:任务完成后调用 browse_quit(会关闭整个浏览器窗口和进程,释放资源)。
|
|
65
|
-
```
|
|
21
|
+
---
|
|
66
22
|
|
|
67
|
-
##
|
|
23
|
+
## 兜底:客户端不支持技能时
|
|
24
|
+
|
|
25
|
+
把下面这段贴进 `CLAUDE.md` / `AGENTS.md` / 自定义指令。**只需要这几行**,其余按需查技能里的
|
|
26
|
+
`references/`。
|
|
68
27
|
|
|
69
28
|
```text
|
|
70
|
-
|
|
71
|
-
real, visible browser window on my first call, and I release it with browse_quit when done. Pages come
|
|
72
|
-
back as a structured semantic tree — no screenshots, no OCR.
|
|
73
|
-
|
|
74
|
-
Standard loop:
|
|
75
|
-
1. browse_navigate { url } load a page
|
|
76
|
-
2. browse_get_tree read the page: nodes carry role / label / value / states /
|
|
77
|
-
actions
|
|
78
|
-
3. browse_act { action, target } act on an element; target is the node's id (its data-ai-id)
|
|
79
|
-
action: click / type (append at caret) / setContent
|
|
80
|
-
(replace all) / clear / focus / hover / scroll_to /
|
|
81
|
-
upload / get_value (read-only) / drag / press / wheel
|
|
82
|
-
4. browse_wait { condition } block until: button_enabled / modal_appeared /
|
|
83
|
-
text_contains / url_contains
|
|
84
|
-
5. browse_evaluate { js } run a JS expression in page context (5000-char cap;
|
|
85
|
-
process. / require / child_process are rejected)
|
|
86
|
-
|
|
87
|
-
Saving tokens (browse_get_tree params):
|
|
88
|
-
- { "subset": "interactive" } drop layout-only branches — use this by default
|
|
89
|
-
- { "ax": true } AX semantic layer; links carry url
|
|
90
|
-
- { "mode": "diff" } only interactive nodes added/hidden since my last read
|
|
91
|
-
|
|
92
|
-
Verifying writes (success is not evidence): browse_diff snapshots a baseline, then diffs after the
|
|
93
|
-
write and returns an explicit verdict of changed / unchanged.
|
|
94
|
-
|
|
95
|
-
When an element cannot be found, or a click does nothing — work down this list, do **not** reach for a
|
|
96
|
-
site-specific selector first:
|
|
97
|
-
1. re-read with browse_get_tree, or browse_wait — an SPA usually just has not rendered yet
|
|
98
|
-
2. the node is in the tree but the action fails → an **action-layer** gap (cross-process iframe,
|
|
99
|
-
canvas, trusted input, unmounted upload input); a selector will not fix it
|
|
100
|
-
3. the node is missing though the page clearly renders it → a **read-layer** gap, i.e. something the
|
|
101
|
-
browser should fix; quantify it with `tools/coverage_probe.cjs <url>` before working around it
|
|
102
|
-
4. only then pin it with a selector inside your own script — and never record "the selector for site X"
|
|
103
|
-
as general knowledge
|
|
104
|
-
|
|
105
|
-
Also: browse_scroll (document scrolling; use the wheel action on canvas apps); browse_list_tabs /
|
|
106
|
-
browse_new_tab / browse_close_tab / browse_set_active_tab for tabs; browse_network for captured
|
|
107
|
-
requests including POST bodies (no subscription); browse_capabilities to pick a route on an
|
|
108
|
-
unfamiliar page (structured data / semantic tree / pointer gestures); browse_canvas reads text
|
|
109
|
-
straight from canvas draw calls (no OCR); browse_downloads for on-disk download paths;
|
|
110
|
-
browse_repaint when the window shows a stale page.
|
|
111
|
-
|
|
112
|
-
When I need a human (login, QR code, SMS code, slider/CAPTCHA, or anything I genuinely cannot do):
|
|
113
|
-
the browser window is real and visible — the user can just use their mouse and keyboard in it. I should
|
|
114
|
-
say so plainly in the conversation (e.g. "please scan the QR code in the browser window and reply
|
|
115
|
-
'done'"), then stop and wait for that reply. After "done", I re-read with browse_get_tree and continue.
|
|
116
|
-
I never invent credentials or loop on clicks that cannot work. There is no hand-off API on the browser
|
|
117
|
-
side, and none is needed — the collaboration happens right here in our conversation.
|
|
118
|
-
|
|
119
|
-
Wrap up: call browse_quit when the task is finished (it closes the window and exits the process).
|
|
120
|
-
```
|
|
29
|
+
我可以用 ai-browser 操作一个真实可见的浏览器(MCP: browse_* 工具)。
|
|
121
30
|
|
|
122
|
-
|
|
31
|
+
循环:browse_navigate {url} → browse_get_tree {subset:"interactive"} 取节点 id →
|
|
32
|
+
browse_act {action:"click"|"type", target:id, text?}
|
|
123
33
|
|
|
124
|
-
|
|
34
|
+
success:true 只代表动作发出去了,不代表页面变了。要证明生效:browse_diff 先 snapshot、
|
|
35
|
+
操作后 diff,看 verdict。unchanged 说明要么点错了、要么页面没响应,别接着瞎点。
|
|
125
36
|
|
|
126
|
-
|
|
127
|
-
|
|
37
|
+
卡住时的顺序:节点没有 → browse_wait 再重读树;有节点但操作失败 → 动作层问题(iframe /
|
|
38
|
+
canvas / 富文本),换动作不是换选择器;点了没反应 → browse_screenshot 看视觉 或
|
|
39
|
+
browse_network 看请求;怀疑掉登录 → browse_auth_state;都不是 → browse_errors。
|
|
128
40
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
"command": "npx",
|
|
134
|
-
"args": ["-y", "@glittering0717/ai-browser", "mcp"]
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
}
|
|
41
|
+
需要人(登录/扫码/滑块/支付):在对话里直接请用户在窗口里操作,收到"好了"再 browse_get_tree
|
|
42
|
+
确认。不要编造凭据,不要重试注定失败的点击。窗口是真的,人随时能抢鼠标。
|
|
43
|
+
|
|
44
|
+
收尾:browse_quit 关掉窗口和进程(人正在操作时不要调)。
|
|
138
45
|
```
|
|
139
46
|
|
|
140
|
-
|
|
141
|
-
`/你的/绝对路径/ai-browser/src/main/mcp_server.js`(必须是绝对路径)。
|
|
47
|
+
---
|
|
142
48
|
|
|
143
|
-
|
|
144
|
-
`ELECTRON_MIRROR=https://npmmirror.com/mirrors/electron/`。
|
|
145
|
-
* 无需事先启动浏览器:MCP 服务器会探测 `9223` 端口,没人在监听就自己 spawn Electron(最多等 30 秒)。
|
|
146
|
-
* 想常驻浏览器也可以:在项目根目录跑 `npm start`,之后所有客户端共用这一个窗口。
|
|
147
|
-
* 不装 MCP 也能用:原始 WebSocket JSON-RPC 在 `ws://localhost:9223`,见 `raw-ws.mjs`。
|
|
49
|
+
## 详细内容在哪
|
|
148
50
|
|
|
149
|
-
|
|
51
|
+
按需读,不要一次读完:
|
|
150
52
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
|
154
|
-
|
|
155
|
-
|
|
|
156
|
-
|
|
|
157
|
-
|
|
|
158
|
-
| `browse_evaluate` | `js`, `tab?` | 表达式(非语句),5000 字符上限 |
|
|
159
|
-
| `browse_scroll` | `direction`, `amount`, `target`, `tab` | 滚页面或滚到某元素 |
|
|
160
|
-
| `browse_wait` | `condition`, `target?`, `text?`, `timeout_ms?` | 轮询直到成立或超时 |
|
|
161
|
-
| `browse_diff` | `operation` (`snapshot`/`diff`), `js?`, `snapshot?`, `rearm?`, `forget?` | 快照基线 + 比对,证明写操作真的生效 |
|
|
162
|
-
| `browse_capabilities` | `include_network?`, `network_limit?` | 探明页面路线:L1 结构化数据 / L2 语义树 / L3 指针手势 |
|
|
163
|
-
| `browse_network` | `operation` (`list`/`get`/`clear`/`configure`), `method?`, `url_contains?`, `status_min/max?` | 网络请求检查,含 POST body,默认免订阅 |
|
|
164
|
-
| `browse_canvas` | `operation` (`list`/`read`/`configure`/`capture`), `canvas_id?` | 读 canvas 绘制调用文本;WebGL 才用 `capture` |
|
|
165
|
-
| `browse_downloads` | — | 查下载项与落盘路径(下载异步,点完后轮询) |
|
|
166
|
-
| `browse_repaint` | — | 窗口画面陈旧时强制合成器重绘,返回 `repaint_count` |
|
|
167
|
-
| `browse_list_tabs` / `browse_new_tab` / `browse_close_tab` / `browse_set_active_tab` | `tab?`, `url?` | 标签页管理 |
|
|
168
|
-
| `browse_quit` | — | 关闭窗口并退出进程 |
|
|
169
|
-
|
|
170
|
-
> 实时事件订阅(`dom_change` / `network_response` / `js_error` 等)只在原始 WebSocket 层
|
|
171
|
-
> 提供(`ui.subscribe`),MCP stdio 不支持 server→client 通知,所以没有 `browse_subscribe`;
|
|
172
|
-
> 拉取式的检查一律用 `browse_network`。
|
|
53
|
+
| 想知道 | 读 |
|
|
54
|
+
|---|---|
|
|
55
|
+
| 失败排查顺序、四条别做、省 token 的读树参数 | `skills/ai-browser/references/troubleshooting.md` |
|
|
56
|
+
| canvas / 富文本 / 跨进程 iframe / 上传 | `skills/ai-browser/references/hard-pages.md` |
|
|
57
|
+
| 抓请求、从页面取结构化数据 | `skills/ai-browser/references/network.md` |
|
|
58
|
+
| 登录、表单、多标签、证明生效 | `skills/ai-browser/references/tasks.md` |
|
|
59
|
+
| 全部参数 | 工具的 `inputSchema`(每轮都在,不用另读) |
|
|
173
60
|
|
|
174
|
-
|
|
61
|
+
## 配置 MCP 服务器
|
|
175
62
|
|
|
176
|
-
|
|
63
|
+
见 `examples/mcp-config.json`,或:
|
|
177
64
|
|
|
178
|
-
|
|
65
|
+
```json
|
|
66
|
+
{ "mcpServers": { "ai-browser": {
|
|
67
|
+
"command": "npx", "args": ["-y", "@glittering0717/ai-browser", "mcp"]
|
|
68
|
+
} } }
|
|
69
|
+
```
|
|
179
70
|
|
|
180
|
-
|
|
181
|
-
2. agent 做不了的事,**在对话里说一句人话请求用户去做**,然后停下来等回复。
|
|
182
|
-
3. 用户做完回一句"好了",agent 重新 `browse_get_tree` 确认状态,继续往下走。
|
|
183
|
-
4. **浏览器侧没有任何"交接"接口,也不需要。** 交接是一个对话协议,不是一个 API。
|
|
184
|
-
脚本版的等价实现见 `with-human.mjs`:它只是 `readline` 打印一句提示 + 等回车,
|
|
185
|
-
浏览器对此毫不知情。
|
|
186
|
-
5. 登录态存在 `persist:` partition 里,重启浏览器也还在,通常只需人工介入一次。
|
|
71
|
+
浏览器由 agent 在第一次调用时自己拉起,不需要你手动开。
|
package/llms-install.md
CHANGED
|
@@ -84,6 +84,21 @@ Cline, etc.):
|
|
|
84
84
|
From-source alternative: `"command": "node"` with the absolute path to
|
|
85
85
|
`src/main/mcp_server.js`.
|
|
86
86
|
|
|
87
|
+
> **Match the version, or nothing lines up.** The skill shipped in this package documents the
|
|
88
|
+
> tool surface of the *same* version. Before pasting `npx` above, compare
|
|
89
|
+
> `npm view @glittering0717/ai-browser version` with this repo's `package.json`. If they differ
|
|
90
|
+
> (they have: npm 1.3.1 vs repo 2.0.0), use the from-source form with the source path — a 1.x
|
|
91
|
+
> server behind a 2.x skill means every tool name and parameter in the skill is wrong, and a
|
|
92
|
+
> small model cannot tell that apart from "the MCP server is down". A concrete field report:
|
|
93
|
+
> a 35B local model spent a long session concluding "the skill is installed but the MCP server
|
|
94
|
+
> is not mounted", correctly diagnosing a missing config entry while the real trap was that
|
|
95
|
+
> nothing had been registered at all — see the `ai-browser` entry in the client's `mcp.json`.
|
|
96
|
+
>
|
|
97
|
+
> **No MCP at all?** The CLI speaks the same JSON-RPC and needs no registration:
|
|
98
|
+
> `ai-browser status`, `ai-browser call ui.navigate '{"url":"..."}'`. Installing the package
|
|
99
|
+
> globally (`npm i -g @glittering0717/ai-browser`) puts that binary on PATH; from a source
|
|
100
|
+
> checkout, `npm link` does the same.
|
|
101
|
+
|
|
87
102
|
Do **not** launch the browser yourself first: on the first `browse_*` call the MCP server probes
|
|
88
103
|
port 9223, spawns Electron detached if nothing is listening, and waits up to 30 s (the very first
|
|
89
104
|
call can take longer if the engine download hasn't happened yet — the progress goes to stderr).
|
|
@@ -100,6 +115,11 @@ focuses the existing window (single instance lock).
|
|
|
100
115
|
`setContent` (replace all), `clear`, `focus`, `hover`, `scroll_to`, `upload`,
|
|
101
116
|
`get_value` (read-only full value; tree truncates at 200 chars), `drag`,
|
|
102
117
|
`press`, `wheel`.
|
|
118
|
+
Every result carries `effect_state: "committed" | "none" | "unknown"` plus a
|
|
119
|
+
`retry` verdict (`no` / `safe` / `verify_first`). Treat `unknown` as "read the
|
|
120
|
+
page before touching it again" — a blind retry can submit twice. A missing
|
|
121
|
+
target adds `stale_ref` + `ref_diagnosis` (`stale` → re-read the tree;
|
|
122
|
+
`never_seen` → the ref itself is wrong; `present_in_tree` → iframe/shadow DOM).
|
|
103
123
|
- `browse_diff` — `snapshot` a baseline, then `diff`; returns an explicit
|
|
104
124
|
`verdict: "changed" | "unchanged"` with before/after JSON paths. Wrap every
|
|
105
125
|
write with this — a success return is not evidence the write worked.
|
|
@@ -118,10 +138,15 @@ focuses the existing window (single instance lock).
|
|
|
118
138
|
- `browse_repaint` — force compositor redraw when the window shows a stale page.
|
|
119
139
|
- `browse_downloads` — list downloads with on-disk paths (poll after clicking
|
|
120
140
|
export/download; downloads are asynchronous).
|
|
121
|
-
- `
|
|
122
|
-
`
|
|
141
|
+
- `browse_tabs` — tab management: `{op: "list" | "new" | "close" | "switch"}`.
|
|
142
|
+
`switch` sets the default target for later calls, so you rarely pass `tab`.
|
|
123
143
|
- `browse_quit` — shut the whole browser down and release the process.
|
|
124
144
|
|
|
145
|
+
Prefer MCP when an agent client drives the browser. For shell / CI / debugging there is a CLI
|
|
146
|
+
that speaks the same JSON-RPC: `ai-browser status`, `ai-browser call ui.list_tabs`,
|
|
147
|
+
`ai-browser call ui.navigate '{"url":"..."}'`. One implementation, three doors — no separate
|
|
148
|
+
script needed to call a method.
|
|
149
|
+
|
|
125
150
|
There is no `browse_subscribe` and no `browse_network_body`: real-time events are
|
|
126
151
|
raw-WebSocket only (`ui.subscribe`), because MCP stdio drops server→client
|
|
127
152
|
notifications. Use `browse_network` for pull-based inspection.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@glittering0717/ai-browser",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "AI-native browser — renders pixels for humans, structured semantic-tree API for agents. No screenshots, no OCR, no coordinate guessing.",
|
|
5
5
|
"main": "src/main/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
"bin",
|
|
11
11
|
"scripts",
|
|
12
12
|
"src",
|
|
13
|
+
"skills",
|
|
13
14
|
"tools/coverage_probe.cjs",
|
|
14
15
|
"examples/AGENT_INSTRUCTIONS.md",
|
|
15
16
|
"examples/mcp-config.json",
|
|
@@ -70,6 +71,7 @@
|
|
|
70
71
|
"nav": "node e2e/nav_ws.cjs",
|
|
71
72
|
"audit": "node e2e/audit_ws.cjs",
|
|
72
73
|
"e2e:all": "node e2e/run_all.cjs",
|
|
74
|
+
"readcov": "node e2e/readcov_ws.cjs",
|
|
73
75
|
"coverage": "node tools/coverage_probe.cjs"
|
|
74
76
|
},
|
|
75
77
|
"dependencies": {
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// scripts/cli_call.cjs — 命令行直接调浏览器,**不用写任何客户端脚本**。
|
|
3
|
+
//
|
|
4
|
+
// 为什么要有这个(真实抱怨,2026-10-19):
|
|
5
|
+
// 「我看操作这个项目,经常需要写 cjs 代码,这是必须的吗?」
|
|
6
|
+
// 不是必须的,但以前确实没有别的办法:对外只有两个入口 —— MCP stdio(给 agent
|
|
7
|
+
// 客户端用)和裸 WebSocket JSON-RPC(ws://127.0.0.1:9223)。想从 shell 里问一句
|
|
8
|
+
// "现在有哪些标签页",就得先 npm i ws、再写一个连上发 JSON 的小脚本、再等回包。
|
|
9
|
+
// 一个本来只是一行命令的事情,被迫变成"写代码"。
|
|
10
|
+
//
|
|
11
|
+
// 所以这里补的是**通道**,不是新能力:它做的就是把一条 JSON-RPC 发出去、把 result
|
|
12
|
+
// 打出来。所有能力仍然只有一套实现(page_manager),MCP 和 CLI 都是它的薄壳。
|
|
13
|
+
//
|
|
14
|
+
// 用法:
|
|
15
|
+
// ai-browser call ui.list_tabs
|
|
16
|
+
// ai-browser call ui.get_tree '{"subset":"interactive"}'
|
|
17
|
+
// ai-browser call ui.navigate '{"url":"https://example.com"}'
|
|
18
|
+
// echo '{"url":"https://example.com"}' | ai-browser call ui.navigate -
|
|
19
|
+
// ai-browser call ui.act '{"action":"click","target":"e:a-1"}'
|
|
20
|
+
// ai-browser status
|
|
21
|
+
//
|
|
22
|
+
// 选项:
|
|
23
|
+
// --port <n> 默认 9223(也读 AI_BROWSER_WS_PORT)
|
|
24
|
+
// --timeout <ms> 默认 30000
|
|
25
|
+
// --compact 单行输出(默认就是紧凑 JSON,加这个只是显式)
|
|
26
|
+
// --pretty 缩进输出(给人看时用)
|
|
27
|
+
'use strict';
|
|
28
|
+
|
|
29
|
+
const WS_PORT_DEFAULT = 9223;
|
|
30
|
+
const TIMEOUT_DEFAULT = 30000;
|
|
31
|
+
|
|
32
|
+
function parseArgs(argv) {
|
|
33
|
+
const out = { port: null, timeout: TIMEOUT_DEFAULT, pretty: false, rest: [] };
|
|
34
|
+
for (let i = 0; i < argv.length; i++) {
|
|
35
|
+
const a = argv[i];
|
|
36
|
+
if (a === '--port') { out.port = Number(argv[++i]); continue; }
|
|
37
|
+
if (a === '--timeout') { out.timeout = Number(argv[++i]); continue; }
|
|
38
|
+
if (a === '--pretty') { out.pretty = true; continue; }
|
|
39
|
+
if (a === '--compact') { out.pretty = false; continue; }
|
|
40
|
+
out.rest.push(a);
|
|
41
|
+
}
|
|
42
|
+
if (!out.port || !Number.isFinite(out.port)) {
|
|
43
|
+
const envPort = Number(process.env.AI_BROWSER_WS_PORT);
|
|
44
|
+
out.port = Number.isFinite(envPort) && envPort > 0 ? envPort : WS_PORT_DEFAULT;
|
|
45
|
+
}
|
|
46
|
+
if (!Number.isFinite(out.timeout) || out.timeout <= 0) out.timeout = TIMEOUT_DEFAULT;
|
|
47
|
+
return out;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** 把命令行里的 JSON 载荷解析成对象;空/缺省 → {}。 */
|
|
51
|
+
function parsePayload(text) {
|
|
52
|
+
if (text === undefined) return {};
|
|
53
|
+
const t = String(text).trim();
|
|
54
|
+
if (!t) return {};
|
|
55
|
+
let val;
|
|
56
|
+
try {
|
|
57
|
+
val = JSON.parse(t);
|
|
58
|
+
} catch (e) {
|
|
59
|
+
const err = new Error(`payload is not valid JSON: ${e.message}\n got: ${t.slice(0, 120)}`);
|
|
60
|
+
err.code = 'bad_json';
|
|
61
|
+
throw err;
|
|
62
|
+
}
|
|
63
|
+
if (val === null || typeof val !== 'object' || Array.isArray(val)) {
|
|
64
|
+
const err = new Error('payload must be a JSON object, e.g. {"url":"https://example.com"}');
|
|
65
|
+
err.code = 'bad_json';
|
|
66
|
+
throw err;
|
|
67
|
+
}
|
|
68
|
+
return val;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* 发一条 JSON-RPC 并等回包。**不在这里做重试/重连** —— CLI 是同步语义:
|
|
73
|
+
* 一条命令一个结果,失败就把原因原样报出来,别替调用方猜。
|
|
74
|
+
*/
|
|
75
|
+
function rpcCall({ port, timeout, method, params }) {
|
|
76
|
+
// require 放在函数里:只有真的要连的时候才需要 ws(--help 之类不该加载它)。
|
|
77
|
+
const WebSocket = require('ws');
|
|
78
|
+
return new Promise((resolve, reject) => {
|
|
79
|
+
let settled = false;
|
|
80
|
+
const done = (fn, arg) => { if (!settled) { settled = true; try { ws.close(); } catch (_) {} fn(arg); } };
|
|
81
|
+
const ws = new WebSocket(`ws://127.0.0.1:${port}`);
|
|
82
|
+
const timer = setTimeout(() => {
|
|
83
|
+
done(reject, Object.assign(new Error(`no reply from ${method} within ${timeout}ms`), { code: 'timeout' }));
|
|
84
|
+
}, timeout);
|
|
85
|
+
|
|
86
|
+
ws.on('open', () => {
|
|
87
|
+
ws.send(JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }));
|
|
88
|
+
});
|
|
89
|
+
ws.on('message', (data) => {
|
|
90
|
+
let msg;
|
|
91
|
+
try { msg = JSON.parse(String(data)); } catch (_) { return; } // 事件广播不是回包,忽略
|
|
92
|
+
if (msg.id !== 1) return;
|
|
93
|
+
clearTimeout(timer);
|
|
94
|
+
if (msg.error) {
|
|
95
|
+
// 带上方法名:服务端只会说 "Method not found",不带名字的话调用方还得自己
|
|
96
|
+
// 回想刚才打的是哪个(一条命令里可能有变量拼出来的方法名)。
|
|
97
|
+
const why = msg.error.message || 'rpc error';
|
|
98
|
+
done(reject, Object.assign(new Error(`${why} (${method})`), { code: 'rpc_error', rpc: msg.error }));
|
|
99
|
+
} else {
|
|
100
|
+
done(resolve, msg.result);
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
ws.on('error', (e) => {
|
|
104
|
+
clearTimeout(timer);
|
|
105
|
+
const hint = /ECONNREFUSED/i.test(e.message)
|
|
106
|
+
? `浏览器没在 127.0.0.1:${port} 上跑。先 \`ai-browser start\`(MCP 客户端会自己拉起),或用 --port 指定别的端口。`
|
|
107
|
+
: e.message;
|
|
108
|
+
done(reject, Object.assign(new Error(hint), { code: 'connect_failed' }));
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function printResult(result, pretty) {
|
|
114
|
+
if (result === undefined) { process.stdout.write('null\n'); return; }
|
|
115
|
+
const text = pretty ? JSON.stringify(result, null, 2) : JSON.stringify(result);
|
|
116
|
+
process.stdout.write(text + '\n');
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
async function runCall(argv) {
|
|
120
|
+
const opt = parseArgs(argv);
|
|
121
|
+
const method = opt.rest[0];
|
|
122
|
+
if (!method) {
|
|
123
|
+
process.stderr.write('usage: ai-browser call <method> [json|-] [--port n] [--timeout ms] [--pretty]\n'
|
|
124
|
+
+ ' e.g. ai-browser call ui.list_tabs\n'
|
|
125
|
+
+ ' ai-browser call ui.get_tree \'{"subset":"interactive"}\'\n');
|
|
126
|
+
return 2;
|
|
127
|
+
}
|
|
128
|
+
let params;
|
|
129
|
+
try {
|
|
130
|
+
const raw = opt.rest[1] === '-' ? readStdin() : opt.rest[1];
|
|
131
|
+
params = parsePayload(raw);
|
|
132
|
+
} catch (e) {
|
|
133
|
+
process.stderr.write(`[ai-browser] ${e.message}\n`);
|
|
134
|
+
return 2;
|
|
135
|
+
}
|
|
136
|
+
try {
|
|
137
|
+
const result = await rpcCall({ port: opt.port, timeout: opt.timeout, method, params });
|
|
138
|
+
printResult(result, opt.pretty);
|
|
139
|
+
return 0;
|
|
140
|
+
} catch (e) {
|
|
141
|
+
process.stderr.write(`[ai-browser] ${e.message}\n`);
|
|
142
|
+
return 1;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
async function runStatus(argv) {
|
|
147
|
+
const opt = parseArgs(argv);
|
|
148
|
+
try {
|
|
149
|
+
const info = await rpcCall({ port: opt.port, timeout: Math.min(opt.timeout, 5000), method: 'ui.server_info', params: {} });
|
|
150
|
+
let tabs = null;
|
|
151
|
+
try { tabs = await rpcCall({ port: opt.port, timeout: Math.min(opt.timeout, 5000), method: 'ui.list_tabs', params: {} }); } catch (_) {}
|
|
152
|
+
const out = {
|
|
153
|
+
running: true,
|
|
154
|
+
port: opt.port,
|
|
155
|
+
pid: info && info.pid,
|
|
156
|
+
started_at: info && info.started_at,
|
|
157
|
+
platform: info && info.platform,
|
|
158
|
+
tabs: tabs && tabs.tabs ? tabs.tabs.map((t) => ({ id: t.id, active: t.active, title: t.title, url: t.url })) : null,
|
|
159
|
+
};
|
|
160
|
+
printResult(out, opt.pretty);
|
|
161
|
+
return 0;
|
|
162
|
+
} catch (e) {
|
|
163
|
+
// 没跑**不是错误状态**:这是用户最常问的第一个问题,答案要清楚、可执行。
|
|
164
|
+
printResult({ running: false, port: opt.port, reason: e.message }, opt.pretty);
|
|
165
|
+
return 1;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// stdin 读成字符串(供 `... | ai-browser call ui.navigate -` 用)
|
|
170
|
+
function readStdin() {
|
|
171
|
+
try {
|
|
172
|
+
return require('node:fs').readFileSync(0, 'utf8');
|
|
173
|
+
} catch (_) {
|
|
174
|
+
return '';
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
module.exports = { runCall, runStatus, parsePayload, parseArgs, rpcCall, WS_PORT_DEFAULT };
|