@xiaobuyu/nodesign 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -106
- package/bin/nodesign.js +0 -0
- package/package.json +1 -1
- package/server/api/README.md +45 -45
- package/server/api/_guard.js +75 -75
- package/server/api/admin.js +187 -187
- package/server/api/assets/docx-page.js +103 -103
- package/server/api/assets/helpers.js +22 -22
- package/server/api/assets/notes.js +144 -144
- package/server/api/assets.js +896 -896
- package/server/api/board.js +57 -57
- package/server/api/browse.js +138 -138
- package/server/api/canvas.js +444 -444
- package/server/api/chatai.js +324 -324
- package/server/api/exports/build-standalone.js +980 -980
- package/server/api/exports/cards.js +118 -118
- package/server/api/exports/docx-pdf.js +46 -46
- package/server/api/exports/export-page.js +142 -142
- package/server/api/exports/handoff.js +147 -147
- package/server/api/exports.js +690 -690
- package/server/api/instruction.js +75 -75
- package/server/api/me.js +116 -116
- package/server/api/memory.js +185 -185
- package/server/api/pending-changes.js +366 -366
- package/server/api/plugins.js +164 -164
- package/server/api/projects.js +220 -220
- package/server/api/publish.js +64 -64
- package/server/api/recent.js +73 -73
- package/server/api/sessions.js +589 -589
- package/server/api/skills.js +77 -77
- package/server/api/standalone-fit.js +168 -168
- package/server/api/turn-compose.js +180 -180
- package/server/api/turn-inflight.js +44 -44
- package/server/auth/README.md +17 -17
- package/server/auth/middleware.js +138 -138
- package/server/auth/origin-guard.js +71 -71
- package/server/auth/session.js +108 -108
- package/server/auth/tier.js +113 -113
- package/server/auth/users-store.js +348 -348
- package/server/edit/README.md +29 -29
- package/server/engine/README.md +113 -113
- package/server/engine/agent/agent-shared.js +550 -550
- package/server/engine/agent/auto-mode-default-hard-deny.txt +4 -4
- package/server/engine/agent/auto-mode-rules.js +78 -78
- package/server/engine/agent/context.js +418 -418
- package/server/engine/agent/events.js +464 -464
- package/server/engine/agent/hooks/canvas-validate.js +199 -199
- package/server/engine/agent/hooks/failure.js +120 -120
- package/server/engine/agent/hooks/file-events.js +75 -75
- package/server/engine/agent/hooks/lifecycle.js +233 -233
- package/server/engine/agent/hooks/post-canvas-focus.js +80 -80
- package/server/engine/agent/hooks/post-guidance.js +206 -206
- package/server/engine/agent/hooks/post-subagent-report.js +104 -104
- package/server/engine/agent/hooks/post-trim.js +58 -58
- package/server/engine/agent/hooks/pre-board-neighborhood.js +34 -34
- package/server/engine/agent/hooks/pre-defaults.js +84 -84
- package/server/engine/agent/hooks/pre-injectors.js +277 -277
- package/server/engine/agent/hooks/pre-performance-log-guard.js +78 -78
- package/server/engine/agent/hooks/pre-starter-files.js +61 -61
- package/server/engine/agent/hooks/pre-workspace-scope-guard.js +78 -78
- package/server/engine/agent/hooks/site-validate.js +94 -94
- package/server/engine/agent/hooks/tool-prompts.js +31 -31
- package/server/engine/agent/hooks/turn-state-memory.js +56 -56
- package/server/engine/agent/hooks/user-prompt-submit.js +215 -215
- package/server/engine/agent/hooks.js +354 -354
- package/server/engine/agent/init-contract.js +69 -69
- package/server/engine/agent/isolation.js +159 -159
- package/server/engine/agent/model-table.js +292 -292
- package/server/engine/agent/plugin-loader.js +266 -266
- package/server/engine/agent/prompts/nodesign-prelude.md +336 -336
- package/server/engine/agent/prompts/tools/ask-user-question-protocol.md +71 -71
- package/server/engine/agent/prompts/tools/direct-edit-protocol.md +98 -98
- package/server/engine/agent/prompts/tools/generate-image-cookbook.gemini-gateway.md +583 -583
- package/server/engine/agent/prompts/tools/generate-image-cookbook.md +483 -483
- package/server/engine/agent/prompts/tools/hybrid-reference.md +100 -100
- package/server/engine/agent/prompts/tools/paint-still-cookbook.md +270 -270
- package/server/engine/agent/prompts/tools/roll-film-cookbook.md +70 -70
- package/server/engine/agent/prompts/tools/site-reference.md +106 -106
- package/server/engine/agent/prompts/tools/tweaks-syntax.md +125 -125
- package/server/engine/agent/prompts/tools/vision-checker-dispatch.md +48 -48
- package/server/engine/agent/session-loop.js +1024 -1024
- package/server/engine/agent/session-model.js +164 -164
- package/server/engine/agent/skill.js +204 -204
- package/server/engine/agent/task-events.js +78 -78
- package/server/engine/agents/ds-extractor.md +132 -132
- package/server/engine/agents/explorer.md +190 -190
- package/server/engine/agents/index.js +230 -230
- package/server/engine/agents/schemas/design-system.json +156 -156
- package/server/engine/agents/schemas/tweak-schema.json +113 -113
- package/server/engine/agents/tweak-proposer.md +127 -127
- package/server/engine/agents/vision-checker.md +254 -254
- package/server/engine/browse/capture.js +422 -422
- package/server/engine/browse/card.js +137 -137
- package/server/engine/browse/handover.js +59 -59
- package/server/engine/browse/page-digest.js +119 -119
- package/server/engine/browse/refs.js +196 -196
- package/server/engine/browse/registry.js +307 -307
- package/server/engine/browse/screencast.js +170 -170
- package/server/engine/browse/state.js +121 -121
- package/server/engine/chatai/chat-log.js +79 -79
- package/server/engine/chatai/index.js +193 -193
- package/server/engine/chatai/openai-compat.js +152 -152
- package/server/engine/chatai/orchestrate.js +309 -309
- package/server/engine/chatai/perform.js +40 -40
- package/server/engine/chatai/summarize.js +79 -79
- package/server/engine/mcp/capability-gate.js +52 -52
- package/server/engine/mcp/index.js +339 -339
- package/server/engine/mcp/param-sanitizer.js +142 -142
- package/server/engine/mcp/tools/arrange-on-board.js +124 -124
- package/server/engine/mcp/tools/artifact-session.js +278 -278
- package/server/engine/mcp/tools/browse-computer.js +421 -421
- package/server/engine/mcp/tools/browse-find-batch.js +183 -183
- package/server/engine/mcp/tools/browse-screenshot.js +157 -157
- package/server/engine/mcp/tools/browse.js +480 -480
- package/server/engine/mcp/tools/build-docx.js +101 -101
- package/server/engine/mcp/tools/clear-pending-changes.js +99 -99
- package/server/engine/mcp/tools/create-on-board.js +155 -155
- package/server/engine/mcp/tools/crystallize-skill.js +168 -168
- package/server/engine/mcp/tools/explain-style.js +237 -237
- package/server/engine/mcp/tools/export-handoff.js +133 -133
- package/server/engine/mcp/tools/expose-tweaks.js +149 -149
- package/server/engine/mcp/tools/generate-image.js +842 -842
- package/server/engine/mcp/tools/get-computed-styles.js +164 -164
- package/server/engine/mcp/tools/get-pending-changes.js +205 -205
- package/server/engine/mcp/tools/h3box-ssh.js +84 -84
- package/server/engine/mcp/tools/helpers/acquire-page.js +82 -82
- package/server/engine/mcp/tools/helpers/motion-lab.js +554 -554
- package/server/engine/mcp/tools/helpers/motion-scroll.js +112 -112
- package/server/engine/mcp/tools/helpers/perception-page.js +215 -215
- package/server/engine/mcp/tools/helpers/reference-download.js +103 -103
- package/server/engine/mcp/tools/helpers/rembg-bridge.py +80 -80
- package/server/engine/mcp/tools/helpers/shot-pipeline.js +280 -280
- package/server/engine/mcp/tools/highlight.js +68 -68
- package/server/engine/mcp/tools/list-pages.js +222 -222
- package/server/engine/mcp/tools/lookup-tags.js +59 -59
- package/server/engine/mcp/tools/navigate-to-page.js +60 -60
- package/server/engine/mcp/tools/organize-board.js +64 -64
- package/server/engine/mcp/tools/paint-still.js +434 -434
- package/server/engine/mcp/tools/pin-to-board.js +130 -130
- package/server/engine/mcp/tools/preview-deck.js +130 -130
- package/server/engine/mcp/tools/profile-scroll.js +305 -305
- package/server/engine/mcp/tools/publish-site.js +102 -102
- package/server/engine/mcp/tools/query-elements.js +195 -195
- package/server/engine/mcp/tools/read-board.js +103 -103
- package/server/engine/mcp/tools/read-document.js +93 -93
- package/server/engine/mcp/tools/read-page.js +264 -264
- package/server/engine/mcp/tools/read-tavern-json.js +143 -143
- package/server/engine/mcp/tools/record-decision.js +140 -140
- package/server/engine/mcp/tools/relate-on-board.js +110 -110
- package/server/engine/mcp/tools/report-issue.js +115 -115
- package/server/engine/mcp/tools/roll-film.js +266 -266
- package/server/engine/mcp/tools/screenshot-docx.js +138 -138
- package/server/engine/mcp/tools/screenshot-url.js +195 -195
- package/server/engine/mcp/tools/screenshot.js +583 -583
- package/server/engine/mcp/tools/tier-gate.js +96 -96
- package/server/engine/mcp/tools/trace-motion.js +215 -215
- package/server/engine/mcp/tools/web-search.js +548 -548
- package/server/engine/motion/inventory.js +349 -349
- package/server/engine/perception/session.js +235 -235
- package/server/engine/plugins/nodesign/.claude-plugin/plugin.json +5 -5
- package/server/engine/plugins/nodesign/skills/deskskill-engine-mini/SKILL.md +192 -192
- package/server/engine/plugins/nodesign/skills/deskskill-engine-mini/canvas.template.html +667 -667
- package/server/engine/plugins/nodesign/skills/deskskill-engine-mini/patterns/hybrid-grid.md +22 -22
- package/server/engine/plugins/nodesign/skills/deskskill-engine-mini/patterns/image-led-cover.md +25 -25
- package/server/engine/plugins/nodesign/skills/deskskill-engine-mini/patterns/portrait.md +23 -23
- package/server/engine/plugins/nodesign/skills/deskskill-engine-mini/patterns/quote-backdrop.md +21 -21
- package/server/engine/plugins/nodesign/skills/deskskill-engine-mini/patterns/section-divider.md +23 -23
- package/server/engine/plugins/nodesign/skills/deskskill-engine-mini/patterns/text-led.md +21 -21
- package/server/engine/plugins/nodesign/skills/docx-craft/SKILL.md +162 -162
- package/server/engine/plugins/nodesign/skills/docx-craft/references/token-schema.md +407 -407
- package/server/engine/plugins/nodesign/skills/docx-craft//346/226/207/346/241/243.template.json +80 -80
- package/server/engine/plugins/nodesign/skills/rp-craft/SKILL.md +228 -228
- package/server/engine/plugins/nodesign/skills/rp-craft/patterns//346/274/224/345/207/272/351/241/265-/346/234/200/345/260/217/345/256/236/347/216/260.html +78 -78
- package/server/engine/plugins/nodesign/skills/rp-craft//346/274/224/345/207/272.template.js +268 -268
- package/server/engine/plugins/nodesign/skills/site-craft/SKILL.md +323 -323
- package/server/engine/plugins/nodesign/skills/site-craft/patterns/build-lane.md +134 -134
- package/server/engine/plugins/nodesign/skills/site-craft/patterns/cutout-collage.md +61 -61
- package/server/engine/plugins/nodesign/skills/site-craft/patterns/mock-app.md +163 -163
- package/server/engine/plugins/nodesign/skills/site-craft/patterns/page-transitions.md +86 -86
- package/server/engine/runs/active-runs.js +698 -698
- package/server/engine/runs/live-turn.js +252 -252
- package/server/engine/runs/store.js +364 -364
- package/server/engine/runs/turn-relay.js +217 -217
- package/server/engine/runtime/workspace.js +164 -164
- package/server/lib/artifact-file-path.js +80 -80
- package/server/lib/artifact-target.js +284 -284
- package/server/lib/asset-refs.js +239 -239
- package/server/lib/async-queue.js +104 -104
- package/server/lib/auto-relations.js +130 -130
- package/server/lib/binding-types.js +60 -60
- package/server/lib/board-kind-sizes.js +62 -62
- package/server/lib/board-relations.js +133 -133
- package/server/lib/browse-proxy.js +180 -180
- package/server/lib/canvas-id.js +36 -36
- package/server/lib/cover.js +227 -227
- package/server/lib/danbooru-tags.js +291 -291
- package/server/lib/doc-extract.js +156 -156
- package/server/lib/docx/build-from-source.js +260 -260
- package/server/lib/docx/build.js +490 -490
- package/server/lib/docx/dump-styles.js +287 -287
- package/server/lib/docx/fonts/nodesign-cjk.conf +164 -164
- package/server/lib/docx/merge-runs.js +129 -129
- package/server/lib/docx/numbering.js +218 -218
- package/server/lib/docx/order.js +152 -152
- package/server/lib/docx/rawzip.js +172 -172
- package/server/lib/docx/render.js +92 -92
- package/server/lib/docx/text-lint.js +225 -225
- package/server/lib/docx/tokens.js +335 -335
- package/server/lib/docx/units.js +29 -29
- package/server/lib/docx/xml.js +291 -291
- package/server/lib/docx-pages.js +187 -187
- package/server/lib/export-collect.js +289 -289
- package/server/lib/export-package.js +210 -210
- package/server/lib/html-srcset.js +145 -145
- package/server/lib/image-variant.js +460 -460
- package/server/lib/ingress/forward-openai-chat.js +305 -305
- package/server/lib/ingress/openai-chat.js +491 -491
- package/server/lib/ingress/session-notice.js +66 -66
- package/server/lib/ingress/session-routes.js +68 -68
- package/server/lib/ingress/slot-probe.js +130 -130
- package/server/lib/ingress/upstream-billing.js +52 -52
- package/server/lib/ingress/upstream-fail-streak.js +72 -72
- package/server/lib/ingress/upstream-truncation.js +48 -48
- package/server/lib/issues-store.js +182 -182
- package/server/lib/kinds/deck.js +72 -72
- package/server/lib/kinds/docx.js +283 -283
- package/server/lib/kinds/file-kinds.js +113 -113
- package/server/lib/kinds/index.js +202 -202
- package/server/lib/kinds/site.js +161 -161
- package/server/lib/model-ingress.js +543 -543
- package/server/lib/moderation.js +255 -255
- package/server/lib/notice-store.js +103 -103
- package/server/lib/plugin-install.js +104 -104
- package/server/lib/plugin-validator.js +721 -721
- package/server/lib/publish-store.js +115 -115
- package/server/lib/quick-summary.js +52 -52
- package/server/lib/quota.js +270 -270
- package/server/lib/rate-window.js +29 -29
- package/server/lib/reference-assets.js +92 -92
- package/server/lib/region-shot.js +135 -135
- package/server/lib/safe-path.js +73 -73
- package/server/lib/sdk-session.js +33 -33
- package/server/lib/showcase-store.js +92 -92
- package/server/lib/site-publish.js +465 -465
- package/server/lib/ssrf-guard.js +432 -432
- package/server/lib/task-scan.js +158 -158
- package/server/lib/tavern-json.js +154 -154
- package/server/lib/video-variant.js +237 -237
- package/server/lib/workspace-path.js +33 -33
- package/server/ops/fix-sdk-musl.mjs +40 -40
- package/server/ops/install-macos-fonts.sh +114 -114
- package/server/ops/macos-fonts.conf +336 -336
- package/server/ops/sandbox-shim/bwrap +57 -57
- package/server/projects/assets-summary.js +119 -119
- package/server/projects/auto-name.js +53 -53
- package/server/projects/board-store.js +640 -640
- package/server/projects/move-entry.js +103 -103
- package/server/projects/store.js +270 -270
- package/server/projects/ui-config.js +54 -54
- package/server/projects/workspace-templates.js +53 -53
- package/server/projects/workspace.js +1025 -1025
- package/server/runtime/local-config.js +177 -177
- package/server/runtime/profile.js +72 -72
- package/server/services/rembg-service.py +334 -334
- package/server/shared/README.md +25 -25
- package/server/shared/deck.js +54 -54
- package/server/shared/time.js +49 -49
- package/server/style-pipeline/README.md +53 -53
- package/server/ws/broker.js +36 -36
- package/server/ws/browse-channel.js +177 -177
- package/server/ws/index.js +386 -386
- package/web/README.md +89 -89
- package/web/dist/assets/BrowserWindow-BYCgizHw.js +11 -0
- package/web/dist/assets/{DeckWindow-4G0xX7P3.js → DeckWindow-pNydEM4L.js} +1 -1
- package/web/dist/assets/{DocxWindow-BXQsoPkh.js → DocxWindow-PPgzh3Gq.js} +1 -1
- package/web/dist/assets/{SiteWindow-D6eS-2gM.js → SiteWindow-DM0LQPCO.js} +1 -1
- package/web/dist/assets/{index-Cbl3ATvp.js → index-BHTBJlpY.js} +32 -32
- package/web/dist/assets/{pending-edit-apply-DTjo2geW.js → pending-edit-apply-C0I13CbD.js} +1 -1
- package/web/dist/assets/{rotate-cw-LMT3x1Zj.js → rotate-cw-B1UQ8Hld.js} +1 -1
- package/web/dist/assets/{square-dashed-mouse-pointer-CBoAkfhi.js → square-dashed-mouse-pointer-8lrgqmfv.js} +1 -1
- package/web/dist/index.html +17 -17
- package/web/dist/assets/BrowserWindow-Dc7J7CYc.js +0 -11
|
@@ -1,421 +1,421 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* mcp/tools/browse-computer.js — browser_computer:坐标/引用级的指针与键盘(2026-08-21)
|
|
3
|
-
*
|
|
4
|
-
* 形状照 Anthropic 的 browser use toolset(`browser_toolset_20260801`)的指针/键盘/
|
|
5
|
-
* 截图成员:left_click … zoom 那一组,成员名、参数名、语义一字不改;外壳是 MCP 单
|
|
6
|
-
* 工具 + `action` 枚举(Claude in Chrome 的 `computer` 工具就是这么包的)。
|
|
7
|
-
* 规格里的 `target`(coordinate | ref 二选一)拆成 `coordinate` / `ref` 两个字段。
|
|
8
|
-
*
|
|
9
|
-
* ## 为什么不等 toolset 进 Agent SDK
|
|
10
|
-
*
|
|
11
|
-
* toolset 是 Messages API 请求体里的东西,Agent SDK 的设计点是"自带工具 + MCP",
|
|
12
|
-
* 两边不相交(08-21 查过 0.3.237/0.3.238 两版二进制,零痕迹)。而模型认的是
|
|
13
|
-
* **动作名和参数形状**,不是 `toolset_name` 字段 —— Claude in Chrome 用 MCP 包同一
|
|
14
|
-
* 套动作照样熟练。所以这里直接做 MCP,订阅通路和本地模型通路一视同仁。
|
|
15
|
-
*
|
|
16
|
-
* ## 坐标空间 = 截图像素;frame 负责它和页面像素的换算
|
|
17
|
-
*
|
|
18
|
-
* `runAction(page, a, { frame, shot })` 对"页面从哪来"无知:浏览通道传
|
|
19
|
-
* frame={1366,768,scale 1}(视口 1366×768 在 shot-pipeline 阈值内,截图不缩,
|
|
20
|
-
* 1:1);产物会话(artifact_computer)按产物视口算 frame,scale 可能 <1,模型读到
|
|
21
|
-
* 的坐标 ÷ scale 才是页面像素(文档推荐做法)。所有回给模型的坐标都是截图空间。
|
|
22
|
-
*
|
|
23
|
-
* ## 闸不用动
|
|
24
|
-
*
|
|
25
|
-
* 出网闸在进程内 HTTP 代理(lib/browse-proxy.js),在 Playwright 之下;点哪里都
|
|
26
|
-
* 绕不过去。这是 headless 页面,没有地址栏、没有 OS 对话框,不存在桌面 computer
|
|
27
|
-
* use 那种"从浏览器逃到终端"的面。
|
|
28
|
-
*/
|
|
29
|
-
|
|
30
|
-
import { tool } from '@anthropic-ai/claude-agent-sdk';
|
|
31
|
-
import { z } from 'zod';
|
|
32
|
-
import { withBrowser, _limits } from '../../browse/registry.js';
|
|
33
|
-
import { handleForRef, staleRefText } from '../../browse/refs.js';
|
|
34
|
-
import { recordVisit, saveFrame } from '../../browse/state.js';
|
|
35
|
-
import { normalizeShot } from './helpers/shot-pipeline.js';
|
|
36
|
-
|
|
37
|
-
const VP = _limits.VIEWPORT;
|
|
38
|
-
/** 浏览通道的截图空间:视口像素 1:1(browse-computer.test.js 守着这个前提) */
|
|
39
|
-
export const BROWSE_FRAME = { w: VP.width, h: VP.height, scale: 1 };
|
|
40
|
-
const asText = (text, isError = false) => ({ content: [{ type: 'text', text }], ...(isError ? { isError: true } : {}) });
|
|
41
|
-
|
|
42
|
-
export const ACTIONS = [
|
|
43
|
-
'screenshot', 'zoom',
|
|
44
|
-
'left_click', 'right_click', 'middle_click', 'double_click', 'triple_click', 'hover',
|
|
45
|
-
'left_click_drag', 'left_mouse_down', 'left_mouse_up', 'mouse_move',
|
|
46
|
-
'scroll', 'scroll_to',
|
|
47
|
-
'type', 'key', 'hold_key', 'wait',
|
|
48
|
-
];
|
|
49
|
-
|
|
50
|
-
// ── 键名 / 修饰键:模型写的是 xdotool 风格(Return、ctrl+s、Page_Down),
|
|
51
|
-
// Playwright 要 DOM 键名(Enter、Control+s、PageDown)──
|
|
52
|
-
const KEY_ALIAS = {
|
|
53
|
-
return: 'Enter', enter: 'Enter', esc: 'Escape', escape: 'Escape', tab: 'Tab', space: 'Space',
|
|
54
|
-
backspace: 'Backspace', delete: 'Delete', del: 'Delete', insert: 'Insert',
|
|
55
|
-
up: 'ArrowUp', down: 'ArrowDown', left: 'ArrowLeft', right: 'ArrowRight',
|
|
56
|
-
arrowup: 'ArrowUp', arrowdown: 'ArrowDown', arrowleft: 'ArrowLeft', arrowright: 'ArrowRight',
|
|
57
|
-
home: 'Home', end: 'End', pageup: 'PageUp', page_up: 'PageUp', pagedown: 'PageDown', page_down: 'PageDown',
|
|
58
|
-
capslock: 'CapsLock', caps_lock: 'CapsLock',
|
|
59
|
-
};
|
|
60
|
-
const MOD_ALIAS = {
|
|
61
|
-
shift: 'Shift', ctrl: 'Control', control: 'Control', alt: 'Alt', option: 'Alt',
|
|
62
|
-
cmd: 'Meta', meta: 'Meta', super: 'Meta', win: 'Meta', windows: 'Meta',
|
|
63
|
-
};
|
|
64
|
-
|
|
65
|
-
/** 'ctrl+shift' → ['Control','Shift'];不认识的词抛错(别静默吞成无修饰) */
|
|
66
|
-
export function parseModifiers(s) {
|
|
67
|
-
if (!s) return [];
|
|
68
|
-
return String(s).split('+').map(p => p.trim()).filter(Boolean).map((p) => {
|
|
69
|
-
const m = MOD_ALIAS[p.toLowerCase()];
|
|
70
|
-
if (!m) throw new Error(`unknown modifier "${p}" (use shift / ctrl / alt / cmd, joined with +)`);
|
|
71
|
-
return m;
|
|
72
|
-
});
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
/** 'ctrl+s' / 'Return' / 'Backspace Backspace' → [['Control','s'], ...](每个和弦 = 修饰键…+主键) */
|
|
76
|
-
export function parseChords(text) {
|
|
77
|
-
const chords = String(text || '').trim().split(/\s+/).filter(Boolean);
|
|
78
|
-
if (!chords.length) throw new Error('key needs a key name, e.g. "Enter", "ctrl+a", "Backspace Backspace"');
|
|
79
|
-
return chords.map((chord) => {
|
|
80
|
-
const parts = chord.split('+').filter(Boolean);
|
|
81
|
-
if (chord === '+' || !parts.length) parts.splice(0, parts.length, '+');
|
|
82
|
-
const keyRaw = parts[parts.length - 1];
|
|
83
|
-
const mods = parts.slice(0, -1).map((p) => {
|
|
84
|
-
const m = MOD_ALIAS[p.toLowerCase()];
|
|
85
|
-
if (!m) throw new Error(`unknown modifier "${p}" in "${chord}"`);
|
|
86
|
-
return m;
|
|
87
|
-
});
|
|
88
|
-
const lower = keyRaw.toLowerCase();
|
|
89
|
-
let key = KEY_ALIAS[lower];
|
|
90
|
-
if (!key && MOD_ALIAS[lower]) throw new Error(`"${chord}" is only modifiers — add the key to press (e.g. "${chord}+a"), or use hold_key`);
|
|
91
|
-
if (!key && /^f([1-9]|1[0-2])$/i.test(keyRaw)) key = `F${keyRaw.slice(1)}`;
|
|
92
|
-
if (!key) key = keyRaw.length === 1 ? keyRaw : keyRaw[0].toUpperCase() + keyRaw.slice(1);
|
|
93
|
-
return [...mods, key];
|
|
94
|
-
});
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
/** 坐标校验(截图空间):必须落在 frame 里 —— 坐标空间就是截图像素,越界只能是读错了图 */
|
|
98
|
-
export function checkCoord(c, frame = BROWSE_FRAME, label = 'coordinate') {
|
|
99
|
-
if (!Array.isArray(c) || c.length !== 2 || !c.every(n => Number.isFinite(n))) {
|
|
100
|
-
return `${label} must be [x, y] in screenshot pixels`;
|
|
101
|
-
}
|
|
102
|
-
const [x, y] = c;
|
|
103
|
-
if (x < 0 || y < 0 || x > frame.w || y > frame.h) {
|
|
104
|
-
return `${label} (${x}, ${y}) is outside the ${frame.w}×${frame.h} screenshot — coordinates are screenshot pixels, origin top-left`;
|
|
105
|
-
}
|
|
106
|
-
return null;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
const toPage = (v, frame) => v / frame.scale;
|
|
110
|
-
const toShot = (v, frame) => Math.round(v * frame.scale);
|
|
111
|
-
|
|
112
|
-
const cursors = new WeakMap(); // page → {x,y} 页面像素,给省略坐标的 mouse_down/up/scroll 用
|
|
113
|
-
const cursorOf = (page, frame) => cursors.get(page) || { x: toPage(frame.w / 2, frame), y: toPage(frame.h / 2, frame) };
|
|
114
|
-
const moveTo = async (page, x, y) => { await page.mouse.move(x, y); cursors.set(page, { x, y }); };
|
|
115
|
-
|
|
116
|
-
/**
|
|
117
|
-
* coordinate / ref → {x,y(页面像素), sx,sy(截图像素), viaRef}。ref 会先滚进视口再取几何中心。
|
|
118
|
-
*/
|
|
119
|
-
async function resolveTarget(page, { coordinate, ref }, frame, { required = true } = {}) {
|
|
120
|
-
if (ref) {
|
|
121
|
-
const h = await handleForRef(page, ref);
|
|
122
|
-
if (!h) return { error: staleRefText(ref) };
|
|
123
|
-
await h.scrollIntoViewIfNeeded({ timeout: 3000 }).catch(() => {});
|
|
124
|
-
const box = await h.boundingBox();
|
|
125
|
-
if (!box) return { error: `Error: ${ref} has no visible box on the page (hidden or zero-size).` };
|
|
126
|
-
const x = box.x + box.width / 2;
|
|
127
|
-
const y = box.y + box.height / 2;
|
|
128
|
-
const sx = toShot(x, frame); const sy = toShot(y, frame);
|
|
129
|
-
// 滚了还在视口外(典型:藏在屏幕上方的 skip link)—— 别夹到边上去点个错的地方
|
|
130
|
-
if (sx < 0 || sy < 0 || sx > frame.w || sy > frame.h) {
|
|
131
|
-
return { error: `Error: ${ref} sits off-screen at (${sx}, ${sy}) even after scrolling it into view (probably a visually-hidden element) — pick another match or click a coordinate you can see.` };
|
|
132
|
-
}
|
|
133
|
-
// 真跑逮到的:站点自己的开场幕布盖着整页,点 ref 其实点在幕布上,页面一点反应
|
|
134
|
-
// 没有,agent 只能猜。查一下这个点最上层是谁;不是 ref 本身或它的祖/后代就报出来。
|
|
135
|
-
let coveredBy = null;
|
|
136
|
-
try {
|
|
137
|
-
coveredBy = await h.evaluate((el, [px, py]) => {
|
|
138
|
-
const top = document.elementFromPoint(px, py);
|
|
139
|
-
if (!top || top === el || el.contains(top) || top.contains(el)) return null;
|
|
140
|
-
const cls = typeof top.className === 'string' && top.className.trim() ? `.${top.className.trim().split(/\s+/)[0]}` : '';
|
|
141
|
-
return `${top.tagName.toLowerCase()}${top.id ? `#${top.id}` : ''}${cls}`;
|
|
142
|
-
}, [x, y]);
|
|
143
|
-
} catch { /* 查不到就不报 */ }
|
|
144
|
-
return { x, y, sx, sy, viaRef: ref, coveredBy };
|
|
145
|
-
}
|
|
146
|
-
if (coordinate) {
|
|
147
|
-
const bad = checkCoord(coordinate, frame);
|
|
148
|
-
if (bad) return { error: `Error: ${bad}` };
|
|
149
|
-
return { x: toPage(coordinate[0], frame), y: toPage(coordinate[1], frame), sx: coordinate[0], sy: coordinate[1] };
|
|
150
|
-
}
|
|
151
|
-
if (required) return { error: 'Error: this action needs a coordinate [x, y] or a ref (from the find tool).' };
|
|
152
|
-
const c = cursorOf(page, frame);
|
|
153
|
-
return { ...c, sx: toShot(c.x, frame), sy: toShot(c.y, frame) };
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
async function withModifiers(page, mods, fn) {
|
|
157
|
-
for (const m of mods) await page.keyboard.down(m);
|
|
158
|
-
try { return await fn(); } finally {
|
|
159
|
-
for (const m of [...mods].reverse()) await page.keyboard.up(m).catch(() => {});
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
/** 浏览通道的视口截图:存桌面卡预览 + 归一化(1366×768 在阈值内,不缩)→ 文本块在前,图在后 */
|
|
164
|
-
export async function viewportShot(page, projectId, lead) {
|
|
165
|
-
const buf = await page.screenshot({ type: 'png' });
|
|
166
|
-
await saveFrame(projectId, buf);
|
|
167
|
-
const shot = await normalizeShot(buf);
|
|
168
|
-
return {
|
|
169
|
-
content: [
|
|
170
|
-
{ type: 'text', text: [lead, `viewport ${VP.width}×${VP.height} — coordinates you use next are these pixels`, shot.note].filter(Boolean).join(' · ') },
|
|
171
|
-
{ type: 'image', data: shot.data, mimeType: shot.mimeType },
|
|
172
|
-
],
|
|
173
|
-
};
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
async function zoomShot(page, region, frame, lead) {
|
|
177
|
-
if (!Array.isArray(region) || region.length !== 4 || !region.every(n => Number.isFinite(n))) {
|
|
178
|
-
return asText('Error: zoom needs region [x0, y0, x1, y1] in screenshot pixels.', true);
|
|
179
|
-
}
|
|
180
|
-
const [x0, y0, x1, y1] = region.map(Math.round);
|
|
181
|
-
if (x0 < 0 || y0 < 0 || x1 > frame.w || y1 > frame.h || x1 - x0 < 8 || y1 - y0 < 8) {
|
|
182
|
-
return asText(`Error: zoom region [${region.join(', ')}] must lie inside the ${frame.w}×${frame.h} screenshot and be at least 8×8.`, true);
|
|
183
|
-
}
|
|
184
|
-
const clip = { x: toPage(x0, frame), y: toPage(y0, frame), width: toPage(x1 - x0, frame), height: toPage(y1 - y0, frame) };
|
|
185
|
-
const buf = await page.screenshot({ type: 'png', clip });
|
|
186
|
-
const { default: sharp } = await import('sharp');
|
|
187
|
-
const k = Math.min(frame.w / (x1 - x0), frame.h / (y1 - y0)); // 放大到塞满截图空间
|
|
188
|
-
const up = await sharp(buf)
|
|
189
|
-
.resize({ width: Math.round((x1 - x0) * k), height: Math.round((y1 - y0) * k), fit: 'inside', withoutEnlargement: false, kernel: 'lanczos3' })
|
|
190
|
-
.png().toBuffer();
|
|
191
|
-
const shot = await normalizeShot(up);
|
|
192
|
-
return {
|
|
193
|
-
content: [
|
|
194
|
-
{ type: 'text', text: `${lead}zoom of [${x0},${y0},${x1},${y1}] (${x1 - x0}×${y1 - y0}, ×${(k / frame.scale).toFixed(1)}) — the coordinates you use next are STILL full-screenshot pixels, not pixels of this zoomed image` },
|
|
195
|
-
{ type: 'image', data: shot.data, mimeType: shot.mimeType },
|
|
196
|
-
],
|
|
197
|
-
};
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
/**
|
|
201
|
-
* 跑一个动作。返回 CallToolResult。导航变化与闸拦截由调用方补到文本里。
|
|
202
|
-
* @param {import('playwright').Page} page
|
|
203
|
-
* @param {object} a 工具入参
|
|
204
|
-
* @param {{frame:{w:number,h:number,scale:number}, shot:(page:any, lead:string)=>Promise<any>}} env
|
|
205
|
-
*/
|
|
206
|
-
export async function runAction(page, a, { frame, shot }) {
|
|
207
|
-
const { action } = a;
|
|
208
|
-
if (action === 'screenshot') return shot(page, 'screenshot');
|
|
209
|
-
if (action === 'zoom') return zoomShot(page, a.region, frame, '');
|
|
210
|
-
if (action === 'wait') {
|
|
211
|
-
const s = Math.min(30, Math.max(0, Number(a.duration) || 0));
|
|
212
|
-
await page.waitForTimeout(s * 1000);
|
|
213
|
-
return asText(`waited ${s}s`);
|
|
214
|
-
}
|
|
215
|
-
if (action === 'type') {
|
|
216
|
-
if (typeof a.text !== 'string' || !a.text.length) return asText('Error: type needs text.', true);
|
|
217
|
-
await page.keyboard.type(a.text);
|
|
218
|
-
return asText(`typed ${a.text.length} chars at the current focus`);
|
|
219
|
-
}
|
|
220
|
-
if (action === 'key') {
|
|
221
|
-
const chords = parseChords(a.text);
|
|
222
|
-
const n = Math.min(100, Math.max(1, Math.round(Number(a.repeat) || 1)));
|
|
223
|
-
for (let i = 0; i < n; i += 1) for (const c of chords) await page.keyboard.press(c.join('+'));
|
|
224
|
-
return asText(`pressed ${chords.map(c => c.join('+')).join(' ')}${n > 1 ? ` ×${n}` : ''}`);
|
|
225
|
-
}
|
|
226
|
-
if (action === 'hold_key') {
|
|
227
|
-
const [chord] = parseChords(a.text);
|
|
228
|
-
const s = Math.min(30, Math.max(0, Number(a.duration) || 0));
|
|
229
|
-
for (const k of chord) await page.keyboard.down(k);
|
|
230
|
-
try { await page.waitForTimeout(s * 1000); } finally { for (const k of [...chord].reverse()) await page.keyboard.up(k).catch(() => {}); }
|
|
231
|
-
return asText(`held ${chord.join('+')} for ${s}s`);
|
|
232
|
-
}
|
|
233
|
-
if (action === 'scroll_to') {
|
|
234
|
-
if (!a.ref) return asText('Error: scroll_to needs a ref (from the find tool).', true);
|
|
235
|
-
const h = await handleForRef(page, a.ref);
|
|
236
|
-
if (!h) return asText(staleRefText(a.ref), true);
|
|
237
|
-
await h.evaluate(el => el.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' }));
|
|
238
|
-
await page.waitForTimeout(200);
|
|
239
|
-
const box = await h.boundingBox();
|
|
240
|
-
return asText(box ? `scrolled ${a.ref} into view — now at (${toShot(box.x + box.width / 2, frame)}, ${toShot(box.y + box.height / 2, frame)})` : `scrolled toward ${a.ref} (it has no visible box)`);
|
|
241
|
-
}
|
|
242
|
-
if (action === 'scroll') {
|
|
243
|
-
const t = await resolveTarget(page, a, frame, { required: false });
|
|
244
|
-
if (t.error) return asText(t.error, true);
|
|
245
|
-
const dir = a.scroll_direction;
|
|
246
|
-
if (!['up', 'down', 'left', 'right'].includes(dir)) return asText('Error: scroll needs scroll_direction up|down|left|right.', true);
|
|
247
|
-
const amt = Math.min(10, Math.max(1, Math.round(Number(a.scroll_amount) || 3)));
|
|
248
|
-
const mods = parseModifiers(a.modifiers);
|
|
249
|
-
const before = await page.evaluate(() => [window.scrollX, window.scrollY]);
|
|
250
|
-
await moveTo(page, t.x, t.y);
|
|
251
|
-
const d = 100 * amt;
|
|
252
|
-
await withModifiers(page, mods, () => page.mouse.wheel(
|
|
253
|
-
dir === 'left' ? -d : dir === 'right' ? d : 0,
|
|
254
|
-
dir === 'up' ? -d : dir === 'down' ? d : 0,
|
|
255
|
-
));
|
|
256
|
-
await page.waitForTimeout(250);
|
|
257
|
-
const after = await page.evaluate(() => [window.scrollX, window.scrollY]);
|
|
258
|
-
const moved = before[0] !== after[0] || before[1] !== after[1];
|
|
259
|
-
return asText(`scrolled ${dir} ${amt} notches at (${t.sx}, ${t.sy})`
|
|
260
|
-
+ (moved ? ` — scrollY ${before[1]} → ${after[1]}${before[0] !== after[0] ? `, scrollX ${before[0]} → ${after[0]}` : ''}`
|
|
261
|
-
: ' — the window itself did not move (maybe a scrollable panel under the cursor took it, or the page is at its end; try the keyboard: key PageDown)'));
|
|
262
|
-
}
|
|
263
|
-
// ── 指针动作 ──
|
|
264
|
-
const mods = parseModifiers(a.modifiers);
|
|
265
|
-
if (action === 'left_click_drag') {
|
|
266
|
-
const from = await resolveTarget(page, { coordinate: a.start_coordinate }, frame, { required: true });
|
|
267
|
-
if (from.error) return asText(from.error.replace('this action needs a coordinate', 'left_click_drag needs start_coordinate'), true);
|
|
268
|
-
const to = await resolveTarget(page, a, frame);
|
|
269
|
-
if (to.error) return asText(to.error, true);
|
|
270
|
-
await withModifiers(page, mods, async () => {
|
|
271
|
-
await moveTo(page, from.x, from.y);
|
|
272
|
-
await page.mouse.down();
|
|
273
|
-
await page.mouse.move(to.x, to.y, { steps: 12 });
|
|
274
|
-
await page.mouse.up();
|
|
275
|
-
});
|
|
276
|
-
cursors.set(page, { x: to.x, y: to.y });
|
|
277
|
-
return asText(`dragged (${from.sx}, ${from.sy}) → (${to.sx}, ${to.sy})`);
|
|
278
|
-
}
|
|
279
|
-
if (action === 'left_mouse_down' || action === 'left_mouse_up') {
|
|
280
|
-
const t = await resolveTarget(page, a, frame, { required: false });
|
|
281
|
-
if (t.error) return asText(t.error, true);
|
|
282
|
-
await moveTo(page, t.x, t.y);
|
|
283
|
-
if (action === 'left_mouse_down') await page.mouse.down(); else await page.mouse.up();
|
|
284
|
-
return asText(`${action} at (${t.sx}, ${t.sy})`);
|
|
285
|
-
}
|
|
286
|
-
const t = await resolveTarget(page, a, frame);
|
|
287
|
-
if (t.error) return asText(t.error, true);
|
|
288
|
-
const at = `(${t.sx}, ${t.sy})${t.viaRef ? ` = ${t.viaRef}` : ''}`
|
|
289
|
-
+ (t.coveredBy ? ` — ⚠ that point is covered by <${t.coveredBy}>: the ${action} landed on it, not on ${t.viaRef} (overlay / curtain / modal? dismiss it first or screenshot to see)` : '');
|
|
290
|
-
if (action === 'mouse_move' || action === 'hover') {
|
|
291
|
-
await moveTo(page, t.x, t.y);
|
|
292
|
-
await page.waitForTimeout(150);
|
|
293
|
-
return asText(`${action} ${at}`);
|
|
294
|
-
}
|
|
295
|
-
const CLICKS = {
|
|
296
|
-
left_click: { button: 'left', clickCount: 1 },
|
|
297
|
-
right_click: { button: 'right', clickCount: 1 },
|
|
298
|
-
middle_click: { button: 'middle', clickCount: 1 },
|
|
299
|
-
double_click: { button: 'left', clickCount: 2 },
|
|
300
|
-
triple_click: { button: 'left', clickCount: 3 },
|
|
301
|
-
};
|
|
302
|
-
const spec = CLICKS[action];
|
|
303
|
-
if (!spec) return asText(`Error: unknown action "${action}".`, true);
|
|
304
|
-
await withModifiers(page, mods, async () => {
|
|
305
|
-
await moveTo(page, t.x, t.y);
|
|
306
|
-
await page.mouse.click(t.x, t.y, spec);
|
|
307
|
-
});
|
|
308
|
-
return asText(`${action} ${at}${mods.length ? ` with ${mods.join('+')}` : ''}`);
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
/** 动作描述正文,browser_computer / artifact_computer 共用(坐标空间那句由各自补) */
|
|
312
|
-
export const ACTIONS_DOC = `Actions (params in parentheses):
|
|
313
|
-
screenshot () current view as an image
|
|
314
|
-
zoom (region [x0,y0,x1,y1]) region at higher magnification, for small text/icons
|
|
315
|
-
left_click | right_click | middle_click | double_click | triple_click (coordinate or ref, modifiers?)
|
|
316
|
-
hover | mouse_move (coordinate or ref)
|
|
317
|
-
left_click_drag (start_coordinate, coordinate)
|
|
318
|
-
left_mouse_down | left_mouse_up (coordinate?) custom drags; pair them, move in between
|
|
319
|
-
scroll (scroll_direction, scroll_amount? 1-10 notches, coordinate?)
|
|
320
|
-
scroll_to (ref) scroll an element into view
|
|
321
|
-
type (text) literal text at the current focus
|
|
322
|
-
key (text, repeat? 1-100) "Enter", "ctrl+a", "Backspace Backspace", "alt+Tab"
|
|
323
|
-
hold_key (text, duration ≤30s)
|
|
324
|
-
wait (duration ≤30s)`;
|
|
325
|
-
|
|
326
|
-
/** 入参 schema,两个 computer 工具共用 */
|
|
327
|
-
export const COMPUTER_SCHEMA = {
|
|
328
|
-
action: z.enum(ACTIONS).describe('Which action to perform (see the list above).'),
|
|
329
|
-
coordinate: z.array(z.number()).length(2).optional()
|
|
330
|
-
.describe('[x, y] screenshot pixels. Target for clicks/hover/mouse_move/scroll, and the END point of left_click_drag.'),
|
|
331
|
-
start_coordinate: z.array(z.number()).length(2).optional()
|
|
332
|
-
.describe('[x, y] START point for left_click_drag.'),
|
|
333
|
-
ref: z.string().optional()
|
|
334
|
-
.describe('Element reference from the find tool (e.g. "ref_3"). Alternative to coordinate for clicks/hover; required for scroll_to.'),
|
|
335
|
-
region: z.array(z.number()).length(4).optional()
|
|
336
|
-
.describe('[x0, y0, x1, y1] for zoom: top-left and bottom-right corners in screenshot pixels.'),
|
|
337
|
-
text: z.string().optional()
|
|
338
|
-
.describe('For type: the literal text. For key/hold_key: a key or +-chord, space-separated for a sequence ("Enter", "ctrl+s", "shift+Tab Tab").'),
|
|
339
|
-
modifiers: z.string().optional()
|
|
340
|
-
.describe('Modifier chord held during a click or scroll: shift / ctrl / alt / cmd, joined with + (e.g. "ctrl+shift").'),
|
|
341
|
-
scroll_direction: z.enum(['up', 'down', 'left', 'right']).optional().describe('For scroll.'),
|
|
342
|
-
scroll_amount: z.number().int().min(1).max(10).optional().describe('For scroll: wheel notches, default 3.'),
|
|
343
|
-
repeat: z.number().int().min(1).max(100).optional().describe('For key: press the sequence this many times, default 1.'),
|
|
344
|
-
duration: z.number().min(0).max(30).optional().describe('Seconds, for wait and hold_key (max 30).'),
|
|
345
|
-
};
|
|
346
|
-
|
|
347
|
-
/** 动作抛的 Playwright 内部错 → agent 能行动的话(两个 computer 工具共用) */
|
|
348
|
-
export function actionErrorText(action, err) {
|
|
349
|
-
const msg = String(err?.message || err).split('\n')[0];
|
|
350
|
-
if (/Execution context was destroyed|Target closed|frame was detached/i.test(msg)) {
|
|
351
|
-
return `Error: ${action} hit a page navigation in progress (${msg}). The page has changed underneath you — refs from before are stale; find again or screenshot first.`;
|
|
352
|
-
}
|
|
353
|
-
return `Error: ${action} failed: ${msg}`;
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
export function makeBrowserComputerTool({ projectId, ctx }) {
|
|
357
|
-
return tool(
|
|
358
|
-
'browser_computer',
|
|
359
|
-
`Pointer, keyboard and pixel-level capture on the current page of this project's
|
|
360
|
-
browser — the coordinate half of browsing. browser_click/browser_read work by
|
|
361
|
-
selector and text; this one works the way a person does: look at a screenshot,
|
|
362
|
-
click at a pixel, type, press keys, zoom into a region.
|
|
363
|
-
|
|
364
|
-
Coordinates are viewport pixels of the screenshot you saw (origin top-left,
|
|
365
|
-
viewport ${VP.width}×${VP.height}, 1:1 — no scaling). After zoom, coordinates are STILL
|
|
366
|
-
full-viewport pixels. You can also target an element by ref from browser_find
|
|
367
|
-
(pass ref instead of coordinate) — refs survive layout shifts, pixels do not.
|
|
368
|
-
|
|
369
|
-
${ACTIONS_DOC}
|
|
370
|
-
|
|
371
|
-
Each call runs ONE action and returns a short acknowledgment (plus the new
|
|
372
|
-
location if the page navigated). When you can predict two or more steps — click
|
|
373
|
-
a field, type, press Enter, look — use browser_batch, which runs them in one
|
|
374
|
-
round trip and ends with a screenshot. Consent banners: find the button with
|
|
375
|
-
browser_find and click its ref.`,
|
|
376
|
-
COMPUTER_SCHEMA,
|
|
377
|
-
async (a) => {
|
|
378
|
-
try {
|
|
379
|
-
return await withBrowser(projectId, async ({ page, guard }) => {
|
|
380
|
-
const since = guard.blocked.length;
|
|
381
|
-
const before = page.url();
|
|
382
|
-
// 换页的判据是"主帧发出了导航请求",不是"地址栏变了":慢站(隧道 RTT 几百
|
|
383
|
-
// 毫秒)点完 300ms 内响应还没回来、地址没变,但请求已经发出去了。真跑逮到
|
|
384
|
-
// 两次"page changed 迟到一条命令",都是这个原因。
|
|
385
|
-
let navStarted = false;
|
|
386
|
-
const onReq = (req) => { try { if (req.isNavigationRequest() && req.frame() === page.mainFrame()) navStarted = true; } catch { /* */ } };
|
|
387
|
-
page.on('request', onReq);
|
|
388
|
-
let r;
|
|
389
|
-
try {
|
|
390
|
-
r = await runAction(page, a, { frame: BROWSE_FRAME, shot: (p, lead) => viewportShot(p, projectId, lead) });
|
|
391
|
-
} catch (err) {
|
|
392
|
-
page.off('request', onReq);
|
|
393
|
-
if (/Execution context was destroyed|Target closed|frame was detached/i.test(String(err?.message))) {
|
|
394
|
-
await page.waitForLoadState('domcontentloaded', { timeout: 8000 }).catch(() => {});
|
|
395
|
-
}
|
|
396
|
-
return asText(actionErrorText(a.action, err), true);
|
|
397
|
-
}
|
|
398
|
-
// 点击/按键可能换页:留 300ms 让导航请求发出来,发出来了就等 DOM 就绪再回话
|
|
399
|
-
if (/click|key|type/.test(a.action) && !page.isClosed()) await page.waitForTimeout(300);
|
|
400
|
-
page.off('request', onReq);
|
|
401
|
-
if (!page.isClosed() && (navStarted || page.url() !== before)) {
|
|
402
|
-
await page.waitForLoadState('domcontentloaded', { timeout: 8000 }).catch(() => {});
|
|
403
|
-
}
|
|
404
|
-
if (!page.isClosed() && page.url() !== before) {
|
|
405
|
-
await recordVisit(projectId, page);
|
|
406
|
-
try { ctx?.emit?.({ type: 'run.browser_opened', url: page.url(), ts: new Date().toISOString() }); } catch { /* */ }
|
|
407
|
-
const title = await page.title().catch(() => '');
|
|
408
|
-
r.content.unshift({ type: 'text', text: `→ page changed: ${title || '(no title)'} — ${page.url()}` });
|
|
409
|
-
}
|
|
410
|
-
const fresh = guard.blocked.slice(since);
|
|
411
|
-
if (fresh.length) {
|
|
412
|
-
r.content.push({ type: 'text', text: `⛔ network gate blocked ${fresh.length} request(s) during this action (internal/private addresses are a hard boundary).` });
|
|
413
|
-
}
|
|
414
|
-
return r;
|
|
415
|
-
});
|
|
416
|
-
} catch (err) {
|
|
417
|
-
return asText(`browser_computer 失败:${err.message}`, true);
|
|
418
|
-
}
|
|
419
|
-
},
|
|
420
|
-
);
|
|
421
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* mcp/tools/browse-computer.js — browser_computer:坐标/引用级的指针与键盘(2026-08-21)
|
|
3
|
+
*
|
|
4
|
+
* 形状照 Anthropic 的 browser use toolset(`browser_toolset_20260801`)的指针/键盘/
|
|
5
|
+
* 截图成员:left_click … zoom 那一组,成员名、参数名、语义一字不改;外壳是 MCP 单
|
|
6
|
+
* 工具 + `action` 枚举(Claude in Chrome 的 `computer` 工具就是这么包的)。
|
|
7
|
+
* 规格里的 `target`(coordinate | ref 二选一)拆成 `coordinate` / `ref` 两个字段。
|
|
8
|
+
*
|
|
9
|
+
* ## 为什么不等 toolset 进 Agent SDK
|
|
10
|
+
*
|
|
11
|
+
* toolset 是 Messages API 请求体里的东西,Agent SDK 的设计点是"自带工具 + MCP",
|
|
12
|
+
* 两边不相交(08-21 查过 0.3.237/0.3.238 两版二进制,零痕迹)。而模型认的是
|
|
13
|
+
* **动作名和参数形状**,不是 `toolset_name` 字段 —— Claude in Chrome 用 MCP 包同一
|
|
14
|
+
* 套动作照样熟练。所以这里直接做 MCP,订阅通路和本地模型通路一视同仁。
|
|
15
|
+
*
|
|
16
|
+
* ## 坐标空间 = 截图像素;frame 负责它和页面像素的换算
|
|
17
|
+
*
|
|
18
|
+
* `runAction(page, a, { frame, shot })` 对"页面从哪来"无知:浏览通道传
|
|
19
|
+
* frame={1366,768,scale 1}(视口 1366×768 在 shot-pipeline 阈值内,截图不缩,
|
|
20
|
+
* 1:1);产物会话(artifact_computer)按产物视口算 frame,scale 可能 <1,模型读到
|
|
21
|
+
* 的坐标 ÷ scale 才是页面像素(文档推荐做法)。所有回给模型的坐标都是截图空间。
|
|
22
|
+
*
|
|
23
|
+
* ## 闸不用动
|
|
24
|
+
*
|
|
25
|
+
* 出网闸在进程内 HTTP 代理(lib/browse-proxy.js),在 Playwright 之下;点哪里都
|
|
26
|
+
* 绕不过去。这是 headless 页面,没有地址栏、没有 OS 对话框,不存在桌面 computer
|
|
27
|
+
* use 那种"从浏览器逃到终端"的面。
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import { tool } from '@anthropic-ai/claude-agent-sdk';
|
|
31
|
+
import { z } from 'zod';
|
|
32
|
+
import { withBrowser, _limits } from '../../browse/registry.js';
|
|
33
|
+
import { handleForRef, staleRefText } from '../../browse/refs.js';
|
|
34
|
+
import { recordVisit, saveFrame } from '../../browse/state.js';
|
|
35
|
+
import { normalizeShot } from './helpers/shot-pipeline.js';
|
|
36
|
+
|
|
37
|
+
const VP = _limits.VIEWPORT;
|
|
38
|
+
/** 浏览通道的截图空间:视口像素 1:1(browse-computer.test.js 守着这个前提) */
|
|
39
|
+
export const BROWSE_FRAME = { w: VP.width, h: VP.height, scale: 1 };
|
|
40
|
+
const asText = (text, isError = false) => ({ content: [{ type: 'text', text }], ...(isError ? { isError: true } : {}) });
|
|
41
|
+
|
|
42
|
+
export const ACTIONS = [
|
|
43
|
+
'screenshot', 'zoom',
|
|
44
|
+
'left_click', 'right_click', 'middle_click', 'double_click', 'triple_click', 'hover',
|
|
45
|
+
'left_click_drag', 'left_mouse_down', 'left_mouse_up', 'mouse_move',
|
|
46
|
+
'scroll', 'scroll_to',
|
|
47
|
+
'type', 'key', 'hold_key', 'wait',
|
|
48
|
+
];
|
|
49
|
+
|
|
50
|
+
// ── 键名 / 修饰键:模型写的是 xdotool 风格(Return、ctrl+s、Page_Down),
|
|
51
|
+
// Playwright 要 DOM 键名(Enter、Control+s、PageDown)──
|
|
52
|
+
const KEY_ALIAS = {
|
|
53
|
+
return: 'Enter', enter: 'Enter', esc: 'Escape', escape: 'Escape', tab: 'Tab', space: 'Space',
|
|
54
|
+
backspace: 'Backspace', delete: 'Delete', del: 'Delete', insert: 'Insert',
|
|
55
|
+
up: 'ArrowUp', down: 'ArrowDown', left: 'ArrowLeft', right: 'ArrowRight',
|
|
56
|
+
arrowup: 'ArrowUp', arrowdown: 'ArrowDown', arrowleft: 'ArrowLeft', arrowright: 'ArrowRight',
|
|
57
|
+
home: 'Home', end: 'End', pageup: 'PageUp', page_up: 'PageUp', pagedown: 'PageDown', page_down: 'PageDown',
|
|
58
|
+
capslock: 'CapsLock', caps_lock: 'CapsLock',
|
|
59
|
+
};
|
|
60
|
+
const MOD_ALIAS = {
|
|
61
|
+
shift: 'Shift', ctrl: 'Control', control: 'Control', alt: 'Alt', option: 'Alt',
|
|
62
|
+
cmd: 'Meta', meta: 'Meta', super: 'Meta', win: 'Meta', windows: 'Meta',
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/** 'ctrl+shift' → ['Control','Shift'];不认识的词抛错(别静默吞成无修饰) */
|
|
66
|
+
export function parseModifiers(s) {
|
|
67
|
+
if (!s) return [];
|
|
68
|
+
return String(s).split('+').map(p => p.trim()).filter(Boolean).map((p) => {
|
|
69
|
+
const m = MOD_ALIAS[p.toLowerCase()];
|
|
70
|
+
if (!m) throw new Error(`unknown modifier "${p}" (use shift / ctrl / alt / cmd, joined with +)`);
|
|
71
|
+
return m;
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** 'ctrl+s' / 'Return' / 'Backspace Backspace' → [['Control','s'], ...](每个和弦 = 修饰键…+主键) */
|
|
76
|
+
export function parseChords(text) {
|
|
77
|
+
const chords = String(text || '').trim().split(/\s+/).filter(Boolean);
|
|
78
|
+
if (!chords.length) throw new Error('key needs a key name, e.g. "Enter", "ctrl+a", "Backspace Backspace"');
|
|
79
|
+
return chords.map((chord) => {
|
|
80
|
+
const parts = chord.split('+').filter(Boolean);
|
|
81
|
+
if (chord === '+' || !parts.length) parts.splice(0, parts.length, '+');
|
|
82
|
+
const keyRaw = parts[parts.length - 1];
|
|
83
|
+
const mods = parts.slice(0, -1).map((p) => {
|
|
84
|
+
const m = MOD_ALIAS[p.toLowerCase()];
|
|
85
|
+
if (!m) throw new Error(`unknown modifier "${p}" in "${chord}"`);
|
|
86
|
+
return m;
|
|
87
|
+
});
|
|
88
|
+
const lower = keyRaw.toLowerCase();
|
|
89
|
+
let key = KEY_ALIAS[lower];
|
|
90
|
+
if (!key && MOD_ALIAS[lower]) throw new Error(`"${chord}" is only modifiers — add the key to press (e.g. "${chord}+a"), or use hold_key`);
|
|
91
|
+
if (!key && /^f([1-9]|1[0-2])$/i.test(keyRaw)) key = `F${keyRaw.slice(1)}`;
|
|
92
|
+
if (!key) key = keyRaw.length === 1 ? keyRaw : keyRaw[0].toUpperCase() + keyRaw.slice(1);
|
|
93
|
+
return [...mods, key];
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** 坐标校验(截图空间):必须落在 frame 里 —— 坐标空间就是截图像素,越界只能是读错了图 */
|
|
98
|
+
export function checkCoord(c, frame = BROWSE_FRAME, label = 'coordinate') {
|
|
99
|
+
if (!Array.isArray(c) || c.length !== 2 || !c.every(n => Number.isFinite(n))) {
|
|
100
|
+
return `${label} must be [x, y] in screenshot pixels`;
|
|
101
|
+
}
|
|
102
|
+
const [x, y] = c;
|
|
103
|
+
if (x < 0 || y < 0 || x > frame.w || y > frame.h) {
|
|
104
|
+
return `${label} (${x}, ${y}) is outside the ${frame.w}×${frame.h} screenshot — coordinates are screenshot pixels, origin top-left`;
|
|
105
|
+
}
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const toPage = (v, frame) => v / frame.scale;
|
|
110
|
+
const toShot = (v, frame) => Math.round(v * frame.scale);
|
|
111
|
+
|
|
112
|
+
const cursors = new WeakMap(); // page → {x,y} 页面像素,给省略坐标的 mouse_down/up/scroll 用
|
|
113
|
+
const cursorOf = (page, frame) => cursors.get(page) || { x: toPage(frame.w / 2, frame), y: toPage(frame.h / 2, frame) };
|
|
114
|
+
const moveTo = async (page, x, y) => { await page.mouse.move(x, y); cursors.set(page, { x, y }); };
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* coordinate / ref → {x,y(页面像素), sx,sy(截图像素), viaRef}。ref 会先滚进视口再取几何中心。
|
|
118
|
+
*/
|
|
119
|
+
async function resolveTarget(page, { coordinate, ref }, frame, { required = true } = {}) {
|
|
120
|
+
if (ref) {
|
|
121
|
+
const h = await handleForRef(page, ref);
|
|
122
|
+
if (!h) return { error: staleRefText(ref) };
|
|
123
|
+
await h.scrollIntoViewIfNeeded({ timeout: 3000 }).catch(() => {});
|
|
124
|
+
const box = await h.boundingBox();
|
|
125
|
+
if (!box) return { error: `Error: ${ref} has no visible box on the page (hidden or zero-size).` };
|
|
126
|
+
const x = box.x + box.width / 2;
|
|
127
|
+
const y = box.y + box.height / 2;
|
|
128
|
+
const sx = toShot(x, frame); const sy = toShot(y, frame);
|
|
129
|
+
// 滚了还在视口外(典型:藏在屏幕上方的 skip link)—— 别夹到边上去点个错的地方
|
|
130
|
+
if (sx < 0 || sy < 0 || sx > frame.w || sy > frame.h) {
|
|
131
|
+
return { error: `Error: ${ref} sits off-screen at (${sx}, ${sy}) even after scrolling it into view (probably a visually-hidden element) — pick another match or click a coordinate you can see.` };
|
|
132
|
+
}
|
|
133
|
+
// 真跑逮到的:站点自己的开场幕布盖着整页,点 ref 其实点在幕布上,页面一点反应
|
|
134
|
+
// 没有,agent 只能猜。查一下这个点最上层是谁;不是 ref 本身或它的祖/后代就报出来。
|
|
135
|
+
let coveredBy = null;
|
|
136
|
+
try {
|
|
137
|
+
coveredBy = await h.evaluate((el, [px, py]) => {
|
|
138
|
+
const top = document.elementFromPoint(px, py);
|
|
139
|
+
if (!top || top === el || el.contains(top) || top.contains(el)) return null;
|
|
140
|
+
const cls = typeof top.className === 'string' && top.className.trim() ? `.${top.className.trim().split(/\s+/)[0]}` : '';
|
|
141
|
+
return `${top.tagName.toLowerCase()}${top.id ? `#${top.id}` : ''}${cls}`;
|
|
142
|
+
}, [x, y]);
|
|
143
|
+
} catch { /* 查不到就不报 */ }
|
|
144
|
+
return { x, y, sx, sy, viaRef: ref, coveredBy };
|
|
145
|
+
}
|
|
146
|
+
if (coordinate) {
|
|
147
|
+
const bad = checkCoord(coordinate, frame);
|
|
148
|
+
if (bad) return { error: `Error: ${bad}` };
|
|
149
|
+
return { x: toPage(coordinate[0], frame), y: toPage(coordinate[1], frame), sx: coordinate[0], sy: coordinate[1] };
|
|
150
|
+
}
|
|
151
|
+
if (required) return { error: 'Error: this action needs a coordinate [x, y] or a ref (from the find tool).' };
|
|
152
|
+
const c = cursorOf(page, frame);
|
|
153
|
+
return { ...c, sx: toShot(c.x, frame), sy: toShot(c.y, frame) };
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
async function withModifiers(page, mods, fn) {
|
|
157
|
+
for (const m of mods) await page.keyboard.down(m);
|
|
158
|
+
try { return await fn(); } finally {
|
|
159
|
+
for (const m of [...mods].reverse()) await page.keyboard.up(m).catch(() => {});
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** 浏览通道的视口截图:存桌面卡预览 + 归一化(1366×768 在阈值内,不缩)→ 文本块在前,图在后 */
|
|
164
|
+
export async function viewportShot(page, projectId, lead) {
|
|
165
|
+
const buf = await page.screenshot({ type: 'png' });
|
|
166
|
+
await saveFrame(projectId, buf);
|
|
167
|
+
const shot = await normalizeShot(buf);
|
|
168
|
+
return {
|
|
169
|
+
content: [
|
|
170
|
+
{ type: 'text', text: [lead, `viewport ${VP.width}×${VP.height} — coordinates you use next are these pixels`, shot.note].filter(Boolean).join(' · ') },
|
|
171
|
+
{ type: 'image', data: shot.data, mimeType: shot.mimeType },
|
|
172
|
+
],
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
async function zoomShot(page, region, frame, lead) {
|
|
177
|
+
if (!Array.isArray(region) || region.length !== 4 || !region.every(n => Number.isFinite(n))) {
|
|
178
|
+
return asText('Error: zoom needs region [x0, y0, x1, y1] in screenshot pixels.', true);
|
|
179
|
+
}
|
|
180
|
+
const [x0, y0, x1, y1] = region.map(Math.round);
|
|
181
|
+
if (x0 < 0 || y0 < 0 || x1 > frame.w || y1 > frame.h || x1 - x0 < 8 || y1 - y0 < 8) {
|
|
182
|
+
return asText(`Error: zoom region [${region.join(', ')}] must lie inside the ${frame.w}×${frame.h} screenshot and be at least 8×8.`, true);
|
|
183
|
+
}
|
|
184
|
+
const clip = { x: toPage(x0, frame), y: toPage(y0, frame), width: toPage(x1 - x0, frame), height: toPage(y1 - y0, frame) };
|
|
185
|
+
const buf = await page.screenshot({ type: 'png', clip });
|
|
186
|
+
const { default: sharp } = await import('sharp');
|
|
187
|
+
const k = Math.min(frame.w / (x1 - x0), frame.h / (y1 - y0)); // 放大到塞满截图空间
|
|
188
|
+
const up = await sharp(buf)
|
|
189
|
+
.resize({ width: Math.round((x1 - x0) * k), height: Math.round((y1 - y0) * k), fit: 'inside', withoutEnlargement: false, kernel: 'lanczos3' })
|
|
190
|
+
.png().toBuffer();
|
|
191
|
+
const shot = await normalizeShot(up);
|
|
192
|
+
return {
|
|
193
|
+
content: [
|
|
194
|
+
{ type: 'text', text: `${lead}zoom of [${x0},${y0},${x1},${y1}] (${x1 - x0}×${y1 - y0}, ×${(k / frame.scale).toFixed(1)}) — the coordinates you use next are STILL full-screenshot pixels, not pixels of this zoomed image` },
|
|
195
|
+
{ type: 'image', data: shot.data, mimeType: shot.mimeType },
|
|
196
|
+
],
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* 跑一个动作。返回 CallToolResult。导航变化与闸拦截由调用方补到文本里。
|
|
202
|
+
* @param {import('playwright').Page} page
|
|
203
|
+
* @param {object} a 工具入参
|
|
204
|
+
* @param {{frame:{w:number,h:number,scale:number}, shot:(page:any, lead:string)=>Promise<any>}} env
|
|
205
|
+
*/
|
|
206
|
+
export async function runAction(page, a, { frame, shot }) {
|
|
207
|
+
const { action } = a;
|
|
208
|
+
if (action === 'screenshot') return shot(page, 'screenshot');
|
|
209
|
+
if (action === 'zoom') return zoomShot(page, a.region, frame, '');
|
|
210
|
+
if (action === 'wait') {
|
|
211
|
+
const s = Math.min(30, Math.max(0, Number(a.duration) || 0));
|
|
212
|
+
await page.waitForTimeout(s * 1000);
|
|
213
|
+
return asText(`waited ${s}s`);
|
|
214
|
+
}
|
|
215
|
+
if (action === 'type') {
|
|
216
|
+
if (typeof a.text !== 'string' || !a.text.length) return asText('Error: type needs text.', true);
|
|
217
|
+
await page.keyboard.type(a.text);
|
|
218
|
+
return asText(`typed ${a.text.length} chars at the current focus`);
|
|
219
|
+
}
|
|
220
|
+
if (action === 'key') {
|
|
221
|
+
const chords = parseChords(a.text);
|
|
222
|
+
const n = Math.min(100, Math.max(1, Math.round(Number(a.repeat) || 1)));
|
|
223
|
+
for (let i = 0; i < n; i += 1) for (const c of chords) await page.keyboard.press(c.join('+'));
|
|
224
|
+
return asText(`pressed ${chords.map(c => c.join('+')).join(' ')}${n > 1 ? ` ×${n}` : ''}`);
|
|
225
|
+
}
|
|
226
|
+
if (action === 'hold_key') {
|
|
227
|
+
const [chord] = parseChords(a.text);
|
|
228
|
+
const s = Math.min(30, Math.max(0, Number(a.duration) || 0));
|
|
229
|
+
for (const k of chord) await page.keyboard.down(k);
|
|
230
|
+
try { await page.waitForTimeout(s * 1000); } finally { for (const k of [...chord].reverse()) await page.keyboard.up(k).catch(() => {}); }
|
|
231
|
+
return asText(`held ${chord.join('+')} for ${s}s`);
|
|
232
|
+
}
|
|
233
|
+
if (action === 'scroll_to') {
|
|
234
|
+
if (!a.ref) return asText('Error: scroll_to needs a ref (from the find tool).', true);
|
|
235
|
+
const h = await handleForRef(page, a.ref);
|
|
236
|
+
if (!h) return asText(staleRefText(a.ref), true);
|
|
237
|
+
await h.evaluate(el => el.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' }));
|
|
238
|
+
await page.waitForTimeout(200);
|
|
239
|
+
const box = await h.boundingBox();
|
|
240
|
+
return asText(box ? `scrolled ${a.ref} into view — now at (${toShot(box.x + box.width / 2, frame)}, ${toShot(box.y + box.height / 2, frame)})` : `scrolled toward ${a.ref} (it has no visible box)`);
|
|
241
|
+
}
|
|
242
|
+
if (action === 'scroll') {
|
|
243
|
+
const t = await resolveTarget(page, a, frame, { required: false });
|
|
244
|
+
if (t.error) return asText(t.error, true);
|
|
245
|
+
const dir = a.scroll_direction;
|
|
246
|
+
if (!['up', 'down', 'left', 'right'].includes(dir)) return asText('Error: scroll needs scroll_direction up|down|left|right.', true);
|
|
247
|
+
const amt = Math.min(10, Math.max(1, Math.round(Number(a.scroll_amount) || 3)));
|
|
248
|
+
const mods = parseModifiers(a.modifiers);
|
|
249
|
+
const before = await page.evaluate(() => [window.scrollX, window.scrollY]);
|
|
250
|
+
await moveTo(page, t.x, t.y);
|
|
251
|
+
const d = 100 * amt;
|
|
252
|
+
await withModifiers(page, mods, () => page.mouse.wheel(
|
|
253
|
+
dir === 'left' ? -d : dir === 'right' ? d : 0,
|
|
254
|
+
dir === 'up' ? -d : dir === 'down' ? d : 0,
|
|
255
|
+
));
|
|
256
|
+
await page.waitForTimeout(250);
|
|
257
|
+
const after = await page.evaluate(() => [window.scrollX, window.scrollY]);
|
|
258
|
+
const moved = before[0] !== after[0] || before[1] !== after[1];
|
|
259
|
+
return asText(`scrolled ${dir} ${amt} notches at (${t.sx}, ${t.sy})`
|
|
260
|
+
+ (moved ? ` — scrollY ${before[1]} → ${after[1]}${before[0] !== after[0] ? `, scrollX ${before[0]} → ${after[0]}` : ''}`
|
|
261
|
+
: ' — the window itself did not move (maybe a scrollable panel under the cursor took it, or the page is at its end; try the keyboard: key PageDown)'));
|
|
262
|
+
}
|
|
263
|
+
// ── 指针动作 ──
|
|
264
|
+
const mods = parseModifiers(a.modifiers);
|
|
265
|
+
if (action === 'left_click_drag') {
|
|
266
|
+
const from = await resolveTarget(page, { coordinate: a.start_coordinate }, frame, { required: true });
|
|
267
|
+
if (from.error) return asText(from.error.replace('this action needs a coordinate', 'left_click_drag needs start_coordinate'), true);
|
|
268
|
+
const to = await resolveTarget(page, a, frame);
|
|
269
|
+
if (to.error) return asText(to.error, true);
|
|
270
|
+
await withModifiers(page, mods, async () => {
|
|
271
|
+
await moveTo(page, from.x, from.y);
|
|
272
|
+
await page.mouse.down();
|
|
273
|
+
await page.mouse.move(to.x, to.y, { steps: 12 });
|
|
274
|
+
await page.mouse.up();
|
|
275
|
+
});
|
|
276
|
+
cursors.set(page, { x: to.x, y: to.y });
|
|
277
|
+
return asText(`dragged (${from.sx}, ${from.sy}) → (${to.sx}, ${to.sy})`);
|
|
278
|
+
}
|
|
279
|
+
if (action === 'left_mouse_down' || action === 'left_mouse_up') {
|
|
280
|
+
const t = await resolveTarget(page, a, frame, { required: false });
|
|
281
|
+
if (t.error) return asText(t.error, true);
|
|
282
|
+
await moveTo(page, t.x, t.y);
|
|
283
|
+
if (action === 'left_mouse_down') await page.mouse.down(); else await page.mouse.up();
|
|
284
|
+
return asText(`${action} at (${t.sx}, ${t.sy})`);
|
|
285
|
+
}
|
|
286
|
+
const t = await resolveTarget(page, a, frame);
|
|
287
|
+
if (t.error) return asText(t.error, true);
|
|
288
|
+
const at = `(${t.sx}, ${t.sy})${t.viaRef ? ` = ${t.viaRef}` : ''}`
|
|
289
|
+
+ (t.coveredBy ? ` — ⚠ that point is covered by <${t.coveredBy}>: the ${action} landed on it, not on ${t.viaRef} (overlay / curtain / modal? dismiss it first or screenshot to see)` : '');
|
|
290
|
+
if (action === 'mouse_move' || action === 'hover') {
|
|
291
|
+
await moveTo(page, t.x, t.y);
|
|
292
|
+
await page.waitForTimeout(150);
|
|
293
|
+
return asText(`${action} ${at}`);
|
|
294
|
+
}
|
|
295
|
+
const CLICKS = {
|
|
296
|
+
left_click: { button: 'left', clickCount: 1 },
|
|
297
|
+
right_click: { button: 'right', clickCount: 1 },
|
|
298
|
+
middle_click: { button: 'middle', clickCount: 1 },
|
|
299
|
+
double_click: { button: 'left', clickCount: 2 },
|
|
300
|
+
triple_click: { button: 'left', clickCount: 3 },
|
|
301
|
+
};
|
|
302
|
+
const spec = CLICKS[action];
|
|
303
|
+
if (!spec) return asText(`Error: unknown action "${action}".`, true);
|
|
304
|
+
await withModifiers(page, mods, async () => {
|
|
305
|
+
await moveTo(page, t.x, t.y);
|
|
306
|
+
await page.mouse.click(t.x, t.y, spec);
|
|
307
|
+
});
|
|
308
|
+
return asText(`${action} ${at}${mods.length ? ` with ${mods.join('+')}` : ''}`);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/** 动作描述正文,browser_computer / artifact_computer 共用(坐标空间那句由各自补) */
|
|
312
|
+
export const ACTIONS_DOC = `Actions (params in parentheses):
|
|
313
|
+
screenshot () current view as an image
|
|
314
|
+
zoom (region [x0,y0,x1,y1]) region at higher magnification, for small text/icons
|
|
315
|
+
left_click | right_click | middle_click | double_click | triple_click (coordinate or ref, modifiers?)
|
|
316
|
+
hover | mouse_move (coordinate or ref)
|
|
317
|
+
left_click_drag (start_coordinate, coordinate)
|
|
318
|
+
left_mouse_down | left_mouse_up (coordinate?) custom drags; pair them, move in between
|
|
319
|
+
scroll (scroll_direction, scroll_amount? 1-10 notches, coordinate?)
|
|
320
|
+
scroll_to (ref) scroll an element into view
|
|
321
|
+
type (text) literal text at the current focus
|
|
322
|
+
key (text, repeat? 1-100) "Enter", "ctrl+a", "Backspace Backspace", "alt+Tab"
|
|
323
|
+
hold_key (text, duration ≤30s)
|
|
324
|
+
wait (duration ≤30s)`;
|
|
325
|
+
|
|
326
|
+
/** 入参 schema,两个 computer 工具共用 */
|
|
327
|
+
export const COMPUTER_SCHEMA = {
|
|
328
|
+
action: z.enum(ACTIONS).describe('Which action to perform (see the list above).'),
|
|
329
|
+
coordinate: z.array(z.number()).length(2).optional()
|
|
330
|
+
.describe('[x, y] screenshot pixels. Target for clicks/hover/mouse_move/scroll, and the END point of left_click_drag.'),
|
|
331
|
+
start_coordinate: z.array(z.number()).length(2).optional()
|
|
332
|
+
.describe('[x, y] START point for left_click_drag.'),
|
|
333
|
+
ref: z.string().optional()
|
|
334
|
+
.describe('Element reference from the find tool (e.g. "ref_3"). Alternative to coordinate for clicks/hover; required for scroll_to.'),
|
|
335
|
+
region: z.array(z.number()).length(4).optional()
|
|
336
|
+
.describe('[x0, y0, x1, y1] for zoom: top-left and bottom-right corners in screenshot pixels.'),
|
|
337
|
+
text: z.string().optional()
|
|
338
|
+
.describe('For type: the literal text. For key/hold_key: a key or +-chord, space-separated for a sequence ("Enter", "ctrl+s", "shift+Tab Tab").'),
|
|
339
|
+
modifiers: z.string().optional()
|
|
340
|
+
.describe('Modifier chord held during a click or scroll: shift / ctrl / alt / cmd, joined with + (e.g. "ctrl+shift").'),
|
|
341
|
+
scroll_direction: z.enum(['up', 'down', 'left', 'right']).optional().describe('For scroll.'),
|
|
342
|
+
scroll_amount: z.number().int().min(1).max(10).optional().describe('For scroll: wheel notches, default 3.'),
|
|
343
|
+
repeat: z.number().int().min(1).max(100).optional().describe('For key: press the sequence this many times, default 1.'),
|
|
344
|
+
duration: z.number().min(0).max(30).optional().describe('Seconds, for wait and hold_key (max 30).'),
|
|
345
|
+
};
|
|
346
|
+
|
|
347
|
+
/** 动作抛的 Playwright 内部错 → agent 能行动的话(两个 computer 工具共用) */
|
|
348
|
+
export function actionErrorText(action, err) {
|
|
349
|
+
const msg = String(err?.message || err).split('\n')[0];
|
|
350
|
+
if (/Execution context was destroyed|Target closed|frame was detached/i.test(msg)) {
|
|
351
|
+
return `Error: ${action} hit a page navigation in progress (${msg}). The page has changed underneath you — refs from before are stale; find again or screenshot first.`;
|
|
352
|
+
}
|
|
353
|
+
return `Error: ${action} failed: ${msg}`;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
export function makeBrowserComputerTool({ projectId, ctx }) {
|
|
357
|
+
return tool(
|
|
358
|
+
'browser_computer',
|
|
359
|
+
`Pointer, keyboard and pixel-level capture on the current page of this project's
|
|
360
|
+
browser — the coordinate half of browsing. browser_click/browser_read work by
|
|
361
|
+
selector and text; this one works the way a person does: look at a screenshot,
|
|
362
|
+
click at a pixel, type, press keys, zoom into a region.
|
|
363
|
+
|
|
364
|
+
Coordinates are viewport pixels of the screenshot you saw (origin top-left,
|
|
365
|
+
viewport ${VP.width}×${VP.height}, 1:1 — no scaling). After zoom, coordinates are STILL
|
|
366
|
+
full-viewport pixels. You can also target an element by ref from browser_find
|
|
367
|
+
(pass ref instead of coordinate) — refs survive layout shifts, pixels do not.
|
|
368
|
+
|
|
369
|
+
${ACTIONS_DOC}
|
|
370
|
+
|
|
371
|
+
Each call runs ONE action and returns a short acknowledgment (plus the new
|
|
372
|
+
location if the page navigated). When you can predict two or more steps — click
|
|
373
|
+
a field, type, press Enter, look — use browser_batch, which runs them in one
|
|
374
|
+
round trip and ends with a screenshot. Consent banners: find the button with
|
|
375
|
+
browser_find and click its ref.`,
|
|
376
|
+
COMPUTER_SCHEMA,
|
|
377
|
+
async (a) => {
|
|
378
|
+
try {
|
|
379
|
+
return await withBrowser(projectId, async ({ page, guard }) => {
|
|
380
|
+
const since = guard.blocked.length;
|
|
381
|
+
const before = page.url();
|
|
382
|
+
// 换页的判据是"主帧发出了导航请求",不是"地址栏变了":慢站(隧道 RTT 几百
|
|
383
|
+
// 毫秒)点完 300ms 内响应还没回来、地址没变,但请求已经发出去了。真跑逮到
|
|
384
|
+
// 两次"page changed 迟到一条命令",都是这个原因。
|
|
385
|
+
let navStarted = false;
|
|
386
|
+
const onReq = (req) => { try { if (req.isNavigationRequest() && req.frame() === page.mainFrame()) navStarted = true; } catch { /* */ } };
|
|
387
|
+
page.on('request', onReq);
|
|
388
|
+
let r;
|
|
389
|
+
try {
|
|
390
|
+
r = await runAction(page, a, { frame: BROWSE_FRAME, shot: (p, lead) => viewportShot(p, projectId, lead) });
|
|
391
|
+
} catch (err) {
|
|
392
|
+
page.off('request', onReq);
|
|
393
|
+
if (/Execution context was destroyed|Target closed|frame was detached/i.test(String(err?.message))) {
|
|
394
|
+
await page.waitForLoadState('domcontentloaded', { timeout: 8000 }).catch(() => {});
|
|
395
|
+
}
|
|
396
|
+
return asText(actionErrorText(a.action, err), true);
|
|
397
|
+
}
|
|
398
|
+
// 点击/按键可能换页:留 300ms 让导航请求发出来,发出来了就等 DOM 就绪再回话
|
|
399
|
+
if (/click|key|type/.test(a.action) && !page.isClosed()) await page.waitForTimeout(300);
|
|
400
|
+
page.off('request', onReq);
|
|
401
|
+
if (!page.isClosed() && (navStarted || page.url() !== before)) {
|
|
402
|
+
await page.waitForLoadState('domcontentloaded', { timeout: 8000 }).catch(() => {});
|
|
403
|
+
}
|
|
404
|
+
if (!page.isClosed() && page.url() !== before) {
|
|
405
|
+
await recordVisit(projectId, page);
|
|
406
|
+
try { ctx?.emit?.({ type: 'run.browser_opened', url: page.url(), ts: new Date().toISOString() }); } catch { /* */ }
|
|
407
|
+
const title = await page.title().catch(() => '');
|
|
408
|
+
r.content.unshift({ type: 'text', text: `→ page changed: ${title || '(no title)'} — ${page.url()}` });
|
|
409
|
+
}
|
|
410
|
+
const fresh = guard.blocked.slice(since);
|
|
411
|
+
if (fresh.length) {
|
|
412
|
+
r.content.push({ type: 'text', text: `⛔ network gate blocked ${fresh.length} request(s) during this action (internal/private addresses are a hard boundary).` });
|
|
413
|
+
}
|
|
414
|
+
return r;
|
|
415
|
+
});
|
|
416
|
+
} catch (err) {
|
|
417
|
+
return asText(`browser_computer 失败:${err.message}`, true);
|
|
418
|
+
}
|
|
419
|
+
},
|
|
420
|
+
);
|
|
421
|
+
}
|