dsh-browser-plus 0.0.0-stage → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +166 -0
- package/LICENSE +22 -0
- package/NOTICE.md +7 -0
- package/README.en.md +100 -0
- package/README.md +99 -2
- package/assets/dsh-browser-plus-256.png +0 -0
- package/assets/dsh-browser-plus-512.png +0 -0
- package/assets/dsh-browser-plus-small.svg +9 -0
- package/assets/dsh-browser-plus.ico +0 -0
- package/assets/dsh-browser-plus.svg +11 -0
- package/assets/readme-workspace.png +0 -0
- package/cordis.patch.yml +17 -0
- package/docs/MIGRATION.md +48 -0
- package/docs/README.md +22 -0
- package/docs/SOAK-CHECKLIST.md +98 -0
- package/docs/architecture.md +88 -0
- package/docs/tool-reference.md +124 -0
- package/docs/user-guide.md +121 -0
- package/docs/why-browser.md +45 -0
- package/lib/browser/runtime.d.ts +225 -0
- package/lib/browser/runtime.js +302 -0
- package/lib/browser/types.d.ts +668 -0
- package/lib/browser/types.js +18 -0
- package/lib/browser-electron/auth-cookies.d.ts +54 -0
- package/lib/browser-electron/auth-cookies.js +83 -0
- package/lib/browser-electron/chrome-state.d.ts +187 -0
- package/lib/browser-electron/chrome-state.js +12 -0
- package/lib/browser-electron/entry.d.ts +66 -0
- package/lib/browser-electron/entry.js +62 -0
- package/lib/browser-electron/fingerprint.d.ts +29 -0
- package/lib/browser-electron/fingerprint.js +42 -0
- package/lib/browser-electron/host-main.d.ts +18 -0
- package/lib/browser-electron/host-main.js +2494 -0
- package/lib/browser-electron/icon.d.ts +11 -0
- package/lib/browser-electron/icon.js +23 -0
- package/lib/browser-electron/page-chrome.d.ts +21 -0
- package/lib/browser-electron/page-chrome.js +2034 -0
- package/lib/browser-electron/provider.d.ts +709 -0
- package/lib/browser-electron/provider.js +2575 -0
- package/lib/browser-electron/remote-host.d.ts +143 -0
- package/lib/browser-electron/remote-host.js +952 -0
- package/lib/browser-electron/task-summary.d.ts +2 -0
- package/lib/browser-electron/task-summary.js +12 -0
- package/lib/browser-electron/task-thumbnail.d.ts +11 -0
- package/lib/browser-electron/task-thumbnail.js +9 -0
- package/lib/browser-electron/write-guard.d.ts +41 -0
- package/lib/browser-electron/write-guard.js +123 -0
- package/lib/index.d.ts +16 -0
- package/lib/index.js +14 -0
- package/lib/tool-browser/index.d.ts +31 -0
- package/lib/tool-browser/index.js +1931 -0
- package/package.json +95 -4
- package/screenshots.json +3 -0
- package/scripts/build-icons.mjs +80 -0
- package/scripts/capture-window.ps1 +79 -0
- package/scripts/crop-image.ps1 +20 -0
- package/scripts/smoke-browser-tools.mjs +1968 -0
- package/scripts/smoke-chrome-world.mjs +63 -0
- package/scripts/smoke-electron-host.mjs +50 -0
- package/src/browser/runtime.ts +470 -0
- package/src/browser/types.ts +649 -0
- package/src/browser-electron/auth-cookies.ts +125 -0
- package/src/browser-electron/chrome-state.ts +174 -0
- package/src/browser-electron/entry.ts +115 -0
- package/src/browser-electron/fingerprint.ts +45 -0
- package/src/browser-electron/host-main.ts +2330 -0
- package/src/browser-electron/icon.ts +26 -0
- package/src/browser-electron/page-chrome.ts +2046 -0
- package/src/browser-electron/provider.ts +3088 -0
- package/src/browser-electron/remote-host.ts +1004 -0
- package/src/browser-electron/task-summary.ts +10 -0
- package/src/browser-electron/task-thumbnail.ts +17 -0
- package/src/browser-electron/write-guard.ts +134 -0
- package/src/index.ts +52 -0
- package/src/tool-browser/index.ts +1974 -0
- package/src/types/electron-shim.d.ts +143 -0
|
@@ -0,0 +1,3088 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Electron-backed browser provider: `WebContentsView` sessions driven over
|
|
3
|
+
* `webContents.debugger` (CDP). The provider itself does not import Electron — it operates through the {@link ElectronBrowserViewHost} seam, which the
|
|
4
|
+
* desktop shell implements with real Electron objects. That keeps this
|
|
5
|
+
* package testable under plain Node and leaves the Electron dependency to the
|
|
6
|
+
* shell that owns the `BrowserWindow`.
|
|
7
|
+
* @module dsh-browser-plus/browser-electron
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { randomUUID } from 'node:crypto'
|
|
11
|
+
import { appendFileSync, readFileSync, writeFileSync } from 'node:fs'
|
|
12
|
+
import type {
|
|
13
|
+
BrowserChallenge,
|
|
14
|
+
BrowserClearAuthRequest,
|
|
15
|
+
BrowserClearAuthResult,
|
|
16
|
+
BrowserContentRequest,
|
|
17
|
+
BrowserContentResult,
|
|
18
|
+
BrowserControlOwner,
|
|
19
|
+
BrowserDragRequest,
|
|
20
|
+
BrowserDragResult,
|
|
21
|
+
BrowserPointerResult,
|
|
22
|
+
BrowserPointerTarget,
|
|
23
|
+
BrowserExecuteRequest,
|
|
24
|
+
BrowserExecuteResult,
|
|
25
|
+
BrowserFillRequest,
|
|
26
|
+
BrowserFillResult,
|
|
27
|
+
BrowserHandoffState,
|
|
28
|
+
BrowserHistoryEntry,
|
|
29
|
+
|
|
30
|
+
BrowserOpenOptions,
|
|
31
|
+
BrowserOpenRequest,
|
|
32
|
+
BrowserPressKeyRequest,
|
|
33
|
+
BrowserProvider,
|
|
34
|
+
BrowserRefRequest,
|
|
35
|
+
BrowserScrapeRequest,
|
|
36
|
+
BrowserScrapeStatus,
|
|
37
|
+
BrowserScrollIntoViewRequest,
|
|
38
|
+
BrowserScrollRequest,
|
|
39
|
+
BrowserScrollResult,
|
|
40
|
+
BrowserSessionId,
|
|
41
|
+
BrowserSnapshotElement,
|
|
42
|
+
BrowserSnapshotResult,
|
|
43
|
+
BrowserSpaceInfo,
|
|
44
|
+
BrowserTab,
|
|
45
|
+
BrowserTaskInfo,
|
|
46
|
+
BrowserTaskStatus,
|
|
47
|
+
BrowserTaskUpdate,
|
|
48
|
+
BrowserUploadFileRequest,
|
|
49
|
+
BrowserUploadFileResult,
|
|
50
|
+
BrowserWaitForRequest,
|
|
51
|
+
BrowserWaitForResult,
|
|
52
|
+
ExportedCookie,
|
|
53
|
+
} from '../browser/types.ts'
|
|
54
|
+
import { BrowserError } from '../browser/types.ts'
|
|
55
|
+
import { PAGE_CHROME_HOST_ID, PAGE_CHROME_SCRIPT } from './page-chrome.ts'
|
|
56
|
+
import { defaultWriteRoots, resolveReadPath, resolveWritePath } from './write-guard.ts'
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Page-context human-verification (CAPTCHA / bot-detection) detection. Runs
|
|
60
|
+
* inside the page; returns `{ blocked, kind?, reason? }`. Marker-based and
|
|
61
|
+
* best-effort: checks for Cloudflare's interstitial, hCaptcha, reCAPTCHA,
|
|
62
|
+
* Turnstile, and generic challenge wording.
|
|
63
|
+
*/
|
|
64
|
+
const CHALLENGE_DETECT_EXPRESSION = `(() => {
|
|
65
|
+
const title = (document.title || '').trim()
|
|
66
|
+
const bodyText = (document.body && document.body.innerText || '').slice(0, 4000)
|
|
67
|
+
const lower = (title + '\\n' + bodyText).toLowerCase()
|
|
68
|
+
const frameSrcs = [...document.querySelectorAll('iframe')].map(f => f.src || '').join(' ')
|
|
69
|
+
const framesLower = frameSrcs.toLowerCase()
|
|
70
|
+
const hasCfInterstitial = /just a moment|checking your browser|attention required|cf_chl/i.test(lower)
|
|
71
|
+
|| !!document.querySelector('#challenge-running, #challenge-stage, #cf-chl-container')
|
|
72
|
+
const hasHCaptcha = !!window.hcaptcha || !!document.querySelector('.h-captcha') || /hcaptcha\\.com/i.test(framesLower)
|
|
73
|
+
const hasRecaptcha = !!window.grecaptcha || !!document.querySelector('.g-recaptcha') || /recaptcha\\/api|google\\.com\\/recaptcha/i.test(framesLower)
|
|
74
|
+
const hasTurnstile = !!window.turnstile || /challenges\\.cloudflare\\.com/i.test(framesLower) || /turnstile|challenge-platform/i.test(lower)
|
|
75
|
+
const verifyWording = /verify you are human|verify you are not a robot|\\u4eba\\u673a\\u9a8c\\u8bc1|\\u5b89\\u5168\\u9a8c\\u8bc1|enable javascript and cookies|\\u8bf7.*\\u9a8c\\u8bc1/i.test(lower)
|
|
76
|
+
if (hasCfInterstitial) return { blocked: true, kind: 'cloudflare', reason: 'Cloudflare "Just a moment" interstitial' }
|
|
77
|
+
if (hasHCaptcha) return { blocked: true, kind: 'hcaptcha', reason: 'hCaptcha verification' }
|
|
78
|
+
if (hasRecaptcha) return { blocked: true, kind: 'recaptcha', reason: 'Google reCAPTCHA verification' }
|
|
79
|
+
if (hasTurnstile) return { blocked: true, kind: 'turnstile', reason: 'Cloudflare Turnstile verification' }
|
|
80
|
+
if (verifyWording && /challenge|captcha|verification|security check|access denied|blocked|\\u9a8c\\u8bc1/i.test(lower)) {
|
|
81
|
+
return { blocked: true, kind: 'generic', reason: 'Human-verification challenge' }
|
|
82
|
+
}
|
|
83
|
+
return { blocked: false }
|
|
84
|
+
})()`
|
|
85
|
+
|
|
86
|
+
/** Short suppression window so CDP input is not misclassified as physical user input. */
|
|
87
|
+
const AGENT_INPUT_SUPPRESSION_MS = 900
|
|
88
|
+
|
|
89
|
+
/** Stable provider id registered with `ctx.browser`. */
|
|
90
|
+
export const ELECTRON_BROWSER_PROVIDER_ID = 'electron'
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* One tab request raised by the injected chrome because a human used it.
|
|
94
|
+
*
|
|
95
|
+
* The host owns the pixels, the tab strip and the per-view secret, so it is the
|
|
96
|
+
* only party that can authenticate such a request; the provider owns the tab
|
|
97
|
+
* model, so it is the only party that may act on one. `tabId` is the host's own
|
|
98
|
+
* view id, which is also the provider's `ElectronViewHandle.id`, so no extra
|
|
99
|
+
* mapping table is needed.
|
|
100
|
+
*/
|
|
101
|
+
export interface ChromeHostEvent {
|
|
102
|
+
/** For a 'move-tab' event: the index the tab was dropped at, after removal. */
|
|
103
|
+
readonly toIndex?: number
|
|
104
|
+
readonly type: 'new-tab' | 'close-tab' | 'activate-tab' | 'move-tab'
|
|
105
|
+
/** Task key of the chrome that raised it. */
|
|
106
|
+
readonly taskKey: string
|
|
107
|
+
/** Host view id of the tab; absent for `new-tab`. */
|
|
108
|
+
readonly tabId?: string
|
|
109
|
+
/** For `new-tab`: open this http(s) url in the new tab (a bookmark click). */
|
|
110
|
+
readonly url?: string
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* The minimal Electron surface this provider needs. Implemented by the
|
|
115
|
+
* desktop shell with a real `WebContentsView`; a fake implements it in tests.
|
|
116
|
+
*/
|
|
117
|
+
export interface ElectronBrowserViewHost {
|
|
118
|
+
/**
|
|
119
|
+
* Subscribe to tab requests the host raises without being asked (a human
|
|
120
|
+
* clicking the chrome's own `+`, `×` or tab strip). Optional: a host whose
|
|
121
|
+
* chrome cannot speak first simply never raises one.
|
|
122
|
+
* @param listener - invoked once per authenticated chrome request.
|
|
123
|
+
*/
|
|
124
|
+
onChromeEvent?(listener: (event: ChromeHostEvent) => void): void
|
|
125
|
+
/**
|
|
126
|
+
* Create a new browser view and return a handle to its webContents-like
|
|
127
|
+
* surface. `key` (default 'default') identifies an isolated browser task in
|
|
128
|
+
* the shared BrowserWindow; `label` names that task. The host owns view
|
|
129
|
+
* attachment, sizing, task visibility, and removal; the provider owns
|
|
130
|
+
* CDP-driven behavior.
|
|
131
|
+
*/
|
|
132
|
+
createView(key?: string, label?: string): ElectronViewHandle
|
|
133
|
+
/**
|
|
134
|
+
* Destroy a view created by this host. Called on session close; idempotent
|
|
135
|
+
* for an already-destroyed view.
|
|
136
|
+
* @param handle - the handle returned by {@link createView}.
|
|
137
|
+
*/
|
|
138
|
+
destroyView(handle: ElectronViewHandle): void
|
|
139
|
+
/**
|
|
140
|
+
* Notify the host that this session selected a tab. In the shared-window
|
|
141
|
+
* host, a background task updates its active view without changing the
|
|
142
|
+
* human-selected visible task. Optional for headless/probe hosts.
|
|
143
|
+
* @param handle - the handle selected by its session.
|
|
144
|
+
*/
|
|
145
|
+
showView?(handle: ElectronViewHandle): void
|
|
146
|
+
/**
|
|
147
|
+
* Send a CDP `Input.*` command to the host's chrome frame view.
|
|
148
|
+
*
|
|
149
|
+
* The frame is not a tab and has no handle, so this is its own channel. It
|
|
150
|
+
* exists because the chrome can live in a view of its own (which is what lets
|
|
151
|
+
* the page viewport really shrink): input aimed at a page never reaches that
|
|
152
|
+
* view, and CDP input targets a webContents regardless of view stacking, so
|
|
153
|
+
* the page cannot stand in for it. Optional for hosts without a frame view.
|
|
154
|
+
* @param method - a CDP Input domain command, e.g. 'Input.dispatchMouseEvent'.
|
|
155
|
+
* @param params - that command's parameters.
|
|
156
|
+
*/
|
|
157
|
+
chromeInput?(method: string, params?: Record<string, unknown>): Promise<void>
|
|
158
|
+
/**
|
|
159
|
+
* Evaluate an expression inside the chrome frame's own document.
|
|
160
|
+
*
|
|
161
|
+
* The frame is a view of its own, so no page-directed call can read it. This is
|
|
162
|
+
* how the tests assert the toolbar's own animations instead of inferring them
|
|
163
|
+
* from the page's copy of the chrome. Optional for hosts without a frame view.
|
|
164
|
+
* @param expression - JavaScript evaluated in the frame, by value.
|
|
165
|
+
*/
|
|
166
|
+
chromeEval?(expression: string): Promise<unknown>
|
|
167
|
+
/**
|
|
168
|
+
* Append one operation to the human-facing trail for a view. Optional.
|
|
169
|
+
* @param viewId - the view to attribute the operation to.
|
|
170
|
+
* @param entry - the trail entry ({ action, params, ok, at }).
|
|
171
|
+
*/
|
|
172
|
+
trace?(viewId: string, entry: unknown): void
|
|
173
|
+
/**
|
|
174
|
+
* Cheap local usability probe for this host. MUST NOT make network calls and
|
|
175
|
+
* MUST NOT throw (a throw is reported as unavailable). Optional: a host that
|
|
176
|
+
* omits it is assumed usable, which keeps a desktop shell's shell-owned
|
|
177
|
+
* viewHost and test fakes working. A self-hosted host reports false when the
|
|
178
|
+
* pinned Electron binary cannot be resolved, so the seam can pick another
|
|
179
|
+
* provider (BROWSER_PROVIDER_UNAVAILABLE / BROWSER_PROVIDER_AMBIGUOUS)
|
|
180
|
+
* instead of failing later on the first open().
|
|
181
|
+
*/
|
|
182
|
+
isAvailable?(): boolean
|
|
183
|
+
/** List browser tasks with their labels. Legacy method name retained for compatibility. */
|
|
184
|
+
listWindows?(): Promise<Array<{ key: string; label: string }>>
|
|
185
|
+
/** List task summaries when the host exposes a visible workspace. */
|
|
186
|
+
listTasks?(): Promise<readonly BrowserTaskInfo[]>
|
|
187
|
+
/** Read one task summary from the visible workspace. */
|
|
188
|
+
getTask?(key: string): Promise<BrowserTaskInfo | undefined>
|
|
189
|
+
/** Apply a task status/control update to the visible workspace. */
|
|
190
|
+
updateTask?(key: string, update: BrowserTaskUpdate): Promise<BrowserTaskInfo | undefined>
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* A CDP-capable view handle. This is the subset of Electron's
|
|
195
|
+
* `WebContents`/`WebContentsView` the provider drives; the shell's real
|
|
196
|
+
* implementation adapts `webContents.debugger` to it.
|
|
197
|
+
*/
|
|
198
|
+
export interface ElectronViewHandle {
|
|
199
|
+
/** Unique id of the backing view, used for diagnostics. */
|
|
200
|
+
readonly id: string
|
|
201
|
+
/**
|
|
202
|
+
* Send one CDP command and resolve with its result. Rejects when the
|
|
203
|
+
* debugger is not attached or the command fails.
|
|
204
|
+
* @param method - CDP method, e.g. `Page.navigate`.
|
|
205
|
+
* @param params - CDP command parameters.
|
|
206
|
+
* @returns the CDP `result` object.
|
|
207
|
+
*/
|
|
208
|
+
sendCommand(method: string, params?: Record<string, unknown>): Promise<Record<string, unknown>>
|
|
209
|
+
/**
|
|
210
|
+
* Read the most recent auto-accepted JS dialog for this view (and clear it).
|
|
211
|
+
* Optional: hosts without JS-dialog supervision omit it.
|
|
212
|
+
* @returns the dialog detail ({ type, message, prompt? }) or null.
|
|
213
|
+
*/
|
|
214
|
+
clearDialog?(): Promise<unknown>
|
|
215
|
+
/** Optional: hosts without JS-dialog supervision omit it. */
|
|
216
|
+
setDialogPolicy?(policy: DialogPolicy): Promise<unknown>
|
|
217
|
+
/** Optional: bounded console capture. */
|
|
218
|
+
readConsole?(clear?: boolean): Promise<unknown>
|
|
219
|
+
/** Optional: bounded network capture. */
|
|
220
|
+
readNetwork?(clear?: boolean): Promise<unknown>
|
|
221
|
+
/**
|
|
222
|
+
* Remove cookies matching a domain/name filter. Optional: hosts without a
|
|
223
|
+
* deletable cookie store omit it.
|
|
224
|
+
*/
|
|
225
|
+
clearCookies?(filter: { readonly domain?: string; readonly name?: string; readonly all?: boolean }): Promise<{ readonly removed: number; readonly names: readonly string[] }>
|
|
226
|
+
/** Set this view's browser task label; it titles the shared window only when selected. Optional. */
|
|
227
|
+
label?(label: string): Promise<void>
|
|
228
|
+
/**
|
|
229
|
+
* Re-apply the host's own page chrome to the current document. Optional: a host
|
|
230
|
+
* that does not own the chrome omits it, and the provider then injects its own
|
|
231
|
+
* tokenless copy as a fallback.
|
|
232
|
+
*/
|
|
233
|
+
reinstallChrome?(): Promise<void>
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/** Internal selector and fingerprint captured for one snapshot element. */
|
|
237
|
+
interface SnapshotTarget {
|
|
238
|
+
readonly path: string
|
|
239
|
+
readonly fingerprint: string
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** One snapshot retained for exact reference operations. */
|
|
243
|
+
interface SnapshotRecord {
|
|
244
|
+
readonly tabId: string
|
|
245
|
+
readonly url: string
|
|
246
|
+
readonly epoch: number
|
|
247
|
+
readonly targets: ReadonlyMap<number, SnapshotTarget>
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** One tab inside a session: its view plus a stable id and short-lived refs. */
|
|
251
|
+
interface Tab {
|
|
252
|
+
readonly id: string
|
|
253
|
+
readonly handle: ElectronViewHandle
|
|
254
|
+
navigationEpoch: number
|
|
255
|
+
readonly snapshots: Map<string, SnapshotRecord>
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Provider-local fallback state when a host has no visible workspace methods. */
|
|
259
|
+
interface LocalTaskState {
|
|
260
|
+
status: BrowserTaskStatus
|
|
261
|
+
control: BrowserControlOwner
|
|
262
|
+
latestAction?: string
|
|
263
|
+
error?: string
|
|
264
|
+
updatedAt: number
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** One live browser session: an ordered list of tabs, one active. */
|
|
268
|
+
interface Session {
|
|
269
|
+
readonly id: BrowserSessionId
|
|
270
|
+
readonly taskKey: string
|
|
271
|
+
taskLabel: string
|
|
272
|
+
readonly tabs: Tab[]
|
|
273
|
+
activeIndex: number
|
|
274
|
+
/** Chronological operation log (navigate/execute/click/type/fill/download/auth). */
|
|
275
|
+
readonly history: BrowserHistoryEntry[]
|
|
276
|
+
/** Monotonic sequence counter for history entries (survives truncation). */
|
|
277
|
+
nextSeq: number
|
|
278
|
+
/** The most recent JS dialog the host reported, kept for browser_dialog inspect. */
|
|
279
|
+
lastDialog?: unknown
|
|
280
|
+
/** How the host should answer the next JS dialog. Default: accept. */
|
|
281
|
+
dialogPolicy?: DialogPolicy
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* How the host answers a JS dialog (alert/confirm/prompt).
|
|
286
|
+
*
|
|
287
|
+
* A dialog freezes the renderer until it is answered, so the default stays
|
|
288
|
+
* `accept` - automation must never hang on one. `dismiss` is for pages whose
|
|
289
|
+
* confirmation is part of what is being driven (delete prompts and the like),
|
|
290
|
+
* and `promptText` supplies the value for a prompt.
|
|
291
|
+
*/
|
|
292
|
+
/** One captured console message. */
|
|
293
|
+
export interface BrowserConsoleMessage { level: string; text: string; at: string }
|
|
294
|
+
/** One captured network request. */
|
|
295
|
+
export interface BrowserNetworkRequest {
|
|
296
|
+
method: string
|
|
297
|
+
url: string
|
|
298
|
+
status?: number
|
|
299
|
+
mime?: string
|
|
300
|
+
kind?: string
|
|
301
|
+
failed?: string
|
|
302
|
+
ms?: number
|
|
303
|
+
at: string
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/** Device/viewport/media emulation for one tab (browser_emulate). */
|
|
307
|
+
export interface EmulateOptions {
|
|
308
|
+
readonly width?: number
|
|
309
|
+
readonly height?: number
|
|
310
|
+
readonly deviceScaleFactor?: number
|
|
311
|
+
readonly mobile?: boolean
|
|
312
|
+
readonly userAgent?: string
|
|
313
|
+
readonly colorScheme?: 'light' | 'dark' | 'no-preference'
|
|
314
|
+
/** Undo everything this tool set on the tab. */
|
|
315
|
+
readonly clear?: boolean
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
export interface DialogPolicy {
|
|
319
|
+
readonly behavior: 'accept' | 'dismiss'
|
|
320
|
+
readonly promptText?: string
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/** Provider config: navigation admission defaults and snapshot caps. */
|
|
324
|
+
export interface ElectronBrowserProviderConfig {
|
|
325
|
+
/** Allow navigation only to HTTP(S) URLs; reject anything else. Default true. */
|
|
326
|
+
readonly httpOnly?: boolean
|
|
327
|
+
/** Maximum snapshot elements before truncation. Default 60. */
|
|
328
|
+
readonly snapshotMaxElements?: number
|
|
329
|
+
/** Maximum content characters before truncation when no maxChars is given. Default 100_000. */
|
|
330
|
+
readonly contentMaxChars?: number
|
|
331
|
+
/**
|
|
332
|
+
* Absolute directories a screenshot or download may write into. Defaults to
|
|
333
|
+
* the workspace and the OS temp directory ({@link defaultWriteRoots}); an
|
|
334
|
+
* empty list denies every write.
|
|
335
|
+
*/
|
|
336
|
+
readonly writeRoots?: readonly string[]
|
|
337
|
+
/**
|
|
338
|
+
* Absolute directories `browser_upload_file` may read from. Defaults to the
|
|
339
|
+
* same roots as {@link writeRoots}; an empty list denies every upload.
|
|
340
|
+
*/
|
|
341
|
+
readonly readRoots?: readonly string[]
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* CDP method/params for `Page.navigate`, as sent to {@link ElectronViewHandle.sendCommand}.
|
|
346
|
+
*/
|
|
347
|
+
export interface CdpNavigateParams {
|
|
348
|
+
readonly url: string
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* CDP method/params for `Input.dispatchMouseEvent` (a click press+release pair).
|
|
353
|
+
*/
|
|
354
|
+
export interface CdpMouseParams {
|
|
355
|
+
readonly type: 'mousePressed' | 'mouseReleased' | 'mouseMoved'
|
|
356
|
+
readonly x: number
|
|
357
|
+
readonly y: number
|
|
358
|
+
readonly button: 'left' | 'right' | 'middle' | 'none'
|
|
359
|
+
readonly clickCount?: number
|
|
360
|
+
/** Buttons held during the event; 1 while a left drag is in flight. */
|
|
361
|
+
readonly buttons?: number
|
|
362
|
+
/** CDP modifier bitmask (Alt 1, Ctrl 2, Meta 4, Shift 8); see modifierMask. */
|
|
363
|
+
readonly modifiers?: number
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/** CDP method/params for `Input.insertText`. */
|
|
367
|
+
export interface CdpInsertTextParams {
|
|
368
|
+
readonly text: string
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/** CDP method/params for `Runtime.evaluate`. */
|
|
372
|
+
export interface CdpEvaluateParams {
|
|
373
|
+
readonly expression: string
|
|
374
|
+
readonly returnByValue: boolean
|
|
375
|
+
readonly awaitPromise?: boolean
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/** CDP method for a full-page screenshot capture. */
|
|
379
|
+
export const CDP_PAGE_CAPTURE_SCREENSHOT = 'Page.captureScreenshot'
|
|
380
|
+
/** CDP method for runtime evaluation (the execute path). */
|
|
381
|
+
export const CDP_RUNTIME_EVALUATE = 'Runtime.evaluate'
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* Decide how to hand a script to `Runtime.evaluate`.
|
|
385
|
+
*
|
|
386
|
+
* CDP evaluates an *expression*, so a script made of statements is a syntax
|
|
387
|
+
* error there. Try the expression form first - that keeps bare expressions and
|
|
388
|
+
* object literals returning their value, which is what the tool has always
|
|
389
|
+
* done - then fall back to a statement body. `const x = 1; return x` is the
|
|
390
|
+
* shape people actually type, and it used to come back as a bare SyntaxError.
|
|
391
|
+
*/
|
|
392
|
+
export function buildEvaluateBody(script: string): { body: string } | { error: string } {
|
|
393
|
+
try {
|
|
394
|
+
// Parses only; nothing is executed here.
|
|
395
|
+
new Function(`return (${script})`)
|
|
396
|
+
return { body: `return (${script})` }
|
|
397
|
+
} catch { /* not an expression - try it as a body */ }
|
|
398
|
+
try {
|
|
399
|
+
new Function(script)
|
|
400
|
+
return { body: script }
|
|
401
|
+
} catch (error) {
|
|
402
|
+
return { error: error instanceof Error ? error.message : String(error) }
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/** CDP method for keyboard input. */
|
|
407
|
+
export const CDP_INPUT_DISPATCH_KEY_EVENT = 'Input.dispatchKeyEvent'
|
|
408
|
+
/**
|
|
409
|
+
* True when an input command's target view is already gone.
|
|
410
|
+
*
|
|
411
|
+
* The page's own chrome handles Ctrl+W/Ctrl+T by asking the provider to close or
|
|
412
|
+
* open a tab, so the key dispatch that triggered it is still in flight when the
|
|
413
|
+
* view disappears: the host answers `unknown view`, or CDP says the target closed.
|
|
414
|
+
* Measured on the real machine - create and destroy of one view 1.5s apart, then
|
|
415
|
+
* this error on the Ctrl+W that caused the destroy.
|
|
416
|
+
*/
|
|
417
|
+
function isClosedInputTarget(error: unknown): boolean {
|
|
418
|
+
if (!(error instanceof Error)) return false
|
|
419
|
+
return /target closed|unknown view/i.test(error.message)
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
const KEY_VK: Record<string, number> = {
|
|
423
|
+
Backspace: 8,
|
|
424
|
+
Tab: 9,
|
|
425
|
+
Enter: 13,
|
|
426
|
+
Shift: 16,
|
|
427
|
+
Control: 17,
|
|
428
|
+
Alt: 18,
|
|
429
|
+
CapsLock: 20,
|
|
430
|
+
Escape: 27,
|
|
431
|
+
Space: 32,
|
|
432
|
+
PageUp: 33,
|
|
433
|
+
PageDown: 34,
|
|
434
|
+
End: 35,
|
|
435
|
+
Home: 36,
|
|
436
|
+
ArrowLeft: 37,
|
|
437
|
+
ArrowUp: 38,
|
|
438
|
+
ArrowRight: 39,
|
|
439
|
+
ArrowDown: 40,
|
|
440
|
+
Insert: 45,
|
|
441
|
+
Delete: 46,
|
|
442
|
+
Meta: 91,
|
|
443
|
+
F1: 112,
|
|
444
|
+
F2: 113,
|
|
445
|
+
F3: 114,
|
|
446
|
+
F4: 115,
|
|
447
|
+
F5: 116,
|
|
448
|
+
F6: 117,
|
|
449
|
+
F7: 118,
|
|
450
|
+
F8: 119,
|
|
451
|
+
F9: 120,
|
|
452
|
+
F10: 121,
|
|
453
|
+
F11: 122,
|
|
454
|
+
F12: 123,
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/** Printable ASCII with no KEY_VK entry, mapped to its US-layout position. */
|
|
458
|
+
const PRINTABLE_CODES: Record<string, { code: string; vk: number }> = {
|
|
459
|
+
'-': { code: 'Minus', vk: 189 },
|
|
460
|
+
'=': { code: 'Equal', vk: 187 },
|
|
461
|
+
'[': { code: 'BracketLeft', vk: 219 },
|
|
462
|
+
']': { code: 'BracketRight', vk: 221 },
|
|
463
|
+
';': { code: 'Semicolon', vk: 186 },
|
|
464
|
+
"'": { code: 'Quote', vk: 222 },
|
|
465
|
+
',': { code: 'Comma', vk: 188 },
|
|
466
|
+
'.': { code: 'Period', vk: 190 },
|
|
467
|
+
'/': { code: 'Slash', vk: 191 },
|
|
468
|
+
'`': { code: 'Backquote', vk: 192 },
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/** True for the ASCII range a key event can carry as text. */
|
|
472
|
+
function isPrintable(key: string): boolean {
|
|
473
|
+
if (key.length !== 1) return false
|
|
474
|
+
const code = key.charCodeAt(0)
|
|
475
|
+
return code >= 0x20 && code <= 0x7e
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
function keyText(key: string): string | null {
|
|
479
|
+
switch (key) {
|
|
480
|
+
case 'Enter': return '\r'
|
|
481
|
+
case 'Tab': return '\t'
|
|
482
|
+
case 'Space': return ' '
|
|
483
|
+
default: return isPrintable(key) ? key : null
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
function keyDescriptor(key: string): { key: string; code: string; vk: number } {
|
|
488
|
+
const upper = key.toUpperCase()
|
|
489
|
+
// A physical Space produces e.key === ' ' with code 'Space'.
|
|
490
|
+
if (key === 'Space') {
|
|
491
|
+
return { key: ' ', code: 'Space', vk: KEY_VK.Space }
|
|
492
|
+
}
|
|
493
|
+
if (KEY_VK[key] !== undefined) {
|
|
494
|
+
return { key, code: key, vk: KEY_VK[key] }
|
|
495
|
+
}
|
|
496
|
+
if (/^[a-z]$/i.test(key)) {
|
|
497
|
+
// Unshifted letters deliver e.key lowercase; the code keeps the physical form.
|
|
498
|
+
return { key, code: `Key${upper}`, vk: upper.charCodeAt(0) }
|
|
499
|
+
}
|
|
500
|
+
if (/^[0-9]$/.test(key)) {
|
|
501
|
+
return { key, code: `Digit${key}`, vk: key.charCodeAt(0) }
|
|
502
|
+
}
|
|
503
|
+
if (isPrintable(key)) {
|
|
504
|
+
// Punctuation carries its US-layout position so shortcuts such as Ctrl+- and
|
|
505
|
+
// Ctrl+/ reach the page; anything else printable falls back to its own code.
|
|
506
|
+
const known = PRINTABLE_CODES[key]
|
|
507
|
+
return known === undefined
|
|
508
|
+
? { key, code: key, vk: key.toUpperCase().charCodeAt(0) }
|
|
509
|
+
: { key, code: known.code, vk: known.vk }
|
|
510
|
+
}
|
|
511
|
+
throw new BrowserError(`browser: unsupported key "${key}"`, 'BROWSER_KEY_UNKNOWN')
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
function modifierMask(modifiers: readonly ('alt' | 'ctrl' | 'meta' | 'shift')[] | undefined): number {
|
|
515
|
+
let mask = 0
|
|
516
|
+
for (const mod of modifiers ?? []) {
|
|
517
|
+
if (mod === 'alt') mask |= 1
|
|
518
|
+
else if (mod === 'ctrl') mask |= 2
|
|
519
|
+
else if (mod === 'meta') mask |= 4
|
|
520
|
+
else if (mod === 'shift') mask |= 8
|
|
521
|
+
}
|
|
522
|
+
return mask
|
|
523
|
+
}
|
|
524
|
+
/** CDP method for navigation. */
|
|
525
|
+
export const CDP_PAGE_NAVIGATE = 'Page.navigate'
|
|
526
|
+
/** CDP methods used by native browser navigation controls. */
|
|
527
|
+
export const CDP_PAGE_GET_NAVIGATION_HISTORY = 'Page.getNavigationHistory'
|
|
528
|
+
export const CDP_PAGE_NAVIGATE_TO_HISTORY_ENTRY = 'Page.navigateToHistoryEntry'
|
|
529
|
+
export const CDP_PAGE_RELOAD = 'Page.reload'
|
|
530
|
+
export const CDP_PAGE_STOP_LOADING = 'Page.stopLoading'
|
|
531
|
+
|
|
532
|
+
/** Cap on content returned by a snapshot fetch to keep the wire bounded. */
|
|
533
|
+
const SNAPSHOT_LABEL_MAX = 120
|
|
534
|
+
|
|
535
|
+
/** An input dispatch must not outlive this: a blocked renderer never acknowledges. */
|
|
536
|
+
const INPUT_DISPATCH_TIMEOUT_MS = 15_000
|
|
537
|
+
|
|
538
|
+
/**
|
|
539
|
+
* Views whose renderer has already been told to consider itself focused.
|
|
540
|
+
* Weak so a destroyed view does not keep its handle alive.
|
|
541
|
+
*/
|
|
542
|
+
const focusEmulatedViews = new WeakSet<ElectronViewHandle>()
|
|
543
|
+
|
|
544
|
+
/** Gap between the move events of a drag; enough to span several frames. */
|
|
545
|
+
const DRAG_STEP_DELAY_MS = 8
|
|
546
|
+
|
|
547
|
+
/** Upper bound on concurrent scrape workers; each one costs a tab. */
|
|
548
|
+
const MAX_SCRAPE_WORKERS = 8
|
|
549
|
+
|
|
550
|
+
/** Mutable progress for one background scrape batch. */
|
|
551
|
+
interface ScrapeJob {
|
|
552
|
+
readonly id: string
|
|
553
|
+
readonly session: Session
|
|
554
|
+
/**
|
|
555
|
+
* The batch's own tabs, created at start and destroyed when it ends. One per
|
|
556
|
+
* worker. Deliberately never activated: a batch must not race a tool call for
|
|
557
|
+
* the session's active tab, and it must not navigate away from the page the
|
|
558
|
+
* human was reading.
|
|
559
|
+
*/
|
|
560
|
+
readonly tabs: readonly Tab[]
|
|
561
|
+
readonly path: string
|
|
562
|
+
state: 'running' | 'done' | 'stopped'
|
|
563
|
+
readonly total: number
|
|
564
|
+
done: number
|
|
565
|
+
failed: number
|
|
566
|
+
error?: string
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/** The immutable view of a job the seam hands out. */
|
|
570
|
+
function scrapeStatusOf(job: ScrapeJob): BrowserScrapeStatus {
|
|
571
|
+
return {
|
|
572
|
+
id: job.id,
|
|
573
|
+
state: job.state,
|
|
574
|
+
total: job.total,
|
|
575
|
+
done: job.done,
|
|
576
|
+
failed: job.failed,
|
|
577
|
+
path: job.path,
|
|
578
|
+
...job.error === undefined ? {} : { error: job.error },
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/**
|
|
583
|
+
* One JSONL row. A page can return a value JSON cannot carry (a circular object,
|
|
584
|
+
* a BigInt); that must not kill a batch that has already written hundreds of rows.
|
|
585
|
+
*/
|
|
586
|
+
function scrapeRow(row: Record<string, unknown>): string {
|
|
587
|
+
try {
|
|
588
|
+
return JSON.stringify(row) + '\n'
|
|
589
|
+
} catch (error) {
|
|
590
|
+
return JSON.stringify({
|
|
591
|
+
url: row.url,
|
|
592
|
+
ok: false,
|
|
593
|
+
error: `unserializable result: ${String((error as Error)?.message ?? error)}`,
|
|
594
|
+
}) + '\n'
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
/**
|
|
599
|
+
* Normalize one entry of a cookie export, or undefined when it cannot be used.
|
|
600
|
+
*
|
|
601
|
+
* Our own flushAuth emits `url`. Browser cookie editors (Cookie-Editor,
|
|
602
|
+
* EditThisCookie) and Edge's own export emit `domain` + `path` and no `url` at
|
|
603
|
+
* all — so requiring `url` rejected a file straight out of a browser wholesale,
|
|
604
|
+
* which is exactly the workflow this feature exists for. cookies.set wants a
|
|
605
|
+
* URL, so derive one when only the domain is present.
|
|
606
|
+
*/
|
|
607
|
+
function normalizeExportedCookie(value: unknown): ExportedCookie | undefined {
|
|
608
|
+
if (typeof value !== 'object' || value === null) return undefined
|
|
609
|
+
const record = value as Record<string, unknown>
|
|
610
|
+
if (typeof record.name !== 'string' || typeof record.value !== 'string') return undefined
|
|
611
|
+
const path = typeof record.path === 'string' && record.path.startsWith('/') ? record.path : '/'
|
|
612
|
+
const url = typeof record.url === 'string' && record.url !== ''
|
|
613
|
+
? record.url
|
|
614
|
+
: typeof record.domain === 'string' && record.domain !== ''
|
|
615
|
+
// A leading dot marks a domain-wide cookie; the URL host must not carry it.
|
|
616
|
+
? `${record.secure === true ? 'https' : 'http'}://${record.domain.replace(/^\./, '')}${path}`
|
|
617
|
+
: undefined
|
|
618
|
+
if (url === undefined) return undefined
|
|
619
|
+
const sameSite = normalizeSameSite(record.sameSite)
|
|
620
|
+
return {
|
|
621
|
+
url,
|
|
622
|
+
name: record.name,
|
|
623
|
+
value: record.value,
|
|
624
|
+
...typeof record.domain === 'string' ? { domain: record.domain } : {},
|
|
625
|
+
...typeof record.path === 'string' ? { path: record.path } : {},
|
|
626
|
+
...typeof record.secure === 'boolean' ? { secure: record.secure } : {},
|
|
627
|
+
...typeof record.httpOnly === 'boolean' ? { httpOnly: record.httpOnly } : {},
|
|
628
|
+
...typeof record.expirationDate === 'number' ? { expirationDate: record.expirationDate } : {},
|
|
629
|
+
...sameSite === undefined ? {} : { sameSite },
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/**
|
|
634
|
+
* Cookie editors emit Playwright's spelling (None/Lax/Strict) and Chromium's
|
|
635
|
+
* (no_restriction/lax/strict). cookies.set wants the latter.
|
|
636
|
+
*/
|
|
637
|
+
function normalizeSameSite(value: unknown): ExportedCookie['sameSite'] {
|
|
638
|
+
if (typeof value !== 'string') return undefined
|
|
639
|
+
switch (value.toLowerCase()) {
|
|
640
|
+
case 'none':
|
|
641
|
+
case 'no_restriction': return 'no_restriction'
|
|
642
|
+
case 'lax': return 'lax'
|
|
643
|
+
case 'strict': return 'strict'
|
|
644
|
+
case 'unspecified': return 'unspecified'
|
|
645
|
+
default: return undefined
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
/** Total budget for the snapshot's empty-inventory retries. */
|
|
650
|
+
const SNAPSHOT_RETRY_BUDGET_MS = 3_000
|
|
651
|
+
|
|
652
|
+
/**
|
|
653
|
+
* Longest script or typed text kept in one history entry. Entries exist to be
|
|
654
|
+
* replayed, so an over-long value is stored clipped and marked: replay then
|
|
655
|
+
* refuses outright rather than re-issuing a silently shortened script.
|
|
656
|
+
*/
|
|
657
|
+
const HISTORY_PARAM_MAX_CHARS = 32_768
|
|
658
|
+
|
|
659
|
+
/** Clip the replay payloads that would otherwise pin unbounded text in memory. */
|
|
660
|
+
function clampHistoryParams(params: Record<string, unknown>): Record<string, unknown> {
|
|
661
|
+
const tooLong = (value: unknown): value is string => typeof value === 'string' && value.length > HISTORY_PARAM_MAX_CHARS
|
|
662
|
+
if (!tooLong(params.script) && !tooLong(params.text)) return params
|
|
663
|
+
const clipped: Record<string, unknown> = { ...params }
|
|
664
|
+
if (tooLong(params.script)) {
|
|
665
|
+
clipped.script = params.script.slice(0, HISTORY_PARAM_MAX_CHARS)
|
|
666
|
+
clipped.scriptTruncated = true
|
|
667
|
+
}
|
|
668
|
+
if (tooLong(params.text)) {
|
|
669
|
+
clipped.text = params.text.slice(0, HISTORY_PARAM_MAX_CHARS)
|
|
670
|
+
clipped.textTruncated = true
|
|
671
|
+
}
|
|
672
|
+
return clipped
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
/**
|
|
676
|
+
* Browser provider over Electron views. Sessions hold an ordered list of
|
|
677
|
+
* tabs; each tab is one view created by the host. The active tab receives
|
|
678
|
+
* every operation; switching tabs calls the host's optional `showView` and
|
|
679
|
+
* never loses state. Navigation is admitted only for HTTP(S) targets unless
|
|
680
|
+
* {@link ElectronBrowserProviderConfig.httpOnly} is disabled.
|
|
681
|
+
*/
|
|
682
|
+
export class ElectronBrowserProvider implements BrowserProvider {
|
|
683
|
+
readonly id = ELECTRON_BROWSER_PROVIDER_ID
|
|
684
|
+
|
|
685
|
+
private readonly sessions = new Map<BrowserSessionId, Session>()
|
|
686
|
+
/** Stable task-key index so callers can recover a session after tool-layer state loss. */
|
|
687
|
+
private readonly sessionsByTask = new Map<string, BrowserSessionId>()
|
|
688
|
+
private readonly taskStates = new Map<string, LocalTaskState>()
|
|
689
|
+
private readonly httpOnly: boolean
|
|
690
|
+
private readonly snapshotMaxElements: number
|
|
691
|
+
private readonly contentMaxChars: number
|
|
692
|
+
private readonly writeRoots: readonly string[]
|
|
693
|
+
private readonly readRoots: readonly string[]
|
|
694
|
+
/** Background scrape batches, keyed by id; rows live on disk, not here. */
|
|
695
|
+
private readonly scrapes = new Map<string, ScrapeJob>()
|
|
696
|
+
|
|
697
|
+
constructor(
|
|
698
|
+
private readonly host: ElectronBrowserViewHost,
|
|
699
|
+
config: ElectronBrowserProviderConfig = {},
|
|
700
|
+
) {
|
|
701
|
+
this.httpOnly = config.httpOnly ?? true
|
|
702
|
+
this.snapshotMaxElements = config.snapshotMaxElements ?? 60
|
|
703
|
+
this.contentMaxChars = config.contentMaxChars ?? 100_000
|
|
704
|
+
this.writeRoots = config.writeRoots ?? defaultWriteRoots()
|
|
705
|
+
this.readRoots = config.readRoots ?? defaultWriteRoots()
|
|
706
|
+
// A human clicking the injected chrome's own tabs is the one case where the
|
|
707
|
+
// host must tell the provider something: the host can show a different view,
|
|
708
|
+
// but only the provider owns the session's tab list and active index.
|
|
709
|
+
this.host.onChromeEvent?.(event => this.handleChromeEvent(event))
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
/**
|
|
713
|
+
* Apply one authenticated chrome request to the session that owns its task.
|
|
714
|
+
*
|
|
715
|
+
* Every field is re-checked here: the host authenticates the sender, this
|
|
716
|
+
* method decides whether the request still makes sense against the live tab
|
|
717
|
+
* model. A request that resolves to nothing (a stale strip, a tab closed a
|
|
718
|
+
* moment ago, a task with no session) is dropped rather than thrown, because
|
|
719
|
+
* a human click must never surface as an error inside a running tool call.
|
|
720
|
+
*/
|
|
721
|
+
private handleChromeEvent(event: ChromeHostEvent): void {
|
|
722
|
+
if (typeof event !== 'object' || event === null) return
|
|
723
|
+
if (typeof event.taskKey !== 'string' || event.taskKey === '') return
|
|
724
|
+
const sessionId = this.sessionsByTask.get(event.taskKey)
|
|
725
|
+
if (sessionId === undefined) return
|
|
726
|
+
const s = this.sessions.get(sessionId)
|
|
727
|
+
if (s === undefined) return
|
|
728
|
+
try {
|
|
729
|
+
if (event.type === 'new-tab') {
|
|
730
|
+
this.newTab(s)
|
|
731
|
+
// 点收藏栏来的新标签会带 url —— 建完再导航(新标签已经是 active)。
|
|
732
|
+
if (typeof event.url === 'string' && event.url !== '') {
|
|
733
|
+
void this.navigate(sessionId, { url: event.url }).catch(() => undefined)
|
|
734
|
+
}
|
|
735
|
+
return
|
|
736
|
+
}
|
|
737
|
+
if (typeof event.tabId !== 'string') return
|
|
738
|
+
const tab = s.tabs.find(candidate => candidate.handle.id === event.tabId)
|
|
739
|
+
if (tab === undefined) return
|
|
740
|
+
if (event.type === 'close-tab') {
|
|
741
|
+
void this.closeTab(sessionId, tab.id)
|
|
742
|
+
return
|
|
743
|
+
}
|
|
744
|
+
if (event.type === 'move-tab' && typeof event.toIndex === 'number') {
|
|
745
|
+
// The human dragged this tab: mirror the host's order so the session's tab
|
|
746
|
+
// list (what browser_list_tabs reports) matches the strip.
|
|
747
|
+
const from = s.tabs.indexOf(tab)
|
|
748
|
+
if (from >= 0) {
|
|
749
|
+
const active = s.tabs[s.activeIndex]
|
|
750
|
+
const [moved] = s.tabs.splice(from, 1)
|
|
751
|
+
const to = Math.max(0, Math.min(Math.trunc(event.toIndex), s.tabs.length))
|
|
752
|
+
s.tabs.splice(to, 0, moved)
|
|
753
|
+
// activeIndex is positional, so keep it pointing at the same tab.
|
|
754
|
+
const activeNow = s.tabs.indexOf(active)
|
|
755
|
+
if (activeNow >= 0) s.activeIndex = activeNow
|
|
756
|
+
}
|
|
757
|
+
return
|
|
758
|
+
}
|
|
759
|
+
if (event.type === 'activate-tab') {
|
|
760
|
+
const index = s.tabs.indexOf(tab)
|
|
761
|
+
if (index >= 0) s.activeIndex = index
|
|
762
|
+
}
|
|
763
|
+
} catch {
|
|
764
|
+
// A chrome request is best-effort: never let one break the provider.
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
/**
|
|
769
|
+
* Usable whenever the host can create views. A host that exposes a local
|
|
770
|
+
* {@link ElectronBrowserViewHost.isAvailable} probe is believed; a host that
|
|
771
|
+
* omits it (a desktop shell's known-good viewHost, or a test fake) is assumed
|
|
772
|
+
* usable. The probe is cheap and local, so this stays callable from the seam's
|
|
773
|
+
* provider-selection path; the host owns any caching it needs.
|
|
774
|
+
*/
|
|
775
|
+
available(): boolean {
|
|
776
|
+
const probe = this.host.isAvailable
|
|
777
|
+
if (typeof probe !== 'function') return true
|
|
778
|
+
try {
|
|
779
|
+
return probe.call(this.host) === true
|
|
780
|
+
} catch {
|
|
781
|
+
// A probe that throws (e.g. a binary resolver error) means "not usable";
|
|
782
|
+
// provider selection must never surface that error itself.
|
|
783
|
+
return false
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
/**
|
|
788
|
+
* Open or recover the browser session for a task key. The tool layer normally
|
|
789
|
+
* caches this id, but the Provider is authoritative so a scoped tool reload or
|
|
790
|
+
* a lost cache cannot create a second task session with a different tab set.
|
|
791
|
+
* Sessions keep isolated tabs, active tab, and history while the host keeps one
|
|
792
|
+
* human-selected task view visible in the shared BrowserWindow.
|
|
793
|
+
*/
|
|
794
|
+
async open(options?: BrowserOpenOptions): Promise<BrowserSessionId> {
|
|
795
|
+
const taskKey = options?.key ?? 'default'
|
|
796
|
+
const taskLabel = options?.label ?? ''
|
|
797
|
+
const existing = this.sessionForTask(taskKey)
|
|
798
|
+
if (existing !== undefined) {
|
|
799
|
+
if (taskLabel !== '' && existing.taskLabel !== taskLabel) {
|
|
800
|
+
existing.taskLabel = taskLabel
|
|
801
|
+
const active = existing.tabs[existing.activeIndex]?.handle
|
|
802
|
+
const labelable = active as { label?(label: string): Promise<void> } | undefined
|
|
803
|
+
if (typeof labelable?.label === 'function') await labelable.label(taskLabel).catch(() => undefined)
|
|
804
|
+
}
|
|
805
|
+
return existing.id
|
|
806
|
+
}
|
|
807
|
+
const handle = this.host.createView(taskKey, taskLabel === '' ? undefined : taskLabel)
|
|
808
|
+
const id = `browser:${randomUUID()}`
|
|
809
|
+
this.sessions.set(id, { id, taskKey, taskLabel, tabs: [this.createTab(handle)], activeIndex: 0, history: [], nextSeq: 1 })
|
|
810
|
+
this.sessionsByTask.set(taskKey, id)
|
|
811
|
+
if (!this.taskStates.has(taskKey)) {
|
|
812
|
+
this.taskStates.set(taskKey, { status: 'idle', control: 'agent', updatedAt: Date.now() })
|
|
813
|
+
}
|
|
814
|
+
return id
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
/** Open a URL in the active tab (default) or a new tab. */
|
|
818
|
+
async openUrl(session: BrowserSessionId, request: BrowserOpenRequest, signal?: AbortSignal): Promise<void> {
|
|
819
|
+
const s = this.session(session)
|
|
820
|
+
if (request.newTab === true) {
|
|
821
|
+
this.newTab(s)
|
|
822
|
+
}
|
|
823
|
+
await this.navigate(session, { url: request.url }, signal)
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
/** List the session's tabs with their titles. */
|
|
827
|
+
async listTabs(session: BrowserSessionId): Promise<readonly BrowserTab[]> {
|
|
828
|
+
const s = this.session(session)
|
|
829
|
+
const result: BrowserTab[] = []
|
|
830
|
+
for (let i = 0; i < s.tabs.length; i++) {
|
|
831
|
+
const tab = s.tabs[i]
|
|
832
|
+
if (tab === undefined) continue // defensive: array can shift under concurrency
|
|
833
|
+
result.push({
|
|
834
|
+
id: tab.id,
|
|
835
|
+
url: await this.currentUrl(tab.handle).catch(() => ''),
|
|
836
|
+
active: i === s.activeIndex,
|
|
837
|
+
})
|
|
838
|
+
}
|
|
839
|
+
return result
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
/** Switch to a tab by id; background task tabs stay hidden until user-selected. */
|
|
843
|
+
switchTab(session: BrowserSessionId, tabId: string): Promise<void> {
|
|
844
|
+
const s = this.session(session)
|
|
845
|
+
const index = s.tabs.findIndex(tab => tab.id === tabId)
|
|
846
|
+
if (index < 0) {
|
|
847
|
+
throw new BrowserError(`browser: tab "${tabId}" is not open in this session`, 'BROWSER_TAB_UNKNOWN')
|
|
848
|
+
}
|
|
849
|
+
s.activeIndex = index
|
|
850
|
+
this.showActive(s)
|
|
851
|
+
return Promise.resolve()
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
/**
|
|
855
|
+
* Close one tab; closing the active tab activates the next. Resolves false when
|
|
856
|
+
* the id is not open in this session, so a miss is distinguishable from a close.
|
|
857
|
+
*/
|
|
858
|
+
async closeTab(session: BrowserSessionId, tabId: string): Promise<boolean> {
|
|
859
|
+
const s = this.session(session)
|
|
860
|
+
const index = s.tabs.findIndex(tab => tab.id === tabId)
|
|
861
|
+
if (index < 0) return Promise.resolve(false) // idempotent
|
|
862
|
+
const removed = s.tabs[index]
|
|
863
|
+
if (removed !== undefined) {
|
|
864
|
+
s.tabs.splice(index, 1)
|
|
865
|
+
this.ignoreHostFailure(this.host.destroyView(removed.handle))
|
|
866
|
+
}
|
|
867
|
+
if (s.tabs.length === 0) {
|
|
868
|
+
// Session keeps one blank tab so it stays usable. **必须等**:newTab 建宿主视图是异步的,
|
|
869
|
+
// 先 showActive 会去显示一个还不存在的视图 → 宿主报 unknown view(而且是条没人接的 rejection,
|
|
870
|
+
// 会串到别的工具调用上报错)。
|
|
871
|
+
await this.newTab(s)
|
|
872
|
+
} else if (index < s.activeIndex) {
|
|
873
|
+
// Closing a tab before the active one shifts the array left; keep the
|
|
874
|
+
// same tab active by decrementing the index.
|
|
875
|
+
s.activeIndex -= 1
|
|
876
|
+
} else if (s.activeIndex >= s.tabs.length) {
|
|
877
|
+
// The active tab itself was closed; activate the last remaining one.
|
|
878
|
+
s.activeIndex = s.tabs.length - 1
|
|
879
|
+
}
|
|
880
|
+
this.showActive(s)
|
|
881
|
+
return true
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
/** Close every tab and reset to one blank tab. */
|
|
885
|
+
reset(session: BrowserSessionId): Promise<void> {
|
|
886
|
+
const s = this.session(session)
|
|
887
|
+
for (const tab of s.tabs) this.ignoreHostFailure(this.host.destroyView(tab.handle))
|
|
888
|
+
s.tabs.length = 0
|
|
889
|
+
this.newTab(s)
|
|
890
|
+
s.activeIndex = 0
|
|
891
|
+
this.showActive(s)
|
|
892
|
+
return Promise.resolve()
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
/**
|
|
896
|
+
* Dispatch one input command under the same hang guard as the CDP reads. A
|
|
897
|
+
* renderer blocked in synchronous JS never acknowledges, so an unbounded await
|
|
898
|
+
* here would hang the tool call until the caller's budget expired.
|
|
899
|
+
* @param handle - the view to dispatch into.
|
|
900
|
+
* @param method - the CDP input method.
|
|
901
|
+
* @param params - its parameters.
|
|
902
|
+
* @param signal - optional caller signal.
|
|
903
|
+
*/
|
|
904
|
+
private async dispatchInput(
|
|
905
|
+
handle: ElectronViewHandle,
|
|
906
|
+
method: string,
|
|
907
|
+
params: Record<string, unknown>,
|
|
908
|
+
signal?: AbortSignal,
|
|
909
|
+
): Promise<void> {
|
|
910
|
+
try {
|
|
911
|
+
await this.ensureInputFocus(handle)
|
|
912
|
+
await withTimeout(
|
|
913
|
+
handle.sendCommand(method, params),
|
|
914
|
+
INPUT_DISPATCH_TIMEOUT_MS,
|
|
915
|
+
signal,
|
|
916
|
+
`browser: ${method} timed out after ${INPUT_DISPATCH_TIMEOUT_MS}ms`,
|
|
917
|
+
)
|
|
918
|
+
} catch (error) {
|
|
919
|
+
// A tab closed under the keystroke that closed it is the requested outcome,
|
|
920
|
+
// not a failure: there is nothing left for this event to act on.
|
|
921
|
+
if (!isClosedInputTarget(error)) throw error
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
/**
|
|
926
|
+
* Tell a renderer it is focused, once, before synthesized input.
|
|
927
|
+
*
|
|
928
|
+
* Chromium drops a synthesized mouse *press* when the renderer does not
|
|
929
|
+
* believe it has focus — which is the normal state for a background task's
|
|
930
|
+
* view, and on a real page even for the visible one while its window is not
|
|
931
|
+
* active. Moves are not gated, so hover looked fine while every click
|
|
932
|
+
* resolved its target, reported success, and left the page untouched.
|
|
933
|
+
*
|
|
934
|
+
* Focus emulation keeps this on the trusted CDP input path: no synthetic
|
|
935
|
+
* DOM click, so the events stay isTrusted and nothing about the page's
|
|
936
|
+
* view of the browser changes.
|
|
937
|
+
*/
|
|
938
|
+
private async ensureInputFocus(handle: ElectronViewHandle): Promise<void> {
|
|
939
|
+
if (focusEmulatedViews.has(handle)) return
|
|
940
|
+
await handle.sendCommand('Emulation.setFocusEmulationEnabled', { enabled: true })
|
|
941
|
+
.then(() => { focusEmulatedViews.add(handle) })
|
|
942
|
+
.catch(() => undefined)
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
/**
|
|
946
|
+
* Admit one URL for a provider-driven fetch (navigation or download).
|
|
947
|
+
* The whole check is gated by `httpOnly`: when it is disabled, callers are
|
|
948
|
+
* trusted with any scheme. When it is enabled, only HTTP(S) is admitted and
|
|
949
|
+
* URL-embedded credentials are refused, so a target can never be reached
|
|
950
|
+
* with in-URL auth.
|
|
951
|
+
* @param url - the candidate URL.
|
|
952
|
+
* @param subject - the operation name used in the error text.
|
|
953
|
+
*/
|
|
954
|
+
private admitUrl(url: string, subject: 'navigation' | 'download'): void {
|
|
955
|
+
if (!this.httpOnly) return
|
|
956
|
+
let parsed: URL
|
|
957
|
+
try {
|
|
958
|
+
parsed = new URL(url)
|
|
959
|
+
} catch {
|
|
960
|
+
throw new BrowserError(`browser: refusing ${subject} to unparseable URL "${url}"`, 'BROWSER_NAVIGATION_BLOCKED')
|
|
961
|
+
}
|
|
962
|
+
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
|
963
|
+
throw new BrowserError(`browser: refusing ${subject} to non-HTTP(S) URL "${url}"`, 'BROWSER_NAVIGATION_BLOCKED')
|
|
964
|
+
}
|
|
965
|
+
if (parsed.username !== '' || parsed.password !== '') {
|
|
966
|
+
throw new BrowserError(`browser: refusing ${subject} to a URL with embedded credentials`, 'BROWSER_NAVIGATION_BLOCKED')
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
/** Navigate the active tab's view to a URL, honoring HTTP(S)-only admission. */
|
|
971
|
+
async navigate(session: BrowserSessionId, request: { readonly url: string }, signal?: AbortSignal): Promise<void> {
|
|
972
|
+
const s = this.session(session)
|
|
973
|
+
return this.navigateTab(s, this.activeTab(s), request.url, signal)
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
/**
|
|
977
|
+
* Navigate one tab. A scrape worker passes its own tab so a batch never races
|
|
978
|
+
* a tool call for the session's active tab.
|
|
979
|
+
* @param show - bring the tab to the front; a background worker passes false.
|
|
980
|
+
* @param settleMs - post-ready paint delay; a DOM-only reader passes 0.
|
|
981
|
+
*/
|
|
982
|
+
private async navigateTab(s: Session, tab: Tab, url: string, signal?: AbortSignal, show = true, settleMs = 250): Promise<void> {
|
|
983
|
+
const { handle } = tab
|
|
984
|
+
try {
|
|
985
|
+
this.admitUrl(url, 'navigation')
|
|
986
|
+
signal?.throwIfAborted()
|
|
987
|
+
// Page.navigate can hang on an unreachable/slow host; bound it like the
|
|
988
|
+
// evaluate paths so a wedged navigation surfaces as an error instead of
|
|
989
|
+
// blocking the tool call forever.
|
|
990
|
+
const timeoutMs = 30_000
|
|
991
|
+
const result = await withTimeout(
|
|
992
|
+
handle.sendCommand(CDP_PAGE_NAVIGATE, { url } satisfies CdpNavigateParams),
|
|
993
|
+
timeoutMs,
|
|
994
|
+
signal,
|
|
995
|
+
`browser: navigation timed out after ${timeoutMs}ms`,
|
|
996
|
+
)
|
|
997
|
+
// Page.navigate resolves even when the navigation fails; surface the
|
|
998
|
+
// failure instead of leaving a silent white screen.
|
|
999
|
+
const errorText = (result as { errorText?: string }).errorText
|
|
1000
|
+
if (typeof errorText === 'string' && errorText !== '') {
|
|
1001
|
+
throw new BrowserError(`browser: navigation to "${url}" failed: ${errorText}`, 'BROWSER_NAVIGATION_FAILED')
|
|
1002
|
+
}
|
|
1003
|
+
this.invalidateSnapshots(tab)
|
|
1004
|
+
this.record(s, 'navigate', { url }, true)
|
|
1005
|
+
if (show) this.showActive(s)
|
|
1006
|
+
// Page.navigate resolves on commit. Wait best-effort for page load so
|
|
1007
|
+
// browser_open does not snapshot a still-blank renderer.
|
|
1008
|
+
await waitForDocumentReady(handle, signal, settleMs)
|
|
1009
|
+
// Human chrome is page-injected: reapply after every document commit so
|
|
1010
|
+
// the toolbar is present after each navigation.
|
|
1011
|
+
void reinstallPageChrome(handle)
|
|
1012
|
+
} catch (error) {
|
|
1013
|
+
if (!(error instanceof BrowserError && (error as { code?: string }).code === 'BROWSER_NAVIGATION_BLOCKED')) {
|
|
1014
|
+
this.record(s, 'navigate', { url }, false, { error: String(error) })
|
|
1015
|
+
}
|
|
1016
|
+
throw error
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
/** Navigate to the previous history entry when one exists. */
|
|
1021
|
+
async back(session: BrowserSessionId, signal?: AbortSignal): Promise<boolean> {
|
|
1022
|
+
return this.navigateHistory(session, -1, 'back', signal)
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
/** Navigate to the next history entry when one exists. */
|
|
1026
|
+
async forward(session: BrowserSessionId, signal?: AbortSignal): Promise<boolean> {
|
|
1027
|
+
return this.navigateHistory(session, 1, 'forward', signal)
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
/** Reload the active page and restore the browser chrome afterwards. */
|
|
1031
|
+
async reload(session: BrowserSessionId, signal?: AbortSignal): Promise<void> {
|
|
1032
|
+
const s = this.session(session)
|
|
1033
|
+
const tab = this.activeTab(s)
|
|
1034
|
+
signal?.throwIfAborted()
|
|
1035
|
+
await withTimeout(tab.handle.sendCommand(CDP_PAGE_RELOAD, {}), 30_000, signal, 'browser: reload timed out after 30000ms')
|
|
1036
|
+
this.invalidateSnapshots(tab)
|
|
1037
|
+
this.record(s, 'reload', {}, true)
|
|
1038
|
+
this.showActive(s)
|
|
1039
|
+
await waitForDocumentReady(tab.handle, signal)
|
|
1040
|
+
void reinstallPageChrome(tab.handle)
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
/** Stop loading the active page. */
|
|
1044
|
+
async stopLoading(session: BrowserSessionId, signal?: AbortSignal): Promise<void> {
|
|
1045
|
+
const s = this.session(session)
|
|
1046
|
+
const { handle } = this.activeTab(s)
|
|
1047
|
+
signal?.throwIfAborted()
|
|
1048
|
+
await withTimeout(handle.sendCommand(CDP_PAGE_STOP_LOADING, {}), 10_000, signal, 'browser: stop loading timed out after 10000ms')
|
|
1049
|
+
this.record(s, 'stop', {}, true)
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
/** Execute JS in the active tab's page context. */
|
|
1053
|
+
async execute(session: BrowserSessionId, request: BrowserExecuteRequest, signal?: AbortSignal): Promise<BrowserExecuteResult> {
|
|
1054
|
+
const s = this.session(session)
|
|
1055
|
+
return this.executeTab(s, this.activeTab(s), request, signal)
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
/** Evaluate in one tab's page context. */
|
|
1059
|
+
private async executeTab(s: Session, tab: Tab, request: BrowserExecuteRequest, signal?: AbortSignal): Promise<BrowserExecuteResult> {
|
|
1060
|
+
const { handle } = tab
|
|
1061
|
+
signal?.throwIfAborted()
|
|
1062
|
+
await this.drainDialog(s, handle)
|
|
1063
|
+
try {
|
|
1064
|
+
// Wrap the script in a Function so `return` statements are legal and
|
|
1065
|
+
// request.args arrive as `arguments[0..n]` (a real function, not an
|
|
1066
|
+
// arrow, so `arguments` resolves). Args are embedded as a JSON array
|
|
1067
|
+
// literal; unserializable members become null.
|
|
1068
|
+
const built = buildEvaluateBody(request.script)
|
|
1069
|
+
if ('error' in built) {
|
|
1070
|
+
const exception = `browser: execute could not parse the script as an expression or as a statement body: ${built.error}`
|
|
1071
|
+
this.record(s, 'execute', { script: request.script }, false, { error: exception })
|
|
1072
|
+
return { ok: false, exception }
|
|
1073
|
+
}
|
|
1074
|
+
const body = built.body
|
|
1075
|
+
const hasArgs = request.args !== undefined && request.args.length > 0
|
|
1076
|
+
const expression = hasArgs
|
|
1077
|
+
? `(function(){ const __dshArgs = ${JSON.stringify(request.args)}; return Function(${JSON.stringify(body)}).apply(null, __dshArgs) })()`
|
|
1078
|
+
: `(function(){ return Function(${JSON.stringify(body)})() })()`
|
|
1079
|
+
// CDP Runtime.evaluate can hang indefinitely on a not-yet-loaded page
|
|
1080
|
+
// (navigate returned but the renderer has not committed). Bound it so a
|
|
1081
|
+
// stuck call surfaces as BROWSER_EXECUTE_TIMEOUT instead of wedging the
|
|
1082
|
+
// whole tool call. The caller's signal wins when it fires first.
|
|
1083
|
+
const timeoutMs = request.timeoutMs ?? 30_000
|
|
1084
|
+
const result = await withTimeout(
|
|
1085
|
+
handle.sendCommand(CDP_RUNTIME_EVALUATE, {
|
|
1086
|
+
expression,
|
|
1087
|
+
returnByValue: true,
|
|
1088
|
+
awaitPromise: true,
|
|
1089
|
+
} satisfies CdpEvaluateParams),
|
|
1090
|
+
timeoutMs,
|
|
1091
|
+
signal,
|
|
1092
|
+
`browser: execute timed out after ${timeoutMs}ms`,
|
|
1093
|
+
)
|
|
1094
|
+
if (result.exceptionDetails !== undefined) {
|
|
1095
|
+
const detail = result.exceptionDetails as { text?: string; exception?: { description?: string } }
|
|
1096
|
+
const exception = detail.exception?.description ?? detail.text ?? 'unknown exception'
|
|
1097
|
+
this.record(s, 'execute', { script: request.script }, false, { error: exception })
|
|
1098
|
+
return { ok: false, exception }
|
|
1099
|
+
}
|
|
1100
|
+
const value = (result.result as { value?: unknown } | undefined)?.value ?? null
|
|
1101
|
+
this.record(s, 'execute', {
|
|
1102
|
+
script: request.script,
|
|
1103
|
+
...request.args !== undefined && request.args.length > 0 ? { args: request.args } : {},
|
|
1104
|
+
}, true, { result: typeof value === 'string' ? value.slice(0, 500) : JSON.stringify(value).slice(0, 500) })
|
|
1105
|
+
return { ok: true, value }
|
|
1106
|
+
} catch (error) {
|
|
1107
|
+
if (error instanceof Error && error.name === 'TimeoutError') {
|
|
1108
|
+
throw new BrowserError(`browser: execute timed out after ${request.timeoutMs ?? 30_000}ms`, 'BROWSER_EXECUTE_TIMEOUT', { cause: error })
|
|
1109
|
+
}
|
|
1110
|
+
throw new BrowserError(`browser: execute failed: ${String(error)}`, 'BROWSER_EXECUTE_FAILED', { cause: error })
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
/** Produce an AI-friendly snapshot of the active tab. */
|
|
1115
|
+
async snapshot(session: BrowserSessionId, options: { query?: string; limit?: number } = {}, signal?: AbortSignal): Promise<BrowserSnapshotResult> {
|
|
1116
|
+
const s = this.session(session)
|
|
1117
|
+
const tab = this.activeTab(s)
|
|
1118
|
+
signal?.throwIfAborted()
|
|
1119
|
+
await this.drainDialog(s, tab.handle)
|
|
1120
|
+
const requested = options.limit === undefined ? this.snapshotMaxElements : Math.max(1, Math.min(1000, Math.trunc(options.limit)))
|
|
1121
|
+
const cap = options.limit === undefined ? this.snapshotMaxElements : requested
|
|
1122
|
+
const query = (options.query ?? '').trim().toLowerCase()
|
|
1123
|
+
const script = `(() => {
|
|
1124
|
+
const cap = ${String(cap)}
|
|
1125
|
+
const query = ${JSON.stringify(query)}
|
|
1126
|
+
const locatorOf = (el) => {
|
|
1127
|
+
if (el.id) return '#' + CSS.escape(el.id)
|
|
1128
|
+
if (el.name) return '[name=' + JSON.stringify(el.name) + ']'
|
|
1129
|
+
const aria = el.getAttribute('aria-label')
|
|
1130
|
+
if (aria) return '[aria-label=' + JSON.stringify(aria) + ']'
|
|
1131
|
+
const tag = el.tagName.toLowerCase()
|
|
1132
|
+
const text = (el.textContent || '').replace(/\\s+/g, ' ').trim().slice(0, 30)
|
|
1133
|
+
if (text) return tag + ':has-text("' + text.replace(/"/g, '\\\\"') + '")'
|
|
1134
|
+
return tag
|
|
1135
|
+
}
|
|
1136
|
+
const pathOf = (el) => {
|
|
1137
|
+
if (el.id) return '#' + CSS.escape(el.id)
|
|
1138
|
+
const parts = []
|
|
1139
|
+
let node = el
|
|
1140
|
+
while (node && node.nodeType === Node.ELEMENT_NODE) {
|
|
1141
|
+
let part = node.tagName.toLowerCase()
|
|
1142
|
+
const parent = node.parentElement
|
|
1143
|
+
if (parent) {
|
|
1144
|
+
const siblings = [...parent.children].filter(sibling => sibling.tagName === node.tagName)
|
|
1145
|
+
if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(node) + 1) + ')'
|
|
1146
|
+
}
|
|
1147
|
+
parts.unshift(part)
|
|
1148
|
+
if (node === document.body) break
|
|
1149
|
+
node = parent
|
|
1150
|
+
}
|
|
1151
|
+
return parts.join(' > ')
|
|
1152
|
+
}
|
|
1153
|
+
const fingerprintOf = (el) => [
|
|
1154
|
+
el.tagName,
|
|
1155
|
+
el.getAttribute('type') || '',
|
|
1156
|
+
el.id || '',
|
|
1157
|
+
el.getAttribute('name') || '',
|
|
1158
|
+
el.getAttribute('aria-label') || '',
|
|
1159
|
+
(el.textContent || el.value || '').toString().replace(/\s+/g, ' ').trim().slice(0, 120),
|
|
1160
|
+
].join('\u001f')
|
|
1161
|
+
const url = location.href
|
|
1162
|
+
const title = document.title || undefined
|
|
1163
|
+
const els = [...document.querySelectorAll('input, textarea, select, button, a[href], [role="button"], [role="searchbox"], [contenteditable="true"]')]
|
|
1164
|
+
const out = []
|
|
1165
|
+
let capped = false
|
|
1166
|
+
for (const el of els) {
|
|
1167
|
+
if (el.closest('[data-dsh-browser-chrome]')) continue
|
|
1168
|
+
const r = el.getBoundingClientRect()
|
|
1169
|
+
const cs = getComputedStyle(el)
|
|
1170
|
+
if (r.width < 4 || r.height < 4 || cs.visibility === 'hidden' || cs.display === 'none') continue
|
|
1171
|
+
const kind = el.tagName === 'INPUT' ? (el.type === 'checkbox' ? 'checkbox' : (el.type === 'submit' || el.type === 'button' ? 'button' : 'input'))
|
|
1172
|
+
: el.tagName === 'TEXTAREA' ? 'textarea'
|
|
1173
|
+
: el.tagName === 'SELECT' ? 'select'
|
|
1174
|
+
: el.tagName === 'BUTTON' ? 'button'
|
|
1175
|
+
: el.tagName === 'A' ? 'link' : 'other'
|
|
1176
|
+
const label = (el.getAttribute('aria-label') || el.placeholder || el.textContent || el.value || el.name || el.id || '').toString().replace(/\\s+/g, ' ').trim().slice(0, ${String(SNAPSHOT_LABEL_MAX)})
|
|
1177
|
+
if (!label && kind !== 'link') continue
|
|
1178
|
+
// 过滤放在计上限**之前**:默认上限是 60,先截断就永远搜不到后面的元素。
|
|
1179
|
+
if (query !== '' && (kind + ' ' + label).toLowerCase().indexOf(query) === -1) continue
|
|
1180
|
+
if (out.length >= cap) { capped = true; break }
|
|
1181
|
+
out.push({
|
|
1182
|
+
ref: out.length + 1,
|
|
1183
|
+
kind,
|
|
1184
|
+
label,
|
|
1185
|
+
selector: el.id ? '#' + CSS.escape(el.id) : (el.name ? '[name=' + JSON.stringify(el.name) + ']' : ''),
|
|
1186
|
+
loc: locatorOf(el),
|
|
1187
|
+
path: pathOf(el),
|
|
1188
|
+
fingerprint: fingerprintOf(el),
|
|
1189
|
+
x: Math.round(r.x + r.width / 2),
|
|
1190
|
+
y: Math.round(r.y + r.height / 2),
|
|
1191
|
+
})
|
|
1192
|
+
}
|
|
1193
|
+
const challenge = ${CHALLENGE_DETECT_EXPRESSION}
|
|
1194
|
+
const chromeHost = document.getElementById('__dsh_browser_chrome_host__')
|
|
1195
|
+
const userControlling = chromeHost?.getAttribute('data-dsh-user-active') === '1'
|
|
1196
|
+
// Only an early break means truncation: exactly cap candidates is a
|
|
1197
|
+
// complete inventory, not a truncated one.
|
|
1198
|
+
return { url, title, elements: out, truncated: capped, challenge, userControlling }
|
|
1199
|
+
})()`
|
|
1200
|
+
// Same hang guard as execute: a renderer that has not committed after
|
|
1201
|
+
// navigate would otherwise block snapshot forever.
|
|
1202
|
+
const timeoutMs = 30_000
|
|
1203
|
+
const result = await withTimeout(
|
|
1204
|
+
handleSendEvaluate(tab.handle, script),
|
|
1205
|
+
timeoutMs,
|
|
1206
|
+
signal,
|
|
1207
|
+
`browser: snapshot timed out after ${timeoutMs}ms`,
|
|
1208
|
+
)
|
|
1209
|
+
if (!result.ok) throw new BrowserError(`browser: snapshot evaluation failed: ${result.exception}`, 'BROWSER_SNAPSHOT_FAILED')
|
|
1210
|
+
type RawSnapshotElement = BrowserSnapshotElement & { readonly path: string; readonly fingerprint: string }
|
|
1211
|
+
type RawSnapshot = Omit<BrowserSnapshotResult, 'snapshotId' | 'elements'> & { readonly elements: readonly RawSnapshotElement[] }
|
|
1212
|
+
let value = result.value as RawSnapshot
|
|
1213
|
+
// Framework apps often hydrate controls after the load event. A short
|
|
1214
|
+
// bounded retry turns premature empty inventories into useful snapshots.
|
|
1215
|
+
// The phase is budgeted because each attempt may itself wait the evaluation
|
|
1216
|
+
// timeout, and an aborted call must surface rather than be retried on.
|
|
1217
|
+
const retryDeadline = Date.now() + SNAPSHOT_RETRY_BUDGET_MS
|
|
1218
|
+
for (let attempt = 0; attempt < 5 && value.elements.length === 0 && value.truncated !== true; attempt++) {
|
|
1219
|
+
const remaining = retryDeadline - Date.now()
|
|
1220
|
+
if (remaining <= 0) break
|
|
1221
|
+
signal?.throwIfAborted()
|
|
1222
|
+
await new Promise(resolve => setTimeout(resolve, Math.min(400, remaining)))
|
|
1223
|
+
if (signal?.aborted === true) break
|
|
1224
|
+
const retry = await withTimeout(
|
|
1225
|
+
handleSendEvaluate(tab.handle, script, signal),
|
|
1226
|
+
Math.min(timeoutMs, Math.max(retryDeadline - Date.now(), 500)),
|
|
1227
|
+
signal,
|
|
1228
|
+
`browser: snapshot timed out after ${timeoutMs}ms`,
|
|
1229
|
+
).catch((error: unknown) => {
|
|
1230
|
+
if (signal?.aborted === true) throw error
|
|
1231
|
+
return undefined
|
|
1232
|
+
})
|
|
1233
|
+
if (retry?.ok) value = retry.value as RawSnapshot
|
|
1234
|
+
}
|
|
1235
|
+
const snapshotId = `snapshot:${randomUUID()}`
|
|
1236
|
+
const targets = new Map<number, SnapshotTarget>()
|
|
1237
|
+
for (const element of value.elements) {
|
|
1238
|
+
targets.set(element.ref, { path: element.path, fingerprint: element.fingerprint })
|
|
1239
|
+
}
|
|
1240
|
+
tab.snapshots.set(snapshotId, { tabId: tab.id, url: value.url, epoch: tab.navigationEpoch, targets })
|
|
1241
|
+
while (tab.snapshots.size > 10) {
|
|
1242
|
+
const oldest = tab.snapshots.keys().next().value as string | undefined
|
|
1243
|
+
if (oldest === undefined) break
|
|
1244
|
+
tab.snapshots.delete(oldest)
|
|
1245
|
+
}
|
|
1246
|
+
return {
|
|
1247
|
+
snapshotId,
|
|
1248
|
+
url: value.url,
|
|
1249
|
+
...value.title !== undefined ? { title: value.title } : {},
|
|
1250
|
+
elements: value.elements.map(({ path: _path, fingerprint: _fingerprint, ...element }) => element),
|
|
1251
|
+
truncated: value.truncated,
|
|
1252
|
+
...value.challenge !== undefined ? { challenge: value.challenge } : {},
|
|
1253
|
+
...value.userControlling !== undefined ? { userControlling: value.userControlling } : {},
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
/** Click one element that belongs to a retained exact page snapshot. */
|
|
1258
|
+
async clickRef(session: BrowserSessionId, request: BrowserRefRequest, signal?: AbortSignal): Promise<void> {
|
|
1259
|
+
const s = this.session(session)
|
|
1260
|
+
const tab = this.activeTab(s)
|
|
1261
|
+
signal?.throwIfAborted()
|
|
1262
|
+
await this.drainDialog(s, tab.handle)
|
|
1263
|
+
const point = await this.resolveSnapshotTarget(tab, request, 'center', signal)
|
|
1264
|
+
await suppressAutoUserControl(tab.handle, signal)
|
|
1265
|
+
await this.dispatchInput(tab.handle, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: point.x, y: point.y, button: 'left', clickCount: 1 } satisfies CdpMouseParams, signal)
|
|
1266
|
+
await this.dispatchInput(tab.handle, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: point.x, y: point.y, button: 'left', clickCount: 1 } satisfies CdpMouseParams, signal)
|
|
1267
|
+
this.record(s, 'clickRef', { snapshotId: request.snapshotId, ref: request.ref }, true)
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
/** Scroll one element that belongs to a retained exact page snapshot into view. */
|
|
1271
|
+
async scrollIntoView(session: BrowserSessionId, request: BrowserScrollIntoViewRequest, signal?: AbortSignal): Promise<BrowserScrollResult> {
|
|
1272
|
+
const s = this.session(session)
|
|
1273
|
+
const tab = this.activeTab(s)
|
|
1274
|
+
signal?.throwIfAborted()
|
|
1275
|
+
const result = await this.resolveSnapshotTarget(tab, request, request.block ?? 'center', signal)
|
|
1276
|
+
this.record(s, 'scrollIntoView', { snapshotId: request.snapshotId, ref: request.ref, block: request.block ?? 'center' }, true)
|
|
1277
|
+
return result
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
/** Check whether a human-verification challenge is blocking the active tab. */
|
|
1281
|
+
async detectChallenge(session: BrowserSessionId, signal?: AbortSignal): Promise<BrowserChallenge> {
|
|
1282
|
+
const s = this.session(session)
|
|
1283
|
+
const tab = this.activeTab(s)
|
|
1284
|
+
signal?.throwIfAborted()
|
|
1285
|
+
const timeoutMs = 15_000
|
|
1286
|
+
const result = await withTimeout(
|
|
1287
|
+
handleSendEvaluate(tab.handle, CHALLENGE_DETECT_EXPRESSION),
|
|
1288
|
+
timeoutMs,
|
|
1289
|
+
signal,
|
|
1290
|
+
`browser: challenge detection timed out after ${timeoutMs}ms`,
|
|
1291
|
+
)
|
|
1292
|
+
if (!result.ok) {
|
|
1293
|
+
throw new BrowserError(`browser: challenge detection failed: ${result.exception}`, 'BROWSER_CHALLENGE_DETECT_FAILED')
|
|
1294
|
+
}
|
|
1295
|
+
const value = result.value as BrowserChallenge
|
|
1296
|
+
return { blocked: value.blocked === true, kind: value.kind, reason: value.reason }
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
/** Fetch page content in a requested format. */
|
|
1300
|
+
async content(session: BrowserSessionId, request: BrowserContentRequest, signal?: AbortSignal): Promise<BrowserContentResult> {
|
|
1301
|
+
const s = this.session(session)
|
|
1302
|
+
const tab = this.activeTab(s)
|
|
1303
|
+
signal?.throwIfAborted()
|
|
1304
|
+
await this.drainDialog(s, tab.handle)
|
|
1305
|
+
const maxChars = request.maxChars ?? this.contentMaxChars
|
|
1306
|
+
const selector = request.selector ?? ''
|
|
1307
|
+
const format = request.format
|
|
1308
|
+
const script = `(() => {
|
|
1309
|
+
const root = ${selector === '' ? 'document.body' : `document.querySelector(${JSON.stringify(selector)})`}
|
|
1310
|
+
if (!root) return { ok: false, reason: 'selector not found' }
|
|
1311
|
+
const fmt = ${JSON.stringify(format)}
|
|
1312
|
+
let content = ''
|
|
1313
|
+
if (fmt === 'txt') content = root.innerText || ''
|
|
1314
|
+
else if (fmt === 'html') content = root.outerHTML || ''
|
|
1315
|
+
else if (fmt === 'json') {
|
|
1316
|
+
// An element has no own enumerable properties, so JSON.stringify(root)
|
|
1317
|
+
// always produced "{}". Serialize a bounded structural view instead, and
|
|
1318
|
+
// pass a genuine JSON payload through unchanged.
|
|
1319
|
+
const raw = (root.textContent || '').trim()
|
|
1320
|
+
let parsed
|
|
1321
|
+
try { parsed = JSON.parse(raw) } catch { parsed = undefined }
|
|
1322
|
+
if (parsed !== undefined) content = JSON.stringify(parsed)
|
|
1323
|
+
else {
|
|
1324
|
+
const shape = (el, depth) => {
|
|
1325
|
+
const node = { tag: el.tagName ? el.tagName.toLowerCase() : undefined }
|
|
1326
|
+
if (depth >= 8) return node
|
|
1327
|
+
if (el.id) node.id = el.id
|
|
1328
|
+
if (typeof el.className === 'string' && el.className !== '') node.class = el.className
|
|
1329
|
+
const kids = el.children ? [...el.children].slice(0, 40) : []
|
|
1330
|
+
if (kids.length > 0) node.children = kids.map(child => shape(child, depth + 1))
|
|
1331
|
+
else {
|
|
1332
|
+
const text = (el.textContent || '').trim().slice(0, 200)
|
|
1333
|
+
if (text !== '') node.text = text
|
|
1334
|
+
}
|
|
1335
|
+
return node
|
|
1336
|
+
}
|
|
1337
|
+
content = JSON.stringify(shape(root, 0))
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
// markdown: headings, links, lists, paragraphs (best-effort). The
|
|
1341
|
+
// renderer is embedded from its own source, so the function under test
|
|
1342
|
+
// is byte-for-byte the one that runs in the page.
|
|
1343
|
+
else content = (${renderMarkdown.toString()})(root)
|
|
1344
|
+
const truncated = content.length > ${String(maxChars)}
|
|
1345
|
+
return { ok: true, content: content.slice(0, ${String(maxChars)}), truncated }
|
|
1346
|
+
})()`
|
|
1347
|
+
// Honor a per-call timeout: content evaluation can hang on a heavy page,
|
|
1348
|
+
// so a caller-supplied budget bounds it. Unlike a bare signal entry check,
|
|
1349
|
+
// withTimeout also interrupts a call already in flight.
|
|
1350
|
+
const timeoutMs = request.timeoutMs ?? 30_000
|
|
1351
|
+
const result = await withTimeout(
|
|
1352
|
+
handleSendEvaluate(tab.handle, script),
|
|
1353
|
+
timeoutMs,
|
|
1354
|
+
signal,
|
|
1355
|
+
`browser: content timed out after ${timeoutMs}ms`,
|
|
1356
|
+
)
|
|
1357
|
+
if (!result.ok) throw new BrowserError(`browser: content evaluation failed: ${result.exception}`, 'BROWSER_CONTENT_FAILED')
|
|
1358
|
+
const value = result.value as { ok: boolean; reason?: string; content?: string; truncated?: boolean }
|
|
1359
|
+
if (!value.ok) throw new BrowserError(`browser: content fetch failed: ${value.reason ?? 'unknown'}`, 'BROWSER_CONTENT_FAILED')
|
|
1360
|
+
return { content: value.content ?? '', truncated: value.truncated ?? false }
|
|
1361
|
+
}
|
|
1362
|
+
|
|
1363
|
+
/** Click at viewport coordinates (CDP mousePressed + mouseReleased). */
|
|
1364
|
+
async click(session: BrowserSessionId, target: BrowserPointerTarget, signal?: AbortSignal): Promise<BrowserPointerResult> {
|
|
1365
|
+
const s = this.session(session)
|
|
1366
|
+
const { handle } = this.activeTab(s)
|
|
1367
|
+
signal?.throwIfAborted()
|
|
1368
|
+
await this.drainDialog(s, handle)
|
|
1369
|
+
const point = await resolvePointerTarget(handle, target, signal)
|
|
1370
|
+
await suppressAutoUserControl(handle, signal)
|
|
1371
|
+
// Electron installs no native context menu, so a right-click reaches the
|
|
1372
|
+
// page's own handler — which is exactly what an agent wants to drive.
|
|
1373
|
+
const button = target.button ?? 'left'
|
|
1374
|
+
const modifiers = modifierMask(target.modifiers)
|
|
1375
|
+
await this.dispatchInput(handle, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: point.x, y: point.y, button, clickCount: 1, modifiers } satisfies CdpMouseParams, signal)
|
|
1376
|
+
await this.dispatchInput(handle, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: point.x, y: point.y, button, clickCount: 1, modifiers } satisfies CdpMouseParams, signal)
|
|
1377
|
+
this.record(s, 'click', { x: point.x, y: point.y, ...point.target === undefined ? {} : { target: point.target }, button, ...target.modifiers !== undefined && target.modifiers.length > 0 ? { modifiers: target.modifiers } : {} }, true)
|
|
1378
|
+
return point
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
/** Double-click a target (physical input; clickCount 2). */
|
|
1382
|
+
async doubleClick(session: BrowserSessionId, target: BrowserPointerTarget, signal?: AbortSignal): Promise<BrowserPointerResult> {
|
|
1383
|
+
const s = this.session(session)
|
|
1384
|
+
const { handle } = this.activeTab(s)
|
|
1385
|
+
signal?.throwIfAborted()
|
|
1386
|
+
await this.drainDialog(s, handle)
|
|
1387
|
+
const point = await resolvePointerTarget(handle, target, signal)
|
|
1388
|
+
await suppressAutoUserControl(handle, signal)
|
|
1389
|
+
const button = target.button ?? 'left'
|
|
1390
|
+
const modifiers = modifierMask(target.modifiers)
|
|
1391
|
+
await this.dispatchInput(handle, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: point.x, y: point.y, button, clickCount: 2, modifiers } satisfies CdpMouseParams, signal)
|
|
1392
|
+
await this.dispatchInput(handle, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: point.x, y: point.y, button, clickCount: 2, modifiers } satisfies CdpMouseParams, signal)
|
|
1393
|
+
this.record(s, 'doubleClick', { x: point.x, y: point.y, ...point.target === undefined ? {} : { target: point.target }, button, ...target.modifiers !== undefined && target.modifiers.length > 0 ? { modifiers: target.modifiers } : {} }, true)
|
|
1394
|
+
return point
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
/** Move the pointer over a target (no click). */
|
|
1398
|
+
async hover(session: BrowserSessionId, target: BrowserPointerTarget, signal?: AbortSignal): Promise<BrowserPointerResult> {
|
|
1399
|
+
const s = this.session(session)
|
|
1400
|
+
const { handle } = this.activeTab(s)
|
|
1401
|
+
signal?.throwIfAborted()
|
|
1402
|
+
await this.drainDialog(s, handle)
|
|
1403
|
+
const point = await resolvePointerTarget(handle, target, signal)
|
|
1404
|
+
await this.dispatchInput(handle, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x: point.x, y: point.y, button: 'none', modifiers: modifierMask(target.modifiers) } satisfies CdpMouseParams, signal)
|
|
1405
|
+
this.record(s, 'hover', { x: point.x, y: point.y, ...point.target === undefined ? {} : { target: point.target } }, true)
|
|
1406
|
+
return point
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
/**
|
|
1410
|
+
* Press on one target, move to another, release.
|
|
1411
|
+
*
|
|
1412
|
+
* A hand does not teleport: the intermediate moves are what pointer-based
|
|
1413
|
+
* sliders and sortable libraries listen for, so a press straight onto the
|
|
1414
|
+
* destination would be ignored. Note this drives *pointer* drags only —
|
|
1415
|
+
* HTML5 drag-and-drop needs dragstart/drop, which synthesized mouse moves do
|
|
1416
|
+
* not produce; use the page's own controls, or a click-based reorder, there.
|
|
1417
|
+
*/
|
|
1418
|
+
async drag(session: BrowserSessionId, request: BrowserDragRequest, signal?: AbortSignal): Promise<BrowserDragResult> {
|
|
1419
|
+
const s = this.session(session)
|
|
1420
|
+
const { handle } = this.activeTab(s)
|
|
1421
|
+
signal?.throwIfAborted()
|
|
1422
|
+
await this.drainDialog(s, handle)
|
|
1423
|
+
const from = await resolvePointerTarget(handle, request.from, signal)
|
|
1424
|
+
const to = await resolvePointerTarget(handle, request.to, signal)
|
|
1425
|
+
await suppressAutoUserControl(handle, signal)
|
|
1426
|
+
const steps = Math.max(1, Math.min(Math.trunc(request.steps ?? 12) || 12, 60))
|
|
1427
|
+
// Hover the source first: some libraries only arm on an enter.
|
|
1428
|
+
await this.dispatchInput(handle, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x: from.x, y: from.y, button: 'none', buttons: 0 } satisfies CdpMouseParams, signal)
|
|
1429
|
+
await this.dispatchInput(handle, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: from.x, y: from.y, button: 'left', clickCount: 1, buttons: 1 } satisfies CdpMouseParams, signal)
|
|
1430
|
+
for (let step = 1; step <= steps; step += 1) {
|
|
1431
|
+
const ratio = step / steps
|
|
1432
|
+
await this.dispatchInput(handle, 'Input.dispatchMouseEvent', {
|
|
1433
|
+
type: 'mouseMoved',
|
|
1434
|
+
x: from.x + (to.x - from.x) * ratio,
|
|
1435
|
+
y: from.y + (to.y - from.y) * ratio,
|
|
1436
|
+
button: 'left',
|
|
1437
|
+
buttons: 1,
|
|
1438
|
+
} satisfies CdpMouseParams, signal)
|
|
1439
|
+
// Distinct events, not one burst: a listener that reads positions per
|
|
1440
|
+
// frame needs the gesture to span more than a single tick.
|
|
1441
|
+
await new Promise(resolve => setTimeout(resolve, DRAG_STEP_DELAY_MS))
|
|
1442
|
+
}
|
|
1443
|
+
await this.dispatchInput(handle, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: to.x, y: to.y, button: 'left', clickCount: 1, buttons: 0 } satisfies CdpMouseParams, signal)
|
|
1444
|
+
this.record(s, 'drag', {
|
|
1445
|
+
from: { x: Math.round(from.x), y: Math.round(from.y), ...from.target === undefined ? {} : { target: from.target } },
|
|
1446
|
+
to: { x: Math.round(to.x), y: Math.round(to.y), ...to.target === undefined ? {} : { target: to.target } },
|
|
1447
|
+
steps,
|
|
1448
|
+
}, true)
|
|
1449
|
+
return { from, to }
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
/** Scroll the active page by CSS-pixel deltas and return the final position. */
|
|
1453
|
+
async scroll(session: BrowserSessionId, request: BrowserScrollRequest, signal?: AbortSignal): Promise<BrowserScrollResult> {
|
|
1454
|
+
const s = this.session(session)
|
|
1455
|
+
const tab = this.activeTab(s)
|
|
1456
|
+
signal?.throwIfAborted()
|
|
1457
|
+
const deltaX = request.deltaX ?? 0
|
|
1458
|
+
const deltaY = request.deltaY ?? 0
|
|
1459
|
+
const hasExplicitDelta = request.deltaX !== undefined || request.deltaY !== undefined
|
|
1460
|
+
const script = `(() => {
|
|
1461
|
+
const deltaX = ${JSON.stringify(deltaX)}
|
|
1462
|
+
const deltaY = ${JSON.stringify(deltaY)}
|
|
1463
|
+
const hasExplicitDelta = ${JSON.stringify(hasExplicitDelta)}
|
|
1464
|
+
const effectiveDeltaY = hasExplicitDelta ? deltaY : Math.max(window.innerHeight * 0.8, 480)
|
|
1465
|
+
window.scrollBy(deltaX, effectiveDeltaY)
|
|
1466
|
+
const root = document.documentElement
|
|
1467
|
+
return {
|
|
1468
|
+
x: window.scrollX,
|
|
1469
|
+
y: window.scrollY,
|
|
1470
|
+
maxX: Math.max(0, root.scrollWidth - window.innerWidth),
|
|
1471
|
+
maxY: Math.max(0, root.scrollHeight - window.innerHeight),
|
|
1472
|
+
}
|
|
1473
|
+
})()`
|
|
1474
|
+
const result = await withTimeout(handleSendEvaluate(tab.handle, script), 15_000, signal, 'browser: scroll timed out')
|
|
1475
|
+
if (!result.ok) throw new BrowserError(`browser: scroll failed: ${result.exception}`, 'BROWSER_SCROLL_FAILED')
|
|
1476
|
+
const value = result.value as Partial<BrowserScrollResult>
|
|
1477
|
+
if (typeof value.x !== 'number' || typeof value.y !== 'number' || typeof value.maxX !== 'number' || typeof value.maxY !== 'number') {
|
|
1478
|
+
throw new BrowserError('browser: scroll returned invalid coordinates', 'BROWSER_SCROLL_FAILED')
|
|
1479
|
+
}
|
|
1480
|
+
this.record(s, 'scroll', { deltaX, deltaY: hasExplicitDelta ? deltaY : 'viewport' }, true)
|
|
1481
|
+
return { x: value.x, y: value.y, maxX: value.maxX, maxY: value.maxY }
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
/**
|
|
1485
|
+
* Attach a local file to the first matching file input. Uses the CDP DOM
|
|
1486
|
+
* domain (nodeId path), which — unlike a synthetic change event — makes the
|
|
1487
|
+
* input's files list true (real file selection), so pages that read
|
|
1488
|
+
* input.files or upload on change behave exactly like a real pick.
|
|
1489
|
+
*/
|
|
1490
|
+
async uploadFile(session: BrowserSessionId, request: BrowserUploadFileRequest, signal?: AbortSignal): Promise<BrowserUploadFileResult> {
|
|
1491
|
+
const s = this.session(session)
|
|
1492
|
+
const { handle } = this.activeTab(s)
|
|
1493
|
+
signal?.throwIfAborted()
|
|
1494
|
+
await this.drainDialog(s, handle)
|
|
1495
|
+
// Same input semantics as click/type: do not let the page hand control to
|
|
1496
|
+
// the human while an agent-driven file selection is in flight.
|
|
1497
|
+
await suppressAutoUserControl(handle, signal)
|
|
1498
|
+
const selector = request.selector ?? 'input[type="file"]'
|
|
1499
|
+
// A file handed to a page leaves the machine, so admission happens before
|
|
1500
|
+
// any DOM work: the path must exist and sit inside the configured roots.
|
|
1501
|
+
const filePath = resolveReadPath(request.filePath, this.readRoots)
|
|
1502
|
+
// Bound the whole DOM sequence: a wedged renderer must not hang the tool.
|
|
1503
|
+
const timeoutMs = 30_000
|
|
1504
|
+
await withTimeout((async () => {
|
|
1505
|
+
const doc = await handle.sendCommand('DOM.getDocument', {})
|
|
1506
|
+
const root = (doc as { root?: { nodeId?: number } }).root
|
|
1507
|
+
const rootId = root?.nodeId
|
|
1508
|
+
if (rootId === undefined) {
|
|
1509
|
+
throw new BrowserError('browser: could not resolve the document node', 'BROWSER_UPLOAD_FAILED')
|
|
1510
|
+
}
|
|
1511
|
+
const query = await handle.sendCommand('DOM.querySelector', { nodeId: rootId, selector })
|
|
1512
|
+
const nodeId = (query as { nodeId?: number }).nodeId
|
|
1513
|
+
if (nodeId === undefined || nodeId === 0) {
|
|
1514
|
+
throw new BrowserError(`browser: no file input matches "${selector}"`, 'BROWSER_UPLOAD_NO_INPUT')
|
|
1515
|
+
}
|
|
1516
|
+
await handle.sendCommand('DOM.setFileInputFiles', { files: [filePath], nodeId })
|
|
1517
|
+
})(), timeoutMs, signal, `browser: upload timed out after ${timeoutMs}ms`)
|
|
1518
|
+
this.record(s, 'uploadFile', { filePath: request.filePath, selector }, true, { result: '1 file attached' })
|
|
1519
|
+
return { path: request.filePath }
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
/**
|
|
1523
|
+
* Poll until an element matching the selector exists (and is visible).
|
|
1524
|
+
* Bounds the total wait; a timeout surfaces as BROWSER_WAIT_TIMEOUT.
|
|
1525
|
+
*/
|
|
1526
|
+
async waitForElement(session: BrowserSessionId, request: BrowserWaitForRequest, signal?: AbortSignal): Promise<BrowserWaitForResult> {
|
|
1527
|
+
const s = this.session(session)
|
|
1528
|
+
return this.waitForElementTab(s, this.activeTab(s), request, signal)
|
|
1529
|
+
}
|
|
1530
|
+
|
|
1531
|
+
/** Poll one tab until the selector matches. */
|
|
1532
|
+
private async waitForElementTab(s: Session, tab: Tab, request: BrowserWaitForRequest, signal?: AbortSignal): Promise<BrowserWaitForResult> {
|
|
1533
|
+
const { handle } = tab
|
|
1534
|
+
signal?.throwIfAborted()
|
|
1535
|
+
const timeoutMs = request.timeoutMs ?? 15_000
|
|
1536
|
+
const visible = request.visible !== false
|
|
1537
|
+
const selector = request.selector
|
|
1538
|
+
const script = `(() => {
|
|
1539
|
+
let el = null
|
|
1540
|
+
try { el = document.querySelector(${JSON.stringify(selector)}) } catch (e) { return { error: String(e) } }
|
|
1541
|
+
if (!el) return null
|
|
1542
|
+
if (${visible}) {
|
|
1543
|
+
const r = el.getBoundingClientRect()
|
|
1544
|
+
const cs = getComputedStyle(el)
|
|
1545
|
+
if (r.width < 4 || r.height < 4 || cs.visibility === 'hidden' || cs.display === 'none') return null
|
|
1546
|
+
}
|
|
1547
|
+
return {
|
|
1548
|
+
found: true,
|
|
1549
|
+
selector: ${JSON.stringify(selector)},
|
|
1550
|
+
tag: el.tagName.toLowerCase(),
|
|
1551
|
+
text: (el.textContent || '').replace(/\\s+/g, ' ').trim().slice(0, 200),
|
|
1552
|
+
}
|
|
1553
|
+
})()`
|
|
1554
|
+
const deadline = Date.now() + timeoutMs
|
|
1555
|
+
let lastError: string | undefined
|
|
1556
|
+
while (Date.now() <= deadline) {
|
|
1557
|
+
signal?.throwIfAborted()
|
|
1558
|
+
const result = await withTimeout(handleSendEvaluate(handle, script, signal), Math.max(deadline - Date.now(), 250), signal, 'browser: wait poll timed out').catch((error: unknown): BrowserExecuteResult => ({ ok: false, exception: String(error) }))
|
|
1559
|
+
if (!result.ok) {
|
|
1560
|
+
lastError = result.exception
|
|
1561
|
+
} else {
|
|
1562
|
+
const value = result.value as { found?: boolean; tag?: string; text?: string; error?: string } | null
|
|
1563
|
+
if (value?.found === true && typeof value.tag === 'string') {
|
|
1564
|
+
this.record(s, 'waitForElement', { selector, timeoutMs, visible }, true, { result: value.tag })
|
|
1565
|
+
return { found: true, selector, tag: value.tag, text: value.text ?? '' }
|
|
1566
|
+
}
|
|
1567
|
+
if (value?.error !== undefined) {
|
|
1568
|
+
// A malformed selector can never start matching, so fail now instead of
|
|
1569
|
+
// polling to the deadline and reporting a misleading timeout.
|
|
1570
|
+
throw new BrowserError(`browser: invalid selector "${selector}": ${value.error}`, 'BROWSER_SELECTOR_INVALID')
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
await new Promise(resolve => setTimeout(resolve, Math.min(250, Math.max(deadline - Date.now(), 1))))
|
|
1574
|
+
}
|
|
1575
|
+
throw new BrowserError(`browser: element "${selector}" did not appear within ${timeoutMs}ms${lastError !== undefined ? ` (${lastError})` : ''}`, 'BROWSER_WAIT_TIMEOUT')
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
/** Type into the focused element. */
|
|
1579
|
+
async type(session: BrowserSessionId, request: { readonly text: string }, signal?: AbortSignal): Promise<void> {
|
|
1580
|
+
const s = this.session(session)
|
|
1581
|
+
const { handle } = this.activeTab(s)
|
|
1582
|
+
signal?.throwIfAborted()
|
|
1583
|
+
await this.drainDialog(s, handle)
|
|
1584
|
+
await suppressAutoUserControl(handle, signal)
|
|
1585
|
+
await this.dispatchInput(handle, 'Input.insertText', { text: request.text } satisfies CdpInsertTextParams, signal)
|
|
1586
|
+
// Store the full text so replay re-issues the same input; the history
|
|
1587
|
+
// tool truncates long values when rendering.
|
|
1588
|
+
this.record(s, 'type', { text: request.text }, true)
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
/**
|
|
1592
|
+
* Drive the host's chrome frame view with a raw CDP command.
|
|
1593
|
+
*
|
|
1594
|
+
* The chrome can live in a view of its own so the page viewport can really
|
|
1595
|
+
* shrink; that view is not a tab, so this is the only way to click the toolbar
|
|
1596
|
+
* (the click tests use it, and so does anything that needs to exercise the
|
|
1597
|
+
* chrome the way a person does).
|
|
1598
|
+
*/
|
|
1599
|
+
async chromeInput(method: string, params: Record<string, unknown> = {}): Promise<void> {
|
|
1600
|
+
const host = this.host
|
|
1601
|
+
if (typeof host.chromeInput !== 'function') throw new Error('this browser host has no chrome frame view')
|
|
1602
|
+
await host.chromeInput(method, params)
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
/**
|
|
1606
|
+
* Read a value back out of the chrome frame's own document.
|
|
1607
|
+
*
|
|
1608
|
+
* The frame is not a tab, so a page-directed evaluate cannot see it; without
|
|
1609
|
+
* this the toolbar's animations could only be inferred from the page's copy.
|
|
1610
|
+
*/
|
|
1611
|
+
async chromeEval(expression: string): Promise<unknown> {
|
|
1612
|
+
const host = this.host
|
|
1613
|
+
if (typeof host.chromeEval !== 'function') throw new Error('this browser host has no chrome frame view')
|
|
1614
|
+
return await host.chromeEval(expression)
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
/** Press a key into the page (keyDown + keyUp), as a physical-input path
|
|
1618
|
+
* for shortcuts and keyboard-driven UI. */
|
|
1619
|
+
async pressKey(session: BrowserSessionId, request: BrowserPressKeyRequest, signal?: AbortSignal): Promise<void> {
|
|
1620
|
+
const s = this.session(session)
|
|
1621
|
+
const { handle } = this.activeTab(s)
|
|
1622
|
+
signal?.throwIfAborted()
|
|
1623
|
+
await this.drainDialog(s, handle)
|
|
1624
|
+
await suppressAutoUserControl(handle, signal)
|
|
1625
|
+
const { key, code, vk } = keyDescriptor(request.key)
|
|
1626
|
+
const modifiers = modifierMask(request.modifiers)
|
|
1627
|
+
const text = keyText(request.key)
|
|
1628
|
+
const down: Record<string, unknown> = { type: text === null ? 'rawKeyDown' : 'keyDown', key, code, windowsVirtualKeyCode: vk, nativeVirtualKeyCode: vk, modifiers }
|
|
1629
|
+
if (text !== null) { down.text = text; down.unmodifiedText = text }
|
|
1630
|
+
await this.dispatchInput(handle, CDP_INPUT_DISPATCH_KEY_EVENT, down, signal)
|
|
1631
|
+
await this.dispatchInput(handle, CDP_INPUT_DISPATCH_KEY_EVENT, { type: 'keyUp', key, code, windowsVirtualKeyCode: vk, nativeVirtualKeyCode: vk, modifiers }, signal)
|
|
1632
|
+
this.record(s, 'pressKey', { key: request.key, ...(request.modifiers !== undefined && request.modifiers.length > 0 ? { modifiers: request.modifiers } : {}) }, true)
|
|
1633
|
+
}
|
|
1634
|
+
|
|
1635
|
+
/**
|
|
1636
|
+
* Fill a form's fields in one batch. Runs one page-context script that
|
|
1637
|
+
* resolves each field (selector, or name/label/placeholder among visible
|
|
1638
|
+
* controls), sets its value with the native prototype setter (React/Vue
|
|
1639
|
+
* controlled inputs included) plus input/change events, handles
|
|
1640
|
+
* select/checkbox/radio/contenteditable, and optionally submits the form.
|
|
1641
|
+
*/
|
|
1642
|
+
async fillForm(session: BrowserSessionId, request: BrowserFillRequest, signal?: AbortSignal): Promise<BrowserFillResult> {
|
|
1643
|
+
const s = this.session(session)
|
|
1644
|
+
const tab = this.activeTab(s)
|
|
1645
|
+
signal?.throwIfAborted()
|
|
1646
|
+
await this.drainDialog(s, tab.handle)
|
|
1647
|
+
const specs = JSON.stringify(request.fields.map(f => ({
|
|
1648
|
+
selector: f.selector ?? null,
|
|
1649
|
+
name: f.name ?? null,
|
|
1650
|
+
label: f.label ?? null,
|
|
1651
|
+
placeholder: f.placeholder ?? null,
|
|
1652
|
+
kind: f.kind ?? 'text',
|
|
1653
|
+
value: f.value,
|
|
1654
|
+
})))
|
|
1655
|
+
const submitFlag = request.submit === true
|
|
1656
|
+
const script = `(() => {
|
|
1657
|
+
const specs = ${specs}
|
|
1658
|
+
const out = []
|
|
1659
|
+
const setNative = (el, proto, value) => {
|
|
1660
|
+
const setter = Object.getOwnPropertyDescriptor(proto, 'value')?.set
|
|
1661
|
+
if (setter) setter.call(el, value)
|
|
1662
|
+
else el.value = value
|
|
1663
|
+
}
|
|
1664
|
+
const visible = (el) => {
|
|
1665
|
+
const r = el.getBoundingClientRect()
|
|
1666
|
+
const cs = getComputedStyle(el)
|
|
1667
|
+
return r.width >= 4 && r.height >= 4 && cs.visibility !== 'hidden' && cs.display !== 'none'
|
|
1668
|
+
}
|
|
1669
|
+
const describe = (spec) => spec.selector || spec.name || spec.label || spec.placeholder || '(unspecified)'
|
|
1670
|
+
const matches = (el, spec) => {
|
|
1671
|
+
if (spec.selector) { try { return el.matches(spec.selector) } catch { return false } }
|
|
1672
|
+
if (spec.name && el.name === spec.name) return true
|
|
1673
|
+
if (spec.placeholder && el.placeholder === spec.placeholder) return true
|
|
1674
|
+
if (spec.label) {
|
|
1675
|
+
if (el.getAttribute('aria-label') === spec.label) return true
|
|
1676
|
+
if (el.id) {
|
|
1677
|
+
const lbl = document.querySelector('label[for=' + JSON.stringify(el.id) + ']')
|
|
1678
|
+
if (lbl && (lbl.textContent || '').trim() === spec.label) return true
|
|
1679
|
+
}
|
|
1680
|
+
const wrap = el.closest('label')
|
|
1681
|
+
if (wrap && (wrap.textContent || '').trim() === spec.label) return true
|
|
1682
|
+
}
|
|
1683
|
+
return false
|
|
1684
|
+
}
|
|
1685
|
+
const candidates = (spec) => {
|
|
1686
|
+
const raw = spec.selector
|
|
1687
|
+
? [...document.querySelectorAll(spec.selector)]
|
|
1688
|
+
: [...document.querySelectorAll('input, textarea, select, [contenteditable="true"]')].filter(el => matches(el, spec))
|
|
1689
|
+
const all = raw.filter(el => !el.closest('[data-dsh-browser-chrome]'))
|
|
1690
|
+
const vis = all.filter(visible)
|
|
1691
|
+
return vis.length > 0 ? vis : all
|
|
1692
|
+
}
|
|
1693
|
+
const filledEls = []
|
|
1694
|
+
for (const spec of specs) {
|
|
1695
|
+
let els
|
|
1696
|
+
try {
|
|
1697
|
+
els = candidates(spec)
|
|
1698
|
+
} catch (e) {
|
|
1699
|
+
// A malformed selector must not abort the whole batch; report the
|
|
1700
|
+
// field as failed and continue with the rest.
|
|
1701
|
+
out.push({ ok: false, error: String(e), target: describe(spec) })
|
|
1702
|
+
continue
|
|
1703
|
+
}
|
|
1704
|
+
if (els.length === 0) { out.push({ ok: false, error: 'field not found', target: describe(spec) }); continue }
|
|
1705
|
+
const el = els[0]
|
|
1706
|
+
const tag = el.tagName
|
|
1707
|
+
const type = (el.type || '').toLowerCase()
|
|
1708
|
+
const before = out.length
|
|
1709
|
+
try {
|
|
1710
|
+
if (tag === 'SELECT') {
|
|
1711
|
+
const wanted = String(spec.value)
|
|
1712
|
+
if (el.multiple) {
|
|
1713
|
+
const wantedList = wanted.split(',').map(x => x.trim())
|
|
1714
|
+
let hit = false
|
|
1715
|
+
for (const o of [...el.options]) {
|
|
1716
|
+
o.selected = wantedList.includes(o.value) || wantedList.includes((o.textContent || '').trim())
|
|
1717
|
+
if (o.selected) hit = true
|
|
1718
|
+
}
|
|
1719
|
+
if (!hit) { out.push({ ok: false, error: 'option not found: ' + wanted, target: describe(spec) }); continue }
|
|
1720
|
+
} else {
|
|
1721
|
+
let opt = [...el.options].find(o => o.value === wanted)
|
|
1722
|
+
if (!opt) opt = [...el.options].find(o => (o.textContent || '').trim() === wanted)
|
|
1723
|
+
if (!opt) { out.push({ ok: false, error: 'option not found: ' + wanted, target: describe(spec) }); continue }
|
|
1724
|
+
setNative(el, HTMLSelectElement.prototype, opt.value)
|
|
1725
|
+
}
|
|
1726
|
+
el.dispatchEvent(new Event('input', { bubbles: true }))
|
|
1727
|
+
el.dispatchEvent(new Event('change', { bubbles: true }))
|
|
1728
|
+
out.push({ ok: true, method: 'select', target: describe(spec) })
|
|
1729
|
+
} else if (type === 'file') {
|
|
1730
|
+
out.push({ ok: false, error: 'file inputs cannot be set from script; use browser_download or ask the human', target: describe(spec) })
|
|
1731
|
+
} else if (type === 'checkbox') {
|
|
1732
|
+
const want = spec.value === true || spec.value === 'true' || spec.value === 'on'
|
|
1733
|
+
if (el.checked !== want) el.click()
|
|
1734
|
+
if (el.checked !== want) { out.push({ ok: false, error: 'checkbox did not change (disabled, or a handler prevented it)', target: describe(spec) }); continue }
|
|
1735
|
+
out.push({ ok: true, method: 'checkbox', target: describe(spec) })
|
|
1736
|
+
} else if (type === 'radio') {
|
|
1737
|
+
const wanted = String(spec.value)
|
|
1738
|
+
const radio = [...document.querySelectorAll('input[type="radio"][name=' + JSON.stringify(el.name || '') + ']')]
|
|
1739
|
+
.find(r => r.value === wanted || (r === el && (spec.value === true || spec.value === 'true')))
|
|
1740
|
+
if (!radio) { out.push({ ok: false, error: 'radio option not found: ' + wanted, target: describe(spec) }); continue }
|
|
1741
|
+
if (!radio.checked) radio.click()
|
|
1742
|
+
if (!radio.checked) { out.push({ ok: false, error: 'radio did not change (disabled, or a handler prevented it)', target: describe(spec) }); continue }
|
|
1743
|
+
out.push({ ok: true, method: 'radio', target: describe(spec) })
|
|
1744
|
+
} else if (el.isContentEditable) {
|
|
1745
|
+
el.textContent = String(spec.value)
|
|
1746
|
+
el.dispatchEvent(new Event('input', { bubbles: true }))
|
|
1747
|
+
out.push({ ok: true, method: 'contenteditable', target: describe(spec) })
|
|
1748
|
+
} else if (tag === 'TEXTAREA') {
|
|
1749
|
+
const wanted = String(spec.value)
|
|
1750
|
+
setNative(el, HTMLTextAreaElement.prototype, wanted)
|
|
1751
|
+
el.dispatchEvent(new Event('input', { bubbles: true }))
|
|
1752
|
+
el.dispatchEvent(new Event('change', { bubbles: true }))
|
|
1753
|
+
if (wanted !== '' && el.value === '') { out.push({ ok: false, error: 'the field rejected the value', target: describe(spec) }); continue }
|
|
1754
|
+
out.push({ ok: true, method: 'textarea', target: describe(spec) })
|
|
1755
|
+
} else {
|
|
1756
|
+
const wanted = String(spec.value)
|
|
1757
|
+
setNative(el, HTMLInputElement.prototype, wanted)
|
|
1758
|
+
el.dispatchEvent(new Event('input', { bubbles: true }))
|
|
1759
|
+
el.dispatchEvent(new Event('change', { bubbles: true }))
|
|
1760
|
+
// A constrained input (number/date/email) silently blanks a value it
|
|
1761
|
+
// will not accept. Only the empty outcome is unambiguous: the DOM may
|
|
1762
|
+
// legitimately normalise a value it did accept.
|
|
1763
|
+
if (wanted !== '' && el.value === '') { out.push({ ok: false, error: 'the field rejected the value', target: describe(spec) }); continue }
|
|
1764
|
+
out.push({ ok: true, method: 'input', target: describe(spec) })
|
|
1765
|
+
}
|
|
1766
|
+
} catch (e) {
|
|
1767
|
+
out.push({ ok: false, error: String(e), target: describe(spec) })
|
|
1768
|
+
}
|
|
1769
|
+
// Remember what actually landed, so submit anchors on a form the caller
|
|
1770
|
+
// really filled rather than the first field the page happens to expose.
|
|
1771
|
+
if (out.length > before && out[out.length - 1].ok === true) filledEls.push(el)
|
|
1772
|
+
}
|
|
1773
|
+
let submitted = false
|
|
1774
|
+
let blockReason = ''
|
|
1775
|
+
if (${submitFlag}) {
|
|
1776
|
+
let anchor = null
|
|
1777
|
+
for (let index = filledEls.length - 1; index >= 0; index--) {
|
|
1778
|
+
const candidate = filledEls[index]
|
|
1779
|
+
if (candidate.form || candidate.closest('form')) { anchor = candidate; break }
|
|
1780
|
+
}
|
|
1781
|
+
const form = anchor === null ? null : (anchor.form || anchor.closest('form'))
|
|
1782
|
+
if (form === null) blockReason = 'no containing form'
|
|
1783
|
+
// requestSubmit() runs constraint validation: on an invalid form it
|
|
1784
|
+
// neither throws nor submits, so reporting submitted:true was a silent
|
|
1785
|
+
// false success.
|
|
1786
|
+
else if (typeof form.checkValidity === 'function' && form.checkValidity() !== true) blockReason = 'the form is invalid'
|
|
1787
|
+
else { form.requestSubmit(); submitted = true }
|
|
1788
|
+
}
|
|
1789
|
+
return { fields: out, submitted, blockReason }
|
|
1790
|
+
})()`
|
|
1791
|
+
const timeoutMs = request.timeoutMs ?? 30_000
|
|
1792
|
+
const result = await withTimeout(
|
|
1793
|
+
handleSendEvaluate(tab.handle, script),
|
|
1794
|
+
timeoutMs,
|
|
1795
|
+
signal,
|
|
1796
|
+
`browser: fillForm timed out after ${timeoutMs}ms`,
|
|
1797
|
+
)
|
|
1798
|
+
if (!result.ok) {
|
|
1799
|
+
throw new BrowserError(`browser: fillForm evaluation failed: ${result.exception}`, 'BROWSER_FILL_FAILED')
|
|
1800
|
+
}
|
|
1801
|
+
const value = result.value as BrowserFillResult & { readonly blockReason?: string }
|
|
1802
|
+
const okCount = value.fields.filter(f => f.ok).length
|
|
1803
|
+
const submitNote = value.submitted
|
|
1804
|
+
? ', form submitted'
|
|
1805
|
+
: value.blockReason !== undefined && value.blockReason !== ''
|
|
1806
|
+
? `, submit blocked: ${value.blockReason}`
|
|
1807
|
+
: ''
|
|
1808
|
+
this.record(s, 'fill', { fields: request.fields.length, submit: submitFlag }, okCount === value.fields.length, {
|
|
1809
|
+
result: `${okCount}/${value.fields.length} fields filled${submitNote}`,
|
|
1810
|
+
})
|
|
1811
|
+
return { fields: value.fields, submitted: value.submitted === true }
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
/**
|
|
1815
|
+
* Download a URL to a local file, keeping the session's cookies/login.
|
|
1816
|
+
* Requires the self-hosted host (which implements view-level download); the
|
|
1817
|
+
* desktop shell's embedded views delegate downloads to the real browser UI.
|
|
1818
|
+
*/
|
|
1819
|
+
async download(session: BrowserSessionId, request: { readonly url: string; readonly savePath: string }, signal?: AbortSignal): Promise<{ readonly path: string }> {
|
|
1820
|
+
const s = this.session(session)
|
|
1821
|
+
const { handle } = this.activeTab(s)
|
|
1822
|
+
signal?.throwIfAborted()
|
|
1823
|
+
const downloadable = handle as { download?(url: string, savePath: string): Promise<void> }
|
|
1824
|
+
if (typeof downloadable.download !== 'function') {
|
|
1825
|
+
throw new BrowserError('browser: download is only available on the self-hosted browser', 'BROWSER_DOWNLOAD_UNSUPPORTED')
|
|
1826
|
+
}
|
|
1827
|
+
// A download reaches the network with the session's cookies, so it passes
|
|
1828
|
+
// the same URL admission as navigation, and may only write inside the
|
|
1829
|
+
// configured roots.
|
|
1830
|
+
this.admitUrl(request.url, 'download')
|
|
1831
|
+
const target = resolveWritePath(request.savePath, this.writeRoots)
|
|
1832
|
+
// The child fetches in-page with awaitPromise; a slow/hung network can
|
|
1833
|
+
// block it well past the tool budget, so bound it like every other call.
|
|
1834
|
+
const timeoutMs = 60_000
|
|
1835
|
+
await withTimeout(
|
|
1836
|
+
downloadable.download(request.url, target),
|
|
1837
|
+
timeoutMs,
|
|
1838
|
+
signal,
|
|
1839
|
+
`browser: download timed out after ${timeoutMs}ms`,
|
|
1840
|
+
)
|
|
1841
|
+
this.record(s, 'download', { url: request.url, savePath: request.savePath }, true, { result: request.savePath })
|
|
1842
|
+
return { path: request.savePath }
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
/**
|
|
1846
|
+
* Export the session's cookies (login state) as serializable objects.
|
|
1847
|
+
* Self-hosted only; the desktop shell's embedded views use the real profile.
|
|
1848
|
+
*/
|
|
1849
|
+
async flushAuth(session: BrowserSessionId): Promise<readonly ExportedCookie[]> {
|
|
1850
|
+
const s = this.session(session)
|
|
1851
|
+
const { handle } = this.activeTab(s)
|
|
1852
|
+
const host = handle as { flushAuth?(): Promise<ExportedCookie[]> }
|
|
1853
|
+
if (typeof host.flushAuth !== 'function') {
|
|
1854
|
+
throw new BrowserError('browser: auth export is only available on the self-hosted browser', 'BROWSER_AUTH_UNSUPPORTED')
|
|
1855
|
+
}
|
|
1856
|
+
const timeoutMs = 30_000
|
|
1857
|
+
const cookies = await withTimeout(host.flushAuth(), timeoutMs, undefined, `browser: auth export timed out after ${timeoutMs}ms`)
|
|
1858
|
+
this.record(s, 'flushAuth', {}, true, { result: `${cookies.length} cookies` })
|
|
1859
|
+
return cookies
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
/**
|
|
1863
|
+
* Remove cookies for one site scope. Challenge cookies that rotate their names
|
|
1864
|
+
* (WAF challenges) otherwise pile up generation after generation, and two live
|
|
1865
|
+
* generations in one request can be rejected by the site. Self-hosted only.
|
|
1866
|
+
*/
|
|
1867
|
+
async clearAuth(session: BrowserSessionId, request: BrowserClearAuthRequest): Promise<BrowserClearAuthResult> {
|
|
1868
|
+
const s = this.session(session)
|
|
1869
|
+
const { handle } = this.activeTab(s)
|
|
1870
|
+
const clearable = handle as { clearCookies?(filter: BrowserClearAuthRequest): Promise<{ removed: number; names: readonly string[] }> }
|
|
1871
|
+
if (typeof clearable.clearCookies !== 'function') {
|
|
1872
|
+
throw new BrowserError('browser: cookie clearing is only available on the self-hosted browser', 'BROWSER_AUTH_UNSUPPORTED')
|
|
1873
|
+
}
|
|
1874
|
+
const timeoutMs = 30_000
|
|
1875
|
+
const result = await withTimeout(
|
|
1876
|
+
clearable.clearCookies(request),
|
|
1877
|
+
timeoutMs,
|
|
1878
|
+
undefined,
|
|
1879
|
+
'browser: cookie clear timed out after ' + String(timeoutMs) + 'ms',
|
|
1880
|
+
)
|
|
1881
|
+
this.record(s, 'clearAuth', {
|
|
1882
|
+
...request.domain !== undefined ? { domain: request.domain } : {},
|
|
1883
|
+
...request.name !== undefined ? { name: request.name } : {},
|
|
1884
|
+
...request.all === true ? { all: true } : {},
|
|
1885
|
+
}, true, { result: String(result.removed) + ' cookies' })
|
|
1886
|
+
return { removed: result.removed, names: [...result.names] }
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
/**
|
|
1890
|
+
* Import cookies from a JSON export on disk.
|
|
1891
|
+
*
|
|
1892
|
+
* The path is read-guarded exactly like browser_upload_file: a prompt-injected
|
|
1893
|
+
* path must not turn this into a way to read a file the operator never allowed.
|
|
1894
|
+
* A browser cookie export cannot be produced automatically — Chrome and Edge
|
|
1895
|
+
* 127+ encrypt cookie values with App-Bound Encryption, so a copied profile
|
|
1896
|
+
* yields nothing — which is why this takes a file the user exported.
|
|
1897
|
+
*/
|
|
1898
|
+
async importAuth(session: BrowserSessionId, path: string): Promise<{ restored: number; failed: number }> {
|
|
1899
|
+
const s = this.session(session)
|
|
1900
|
+
const target = resolveReadPath(path, this.readRoots)
|
|
1901
|
+
let parsed: unknown
|
|
1902
|
+
try {
|
|
1903
|
+
parsed = JSON.parse(readFileSync(target, 'utf8')) as unknown
|
|
1904
|
+
} catch (error) {
|
|
1905
|
+
throw new BrowserError(`browser: cannot read the cookie file: ${(error as Error).message}`, 'BROWSER_AUTH_FILE_INVALID')
|
|
1906
|
+
}
|
|
1907
|
+
// Accept both a bare array and the {"cookies": [...]} shape editors emit.
|
|
1908
|
+
const list = Array.isArray(parsed)
|
|
1909
|
+
? parsed
|
|
1910
|
+
: typeof parsed === 'object' && parsed !== null && Array.isArray((parsed as { cookies?: unknown }).cookies)
|
|
1911
|
+
? (parsed as { cookies: unknown[] }).cookies
|
|
1912
|
+
: undefined
|
|
1913
|
+
if (list === undefined) {
|
|
1914
|
+
throw new BrowserError('browser: the cookie file must be a JSON array or {"cookies": [...]}', 'BROWSER_AUTH_FILE_INVALID')
|
|
1915
|
+
}
|
|
1916
|
+
const usable = list.map(normalizeExportedCookie).filter((cookie): cookie is ExportedCookie => cookie !== undefined)
|
|
1917
|
+
if (usable.length === 0) {
|
|
1918
|
+
throw new BrowserError(`browser: the cookie file has no usable entries (${list.length} read)`, 'BROWSER_AUTH_FILE_INVALID')
|
|
1919
|
+
}
|
|
1920
|
+
const restored = await this.restoreAuth(session, usable)
|
|
1921
|
+
this.record(s, 'importAuth', { count: usable.length }, true, { result: `${restored} cookies` })
|
|
1922
|
+
return { restored, failed: list.length - usable.length }
|
|
1923
|
+
}
|
|
1924
|
+
|
|
1925
|
+
/** Import cookies into the session (restore login state). Self-hosted only. */
|
|
1926
|
+
async restoreAuth(session: BrowserSessionId, cookies: readonly ExportedCookie[]): Promise<number> {
|
|
1927
|
+
const s = this.session(session)
|
|
1928
|
+
const { handle } = this.activeTab(s)
|
|
1929
|
+
const host = handle as { restoreAuth?(cookies: readonly ExportedCookie[]): Promise<number> }
|
|
1930
|
+
if (typeof host.restoreAuth !== 'function') {
|
|
1931
|
+
throw new BrowserError('browser: auth restore is only available on the self-hosted browser', 'BROWSER_AUTH_UNSUPPORTED')
|
|
1932
|
+
}
|
|
1933
|
+
const timeoutMs = 30_000
|
|
1934
|
+
const restored = await withTimeout(host.restoreAuth(cookies), timeoutMs, undefined, `browser: auth restore timed out after ${timeoutMs}ms`)
|
|
1935
|
+
this.record(s, 'restoreAuth', { count: cookies.length }, true, { result: `${restored} cookies` })
|
|
1936
|
+
return restored
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1939
|
+
/**
|
|
1940
|
+
* Start a background scrape batch.
|
|
1941
|
+
*
|
|
1942
|
+
* It runs detached on purpose: one tool call has a ~60s budget while a large
|
|
1943
|
+
* batch takes minutes. Progress is polled with scrapeStatus, and each row is
|
|
1944
|
+
* appended the moment it is produced, so a stopped or interrupted batch keeps
|
|
1945
|
+
* everything it managed. `outPath` is write-guarded like any other browser
|
|
1946
|
+
* write, and truncated up front so a re-run never mixes two batches.
|
|
1947
|
+
*/
|
|
1948
|
+
async startScrape(session: BrowserSessionId, request: BrowserScrapeRequest): Promise<BrowserScrapeStatus> {
|
|
1949
|
+
const s = this.session(session)
|
|
1950
|
+
const urls = request.urls.filter(url => typeof url === 'string' && url.trim() !== '')
|
|
1951
|
+
if (urls.length === 0) {
|
|
1952
|
+
throw new BrowserError('browser: scrape needs at least one URL', 'BROWSER_SCRAPE_EMPTY')
|
|
1953
|
+
}
|
|
1954
|
+
if (typeof request.script !== 'string' || request.script.trim() === '') {
|
|
1955
|
+
throw new BrowserError('browser: scrape needs an extraction script', 'BROWSER_SCRAPE_EMPTY')
|
|
1956
|
+
}
|
|
1957
|
+
const target = resolveWritePath(request.outPath, this.writeRoots)
|
|
1958
|
+
writeFileSync(target, '')
|
|
1959
|
+
// Created only after every guard has passed, so a rejected start leaves no
|
|
1960
|
+
// orphaned view behind.
|
|
1961
|
+
const workers = Math.max(1, Math.min(Math.trunc(request.concurrency ?? 1) || 1, MAX_SCRAPE_WORKERS))
|
|
1962
|
+
const tabs: Tab[] = []
|
|
1963
|
+
for (let i = 0; i < workers; i += 1) {
|
|
1964
|
+
const handle = this.host.createView(s.taskKey, s.taskLabel === '' ? undefined : s.taskLabel)
|
|
1965
|
+
const tab = this.createTab(handle)
|
|
1966
|
+
s.tabs.push(tab)
|
|
1967
|
+
tabs.push(tab)
|
|
1968
|
+
}
|
|
1969
|
+
const job: ScrapeJob = {
|
|
1970
|
+
id: `scrape:${randomUUID()}`,
|
|
1971
|
+
session: s,
|
|
1972
|
+
tabs,
|
|
1973
|
+
path: target,
|
|
1974
|
+
state: 'running',
|
|
1975
|
+
total: urls.length,
|
|
1976
|
+
done: 0,
|
|
1977
|
+
failed: 0,
|
|
1978
|
+
}
|
|
1979
|
+
this.scrapes.set(job.id, job)
|
|
1980
|
+
void this.runScrape(job, urls, request).catch(error => {
|
|
1981
|
+
job.state = 'done'
|
|
1982
|
+
job.error = String((error as Error)?.message ?? error)
|
|
1983
|
+
})
|
|
1984
|
+
this.record(s, 'scrape', { total: urls.length }, true, { result: `${urls.length} urls` })
|
|
1985
|
+
return scrapeStatusOf(job)
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
/** Progress of one batch. */
|
|
1989
|
+
async scrapeStatus(id: string): Promise<BrowserScrapeStatus> {
|
|
1990
|
+
return scrapeStatusOf(this.scrapeJob(id))
|
|
1991
|
+
}
|
|
1992
|
+
|
|
1993
|
+
/** Ask a running batch to stop; rows already written stay. */
|
|
1994
|
+
async stopScrape(id: string): Promise<BrowserScrapeStatus> {
|
|
1995
|
+
const job = this.scrapeJob(id)
|
|
1996
|
+
if (job.state === 'running') job.state = 'stopped'
|
|
1997
|
+
return scrapeStatusOf(job)
|
|
1998
|
+
}
|
|
1999
|
+
|
|
2000
|
+
/** Every batch this process knows about, oldest first. */
|
|
2001
|
+
async listScrapes(): Promise<readonly BrowserScrapeStatus[]> {
|
|
2002
|
+
return [...this.scrapes.values()].map(scrapeStatusOf)
|
|
2003
|
+
}
|
|
2004
|
+
|
|
2005
|
+
private scrapeJob(id: string): ScrapeJob {
|
|
2006
|
+
const job = this.scrapes.get(id)
|
|
2007
|
+
if (job === undefined) {
|
|
2008
|
+
throw new BrowserError(`browser: unknown scrape ${id}`, 'BROWSER_SCRAPE_UNKNOWN')
|
|
2009
|
+
}
|
|
2010
|
+
return job
|
|
2011
|
+
}
|
|
2012
|
+
|
|
2013
|
+
/**
|
|
2014
|
+
* Visit each URL once, appending one JSONL row per page.
|
|
2015
|
+
*
|
|
2016
|
+
* Workers pull from one shared index, so `concurrency` sets the throughput
|
|
2017
|
+
* without changing the work. Rows therefore land in completion order; each row
|
|
2018
|
+
* carries its URL's index as `seq` so the caller can restore the original.
|
|
2019
|
+
*/
|
|
2020
|
+
private async runScrape(job: ScrapeJob, urls: readonly string[], request: BrowserScrapeRequest): Promise<void> {
|
|
2021
|
+
const perUrl = request.timeoutMs ?? 30_000
|
|
2022
|
+
let next = 0
|
|
2023
|
+
const take = (): number | undefined => (next < urls.length ? next++ : undefined)
|
|
2024
|
+
|
|
2025
|
+
const worker = async (tab: Tab): Promise<void> => {
|
|
2026
|
+
for (;;) {
|
|
2027
|
+
if (job.state !== 'running') return
|
|
2028
|
+
const index = take()
|
|
2029
|
+
if (index === undefined) return
|
|
2030
|
+
const url = urls[index] ?? ''
|
|
2031
|
+
let row: Record<string, unknown>
|
|
2032
|
+
try {
|
|
2033
|
+
// show: false — the batch works in the background tab it owns.
|
|
2034
|
+
// settleMs: 0 — it reads the DOM, so it must not pay the paint delay.
|
|
2035
|
+
await this.navigateTab(job.session, tab, url, undefined, false, 0)
|
|
2036
|
+
if (request.waitFor !== undefined) {
|
|
2037
|
+
await this.waitForElementTab(job.session, tab, { selector: request.waitFor, timeoutMs: perUrl })
|
|
2038
|
+
}
|
|
2039
|
+
const result = await this.executeTab(job.session, tab, { script: request.script, timeoutMs: perUrl })
|
|
2040
|
+
if (result.ok) {
|
|
2041
|
+
row = { seq: index, url, ok: true, data: result.value }
|
|
2042
|
+
} else {
|
|
2043
|
+
job.failed += 1
|
|
2044
|
+
row = { seq: index, url, ok: false, error: result.exception }
|
|
2045
|
+
}
|
|
2046
|
+
} catch (error) {
|
|
2047
|
+
// One bad page must not end the batch: record it and keep going.
|
|
2048
|
+
job.failed += 1
|
|
2049
|
+
row = { seq: index, url, ok: false, error: String((error as Error)?.message ?? error) }
|
|
2050
|
+
}
|
|
2051
|
+
// appendFileSync blocks, so two workers can never interleave a row.
|
|
2052
|
+
appendFileSync(job.path, scrapeRow(row))
|
|
2053
|
+
job.done += 1
|
|
2054
|
+
}
|
|
2055
|
+
}
|
|
2056
|
+
|
|
2057
|
+
try {
|
|
2058
|
+
await Promise.all(job.tabs.map(tab => worker(tab)))
|
|
2059
|
+
job.state = 'done'
|
|
2060
|
+
} finally {
|
|
2061
|
+
// The tabs outlive the loop only until here; a stopped batch cleans up too.
|
|
2062
|
+
this.destroyScrapeTabs(job)
|
|
2063
|
+
}
|
|
2064
|
+
}
|
|
2065
|
+
|
|
2066
|
+
/** Drop a batch's private tabs (and their views) once the batch is over. */
|
|
2067
|
+
private destroyScrapeTabs(job: ScrapeJob): void {
|
|
2068
|
+
const s = job.session
|
|
2069
|
+
for (const tab of job.tabs) {
|
|
2070
|
+
const index = s.tabs.findIndex(candidate => candidate.id === tab.id)
|
|
2071
|
+
if (index < 0) continue
|
|
2072
|
+
s.tabs.splice(index, 1)
|
|
2073
|
+
this.ignoreHostFailure(this.host.destroyView(tab.handle))
|
|
2074
|
+
// Same index bookkeeping as closeTab: a batch tab is normally not active,
|
|
2075
|
+
// but a tool call could have activated one mid-batch.
|
|
2076
|
+
if (s.tabs.length === 0) {
|
|
2077
|
+
this.newTab(s)
|
|
2078
|
+
} else if (index < s.activeIndex) {
|
|
2079
|
+
s.activeIndex -= 1
|
|
2080
|
+
} else if (s.activeIndex >= s.tabs.length) {
|
|
2081
|
+
s.activeIndex = s.tabs.length - 1
|
|
2082
|
+
}
|
|
2083
|
+
}
|
|
2084
|
+
this.showActive(s)
|
|
2085
|
+
}
|
|
2086
|
+
|
|
2087
|
+
/** Capture the current page, optionally full-page. PNG only (CDP JPEG hangs on Electron 43). */
|
|
2088
|
+
async screenshot(
|
|
2089
|
+
session: BrowserSessionId,
|
|
2090
|
+
request?: { readonly fullPage?: boolean; readonly savePath?: string },
|
|
2091
|
+
signal?: AbortSignal,
|
|
2092
|
+
): Promise<{ readonly dataUrl: string; readonly path?: string }> {
|
|
2093
|
+
const s = this.session(session)
|
|
2094
|
+
const { handle } = this.activeTab(s)
|
|
2095
|
+
signal?.throwIfAborted()
|
|
2096
|
+
// Native capturePage path (self-hosted): CDP Page.captureScreenshot can
|
|
2097
|
+
// hang indefinitely on a view once another (hidden) WebContentsView exists
|
|
2098
|
+
// in the shared window; capturePage is fast for the visible task view and resolves
|
|
2099
|
+
// immediately (empty) for hidden ones.
|
|
2100
|
+
const capturable = handle as { capture?(): Promise<{ base64: string; width: number; height: number }> }
|
|
2101
|
+
if (request?.fullPage !== true && typeof capturable.capture === 'function') {
|
|
2102
|
+
// Ensure the target view is the visible one before capturing.
|
|
2103
|
+
this.showActive(s)
|
|
2104
|
+
const timeoutMs = 30_000
|
|
2105
|
+
const shot = await withTimeout(
|
|
2106
|
+
capturable.capture(),
|
|
2107
|
+
timeoutMs,
|
|
2108
|
+
signal,
|
|
2109
|
+
`browser: screenshot timed out after ${timeoutMs}ms`,
|
|
2110
|
+
)
|
|
2111
|
+
if (shot.base64 === '') {
|
|
2112
|
+
throw new BrowserError('browser: capture returned an empty image (view not painted); retry shortly', 'BROWSER_SCREENSHOT_FAILED')
|
|
2113
|
+
}
|
|
2114
|
+
return this.saveScreenshot(shot.base64, request?.savePath)
|
|
2115
|
+
}
|
|
2116
|
+
// Fallback: a desktop-shell handle (no capture()) or a full-page capture
|
|
2117
|
+
// uses CDP; full-page needs `captureBeyondViewport` which capturePage lacks.
|
|
2118
|
+
const params: Record<string, unknown> = {}
|
|
2119
|
+
if (request?.fullPage === true) {
|
|
2120
|
+
// `captureBeyondViewport` captures the full scrollable content; without
|
|
2121
|
+
// a clip this yields the full-page image (CDP default is the viewport).
|
|
2122
|
+
params.captureBeyondViewport = true
|
|
2123
|
+
}
|
|
2124
|
+
const timeoutMs = 30_000
|
|
2125
|
+
const result = await withTimeout(
|
|
2126
|
+
handle.sendCommand(CDP_PAGE_CAPTURE_SCREENSHOT, params),
|
|
2127
|
+
timeoutMs,
|
|
2128
|
+
signal,
|
|
2129
|
+
`browser: screenshot timed out after ${timeoutMs}ms`,
|
|
2130
|
+
)
|
|
2131
|
+
const data = result.data
|
|
2132
|
+
if (typeof data !== 'string') {
|
|
2133
|
+
throw new BrowserError('browser: screenshot returned no image data', 'BROWSER_SCREENSHOT_FAILED')
|
|
2134
|
+
}
|
|
2135
|
+
return this.saveScreenshot(data, request?.savePath)
|
|
2136
|
+
}
|
|
2137
|
+
|
|
2138
|
+
/** Build the data URL and optionally write the PNG to disk. */
|
|
2139
|
+
private saveScreenshot(base64: string, savePath?: string): { dataUrl: string; path?: string } {
|
|
2140
|
+
if (savePath !== undefined) {
|
|
2141
|
+
// Admit before writing so a denied path never touches disk.
|
|
2142
|
+
const target = resolveWritePath(savePath, this.writeRoots)
|
|
2143
|
+
try {
|
|
2144
|
+
writeFileSync(target, Buffer.from(base64, 'base64'))
|
|
2145
|
+
return { dataUrl: `data:image/png;base64,${base64}`, path: savePath }
|
|
2146
|
+
} catch (error) {
|
|
2147
|
+
// Report the write problem but keep the capture usable.
|
|
2148
|
+
throw new BrowserError(`browser: screenshot save to "${savePath}" failed: ${String(error)}`, 'BROWSER_SCREENSHOT_SAVE_FAILED', { cause: error })
|
|
2149
|
+
}
|
|
2150
|
+
}
|
|
2151
|
+
return { dataUrl: `data:image/png;base64,${base64}` }
|
|
2152
|
+
}
|
|
2153
|
+
|
|
2154
|
+
/**
|
|
2155
|
+
* Pick up (and forget) any JS dialog the host auto-accepted, so the
|
|
2156
|
+
* operation trail shows the human/agent what the page asked. Best-effort.
|
|
2157
|
+
*/
|
|
2158
|
+
private async drainDialog(s: Session, handle: ElectronViewHandle): Promise<void> {
|
|
2159
|
+
const drainable = handle as { clearDialog?(): Promise<unknown> }
|
|
2160
|
+
if (typeof drainable.clearDialog !== 'function') return
|
|
2161
|
+
try {
|
|
2162
|
+
const dialog = await drainable.clearDialog()
|
|
2163
|
+
if (dialog !== null && dialog !== undefined) {
|
|
2164
|
+
// Keep it: browser_dialog inspect reports the last one, and drainDialog is the
|
|
2165
|
+
// only place the host hands it over.
|
|
2166
|
+
s.lastDialog = dialog
|
|
2167
|
+
this.record(s, 'dialog', dialog as Record<string, unknown>, true)
|
|
2168
|
+
}
|
|
2169
|
+
} catch {
|
|
2170
|
+
// Dialog supervision is cosmetic; never fail a page operation for it.
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2173
|
+
|
|
2174
|
+
/**
|
|
2175
|
+
* Set how the host answers the next JS dialog, and report the resulting state.
|
|
2176
|
+
*
|
|
2177
|
+
* The policy lives in the host (it answers the CDP event there, where a
|
|
2178
|
+
* round-trip would already be too late), so this is a push, not a pull.
|
|
2179
|
+
*/
|
|
2180
|
+
async setDialogPolicy(session: BrowserSessionId, policy: DialogPolicy): Promise<{ dialog: unknown; policy: DialogPolicy }> {
|
|
2181
|
+
const s = this.session(session)
|
|
2182
|
+
const { handle } = this.activeTab(s)
|
|
2183
|
+
const normalized: DialogPolicy = policy.behavior === 'dismiss'
|
|
2184
|
+
? (policy.promptText === undefined ? { behavior: 'dismiss' } : { behavior: 'dismiss', promptText: policy.promptText })
|
|
2185
|
+
: (policy.promptText === undefined ? { behavior: 'accept' } : { behavior: 'accept', promptText: policy.promptText })
|
|
2186
|
+
const pushable = handle as { setDialogPolicy?(policy: DialogPolicy): Promise<unknown> }
|
|
2187
|
+
if (typeof pushable.setDialogPolicy === 'function') {
|
|
2188
|
+
await pushable.setDialogPolicy(normalized)
|
|
2189
|
+
}
|
|
2190
|
+
s.dialogPolicy = normalized
|
|
2191
|
+
this.record(s, 'dialog-policy', { ...normalized }, true)
|
|
2192
|
+
return { dialog: s.lastDialog ?? null, policy: normalized }
|
|
2193
|
+
}
|
|
2194
|
+
|
|
2195
|
+
/**
|
|
2196
|
+
* Console messages the host captured for the active tab.
|
|
2197
|
+
*
|
|
2198
|
+
* Reading does NOT clear by default: debugging is usually a look-again loop, so
|
|
2199
|
+
* `clear: true` is explicit. The host keeps a bounded ring, so old entries fall
|
|
2200
|
+
* off on their own.
|
|
2201
|
+
*/
|
|
2202
|
+
async consoleMessages(session: BrowserSessionId, options: { limit?: number; level?: string; clear?: boolean } = {}): Promise<{ messages: BrowserConsoleMessage[] }> {
|
|
2203
|
+
const s = this.session(session)
|
|
2204
|
+
const { handle } = this.activeTab(s)
|
|
2205
|
+
const reader = handle as { readConsole?(clear?: boolean): Promise<unknown> }
|
|
2206
|
+
if (typeof reader.readConsole !== 'function') return { messages: [] }
|
|
2207
|
+
const raw = await reader.readConsole(options.clear === true) as { messages?: unknown } | null | undefined
|
|
2208
|
+
const list = Array.isArray(raw?.messages) ? raw.messages as BrowserConsoleMessage[] : []
|
|
2209
|
+
const filtered = options.level === undefined ? list : list.filter(entry => entry.level === options.level)
|
|
2210
|
+
const limit = Math.max(1, Math.min(200, Math.trunc(options.limit ?? 50)))
|
|
2211
|
+
return { messages: filtered.slice(-limit) }
|
|
2212
|
+
}
|
|
2213
|
+
|
|
2214
|
+
/** Network requests the host captured for the active tab (bounded ring). */
|
|
2215
|
+
async networkRequests(session: BrowserSessionId, options: { limit?: number; failedOnly?: boolean; urlContains?: string; clear?: boolean } = {}): Promise<{ requests: BrowserNetworkRequest[] }> {
|
|
2216
|
+
const s = this.session(session)
|
|
2217
|
+
const { handle } = this.activeTab(s)
|
|
2218
|
+
const reader = handle as { readNetwork?(clear?: boolean): Promise<unknown> }
|
|
2219
|
+
if (typeof reader.readNetwork !== 'function') return { requests: [] }
|
|
2220
|
+
const raw = await reader.readNetwork(options.clear === true) as { requests?: unknown } | null | undefined
|
|
2221
|
+
const list = Array.isArray(raw?.requests) ? raw.requests as BrowserNetworkRequest[] : []
|
|
2222
|
+
const needle = (options.urlContains ?? '').toLowerCase()
|
|
2223
|
+
const filtered = list.filter(entry => (options.failedOnly !== true || entry.failed !== undefined)
|
|
2224
|
+
&& (needle === '' || entry.url.toLowerCase().includes(needle)))
|
|
2225
|
+
const limit = Math.max(1, Math.min(200, Math.trunc(options.limit ?? 50)))
|
|
2226
|
+
return { requests: filtered.slice(-limit) }
|
|
2227
|
+
}
|
|
2228
|
+
|
|
2229
|
+
/**
|
|
2230
|
+
* Apply device/viewport/media emulation to the active tab.
|
|
2231
|
+
*
|
|
2232
|
+
* Plain CDP through the existing command path, so no host change was needed.
|
|
2233
|
+
* `clear` undoes all three: metrics, user agent, and emulated media.
|
|
2234
|
+
*/
|
|
2235
|
+
async emulate(session: BrowserSessionId, options: EmulateOptions = {}): Promise<{ applied: string[] }> {
|
|
2236
|
+
const s = this.session(session)
|
|
2237
|
+
const { handle } = this.activeTab(s)
|
|
2238
|
+
const applied: string[] = []
|
|
2239
|
+
if (options.clear === true) {
|
|
2240
|
+
await handle.sendCommand('Emulation.clearDeviceMetricsOverride', {})
|
|
2241
|
+
await handle.sendCommand('Emulation.setUserAgentOverride', { userAgent: '' })
|
|
2242
|
+
await handle.sendCommand('Emulation.setEmulatedMedia', { media: '', features: [] })
|
|
2243
|
+
this.record(s, 'emulate', { clear: true }, true)
|
|
2244
|
+
return { applied: ['cleared'] }
|
|
2245
|
+
}
|
|
2246
|
+
if (options.width !== undefined && options.height !== undefined) {
|
|
2247
|
+
const width = Math.max(1, Math.trunc(options.width))
|
|
2248
|
+
const height = Math.max(1, Math.trunc(options.height))
|
|
2249
|
+
await handle.sendCommand('Emulation.setDeviceMetricsOverride', {
|
|
2250
|
+
width,
|
|
2251
|
+
height,
|
|
2252
|
+
deviceScaleFactor: options.deviceScaleFactor ?? 0,
|
|
2253
|
+
mobile: options.mobile === true,
|
|
2254
|
+
})
|
|
2255
|
+
applied.push('viewport ' + String(width) + 'x' + String(height) + (options.mobile === true ? ' mobile' : ''))
|
|
2256
|
+
}
|
|
2257
|
+
if (options.userAgent !== undefined) {
|
|
2258
|
+
await handle.sendCommand('Emulation.setUserAgentOverride', { userAgent: options.userAgent })
|
|
2259
|
+
applied.push('user-agent')
|
|
2260
|
+
}
|
|
2261
|
+
if (options.colorScheme !== undefined) {
|
|
2262
|
+
await handle.sendCommand('Emulation.setEmulatedMedia', {
|
|
2263
|
+
media: '',
|
|
2264
|
+
features: [{ name: 'prefers-color-scheme', value: options.colorScheme }],
|
|
2265
|
+
})
|
|
2266
|
+
applied.push('color-scheme ' + options.colorScheme)
|
|
2267
|
+
}
|
|
2268
|
+
this.record(s, 'emulate', { ...applied.length === 0 ? { noop: true } : {} }, true)
|
|
2269
|
+
return { applied }
|
|
2270
|
+
}
|
|
2271
|
+
|
|
2272
|
+
/**
|
|
2273
|
+
* Drain any dialog that opened since the last input call, then report the last
|
|
2274
|
+
* one and the current policy.
|
|
2275
|
+
*
|
|
2276
|
+
* `dialogState` alone is not enough for the tool: the host only hands a dialog
|
|
2277
|
+
* over when something drains it, so an inspect that does not drain misses exactly
|
|
2278
|
+
* the dialog the caller just triggered. (Found on the real machine.)
|
|
2279
|
+
*/
|
|
2280
|
+
async inspectDialog(session: BrowserSessionId): Promise<{ dialog: unknown; policy: DialogPolicy }> {
|
|
2281
|
+
const s = this.session(session)
|
|
2282
|
+
const { handle } = this.activeTab(s)
|
|
2283
|
+
await this.drainDialog(s, handle)
|
|
2284
|
+
return { dialog: s.lastDialog ?? null, policy: s.dialogPolicy ?? { behavior: 'accept' } }
|
|
2285
|
+
}
|
|
2286
|
+
|
|
2287
|
+
/** The last JS dialog the host reported, plus the current policy. */
|
|
2288
|
+
dialogState(session: BrowserSessionId): { dialog: unknown; policy: DialogPolicy } {
|
|
2289
|
+
const s = this.session(session)
|
|
2290
|
+
return { dialog: s.lastDialog ?? null, policy: s.dialogPolicy ?? { behavior: 'accept' } }
|
|
2291
|
+
}
|
|
2292
|
+
|
|
2293
|
+
/** Name this browser task (space). */
|
|
2294
|
+
async setSpace(session: BrowserSessionId, label: string): Promise<void> {
|
|
2295
|
+
const s = this.session(session)
|
|
2296
|
+
const { handle } = this.activeTab(s)
|
|
2297
|
+
const labelable = handle as { label?(label: string): Promise<void> }
|
|
2298
|
+
if (typeof labelable.label === 'function') {
|
|
2299
|
+
await labelable.label(label)
|
|
2300
|
+
} else {
|
|
2301
|
+
throw new BrowserError('browser: space naming is only available on the self-hosted browser', 'BROWSER_SPACE_UNSUPPORTED')
|
|
2302
|
+
}
|
|
2303
|
+
s.taskLabel = label
|
|
2304
|
+
this.record(s, 'setSpace', { label }, true)
|
|
2305
|
+
}
|
|
2306
|
+
|
|
2307
|
+
/** List every browser task (space) with its label. */
|
|
2308
|
+
async listSpaces(): Promise<readonly BrowserSpaceInfo[]> {
|
|
2309
|
+
const host = this.host as { listWindows?(): Promise<Array<{ key: string; label: string }>> }
|
|
2310
|
+
if (typeof host.listWindows !== 'function') return []
|
|
2311
|
+
return host.listWindows()
|
|
2312
|
+
}
|
|
2313
|
+
|
|
2314
|
+
/** List browser tasks with live collaboration status. */
|
|
2315
|
+
async listTasks(): Promise<readonly BrowserTaskInfo[]> {
|
|
2316
|
+
if (typeof this.host.listTasks === 'function') {
|
|
2317
|
+
const tasks = await this.host.listTasks()
|
|
2318
|
+
for (const task of tasks) this.rememberHostedTask(task)
|
|
2319
|
+
return tasks
|
|
2320
|
+
}
|
|
2321
|
+
const tasks = new Map<string, BrowserTaskInfo>()
|
|
2322
|
+
for (const session of this.sessions.values()) {
|
|
2323
|
+
const current = tasks.get(session.taskKey)
|
|
2324
|
+
const next = this.localTaskInfo(session)
|
|
2325
|
+
tasks.set(session.taskKey, current === undefined
|
|
2326
|
+
? next
|
|
2327
|
+
: { ...next, tabs: current.tabs + next.tabs, active: current.active || next.active })
|
|
2328
|
+
}
|
|
2329
|
+
return [...tasks.values()]
|
|
2330
|
+
}
|
|
2331
|
+
|
|
2332
|
+
/** Read the collaboration state for one session's task. */
|
|
2333
|
+
async getTask(session: BrowserSessionId): Promise<BrowserTaskInfo> {
|
|
2334
|
+
const s = this.session(session)
|
|
2335
|
+
const hosted = typeof this.host.getTask === 'function' ? await this.host.getTask(s.taskKey) : undefined
|
|
2336
|
+
if (hosted !== undefined) {
|
|
2337
|
+
this.rememberHostedTask(hosted)
|
|
2338
|
+
return hosted
|
|
2339
|
+
}
|
|
2340
|
+
return this.localTaskInfo(s)
|
|
2341
|
+
}
|
|
2342
|
+
|
|
2343
|
+
/** Apply one visible task state update and mirror it to a supporting host. */
|
|
2344
|
+
async updateTask(session: BrowserSessionId, update: BrowserTaskUpdate): Promise<BrowserTaskInfo> {
|
|
2345
|
+
const s = this.session(session)
|
|
2346
|
+
const previous = this.taskStates.get(s.taskKey) ?? { status: 'idle' as const, control: 'agent' as const, updatedAt: Date.now() }
|
|
2347
|
+
const next: LocalTaskState = {
|
|
2348
|
+
...previous,
|
|
2349
|
+
...update.status !== undefined ? { status: update.status } : {},
|
|
2350
|
+
...update.control !== undefined ? { control: update.control } : {},
|
|
2351
|
+
...update.latestAction !== undefined ? { latestAction: update.latestAction } : {},
|
|
2352
|
+
...update.error !== undefined ? { error: update.error.slice(0, 180) } : {},
|
|
2353
|
+
updatedAt: Date.now(),
|
|
2354
|
+
}
|
|
2355
|
+
if (next.status !== 'failed' && update.error === undefined) delete next.error
|
|
2356
|
+
this.taskStates.set(s.taskKey, next)
|
|
2357
|
+
// Only send fields this call intentionally changes. A page-side handoff
|
|
2358
|
+
// can update the host between two Agent operations; replaying a stale
|
|
2359
|
+
// cached control field here would overwrite the newer human choice.
|
|
2360
|
+
const hosted = typeof this.host.updateTask === 'function'
|
|
2361
|
+
? await this.host.updateTask(s.taskKey, {
|
|
2362
|
+
...update.status !== undefined ? { status: update.status } : {},
|
|
2363
|
+
...update.control !== undefined ? { control: update.control } : {},
|
|
2364
|
+
...update.latestAction !== undefined ? { latestAction: update.latestAction } : {},
|
|
2365
|
+
...update.error !== undefined ? { error: update.error.slice(0, 180) } : {},
|
|
2366
|
+
})
|
|
2367
|
+
: undefined
|
|
2368
|
+
if (hosted !== undefined) {
|
|
2369
|
+
this.rememberHostedTask(hosted)
|
|
2370
|
+
return hosted
|
|
2371
|
+
}
|
|
2372
|
+
return this.localTaskInfo(s)
|
|
2373
|
+
}
|
|
2374
|
+
|
|
2375
|
+
/** Hand control to the user or return it to Agent-driven actions. */
|
|
2376
|
+
async setHandoff(session: BrowserSessionId, state: BrowserHandoffState): Promise<BrowserTaskInfo> {
|
|
2377
|
+
return this.updateTask(session, state === 'waiting-user'
|
|
2378
|
+
? { status: 'waiting-user', control: 'human', latestAction: 'waiting for user' }
|
|
2379
|
+
: { status: 'idle', control: 'agent', latestAction: 'agent resumed' })
|
|
2380
|
+
}
|
|
2381
|
+
|
|
2382
|
+
/** Append one operation to the session's history. */
|
|
2383
|
+
private record(
|
|
2384
|
+
s: Session,
|
|
2385
|
+
action: string,
|
|
2386
|
+
params: Record<string, unknown>,
|
|
2387
|
+
ok: boolean,
|
|
2388
|
+
detail?: { result?: string; error?: string },
|
|
2389
|
+
): void {
|
|
2390
|
+
const entry: BrowserHistoryEntry = {
|
|
2391
|
+
seq: s.nextSeq++,
|
|
2392
|
+
action,
|
|
2393
|
+
params: clampHistoryParams(params),
|
|
2394
|
+
ok,
|
|
2395
|
+
...detail?.result !== undefined ? { result: detail.result } : {},
|
|
2396
|
+
...detail?.error !== undefined ? { error: detail.error } : {},
|
|
2397
|
+
at: Date.now(),
|
|
2398
|
+
}
|
|
2399
|
+
s.history.push(entry)
|
|
2400
|
+
// Bound memory: keep the last 500 operations.
|
|
2401
|
+
if (s.history.length > 500) s.history.splice(0, s.history.length - 500)
|
|
2402
|
+
// Mirror onto the human-facing trail in the shared window (best-effort).
|
|
2403
|
+
const tab = s.tabs[s.activeIndex]
|
|
2404
|
+
const state = this.taskStates.get(s.taskKey)
|
|
2405
|
+
if (state !== undefined) {
|
|
2406
|
+
state.latestAction = action
|
|
2407
|
+
state.updatedAt = entry.at
|
|
2408
|
+
}
|
|
2409
|
+
try {
|
|
2410
|
+
this.host.trace?.(tab?.handle.id ?? 'default', { action, params, ok, at: entry.at })
|
|
2411
|
+
const pending = this.host.updateTask?.(s.taskKey, { latestAction: action })
|
|
2412
|
+
void pending?.catch(() => undefined)
|
|
2413
|
+
} catch { /* trail is cosmetic */ }
|
|
2414
|
+
}
|
|
2415
|
+
|
|
2416
|
+
/** Return the session's chronological operation log (newest last). */
|
|
2417
|
+
async history(session: BrowserSessionId): Promise<readonly BrowserHistoryEntry[]> {
|
|
2418
|
+
return this.session(session).history
|
|
2419
|
+
}
|
|
2420
|
+
|
|
2421
|
+
/**
|
|
2422
|
+
* Replay one recorded operation by sequence number. Navigate/click/type are
|
|
2423
|
+
* re-issued against the current page; execute re-runs its script. The
|
|
2424
|
+
* replayed step is appended to history as a new entry.
|
|
2425
|
+
* @param session - the session id.
|
|
2426
|
+
* @param seq - the recorded entry's sequence number to replay.
|
|
2427
|
+
*/
|
|
2428
|
+
async replay(session: BrowserSessionId, seq: number): Promise<void> {
|
|
2429
|
+
const s = this.session(session)
|
|
2430
|
+
const entry = s.history.find(e => e.seq === seq)
|
|
2431
|
+
if (entry === undefined) {
|
|
2432
|
+
throw new BrowserError(`browser: no history entry with seq ${seq}`, 'BROWSER_HISTORY_UNKNOWN')
|
|
2433
|
+
}
|
|
2434
|
+
switch (entry.action) {
|
|
2435
|
+
case 'navigate': {
|
|
2436
|
+
const url = entry.params.url
|
|
2437
|
+
if (typeof url !== 'string') throw new BrowserError(`browser: history seq ${seq} navigate has no url`, 'BROWSER_HISTORY_INVALID')
|
|
2438
|
+
await this.navigate(session, { url })
|
|
2439
|
+
this.record(s, 'replay', { seq, of: entry.action, url }, true)
|
|
2440
|
+
return
|
|
2441
|
+
}
|
|
2442
|
+
case 'click': {
|
|
2443
|
+
const x = entry.params.x
|
|
2444
|
+
const y = entry.params.y
|
|
2445
|
+
if (typeof x !== 'number' || typeof y !== 'number') throw new BrowserError(`browser: history seq ${seq} click has no coordinates`, 'BROWSER_HISTORY_INVALID')
|
|
2446
|
+
await this.click(session, { x, y })
|
|
2447
|
+
this.record(s, 'replay', { seq, of: entry.action, x, y }, true)
|
|
2448
|
+
return
|
|
2449
|
+
}
|
|
2450
|
+
case 'type': {
|
|
2451
|
+
const text = entry.params.text
|
|
2452
|
+
if (typeof text !== 'string') throw new BrowserError(`browser: history seq ${seq} type has no text`, 'BROWSER_HISTORY_INVALID')
|
|
2453
|
+
if (entry.params.textTruncated === true) {
|
|
2454
|
+
throw new BrowserError(`browser: history seq ${seq} text was too long to keep in full; replay is not possible`, 'BROWSER_HISTORY_TRUNCATED')
|
|
2455
|
+
}
|
|
2456
|
+
await this.type(session, { text })
|
|
2457
|
+
this.record(s, 'replay', { seq, of: entry.action, text }, true)
|
|
2458
|
+
return
|
|
2459
|
+
}
|
|
2460
|
+
case 'execute': {
|
|
2461
|
+
const script = entry.params.script
|
|
2462
|
+
if (typeof script !== 'string') throw new BrowserError(`browser: history seq ${seq} execute has no script`, 'BROWSER_HISTORY_INVALID')
|
|
2463
|
+
if (entry.params.scriptTruncated === true) {
|
|
2464
|
+
throw new BrowserError(`browser: history seq ${seq} script was too long to keep in full; replay is not possible`, 'BROWSER_HISTORY_TRUNCATED')
|
|
2465
|
+
}
|
|
2466
|
+
const recordedArgs = entry.params.args
|
|
2467
|
+
const args = Array.isArray(recordedArgs) ? recordedArgs.filter((a): a is string => typeof a === 'string') : undefined
|
|
2468
|
+
const result = await this.execute(session, { script, ...args !== undefined && args.length > 0 ? { args } : {} })
|
|
2469
|
+
this.record(s, 'replay', { seq, of: entry.action, script, ...args !== undefined && args.length > 0 ? { args } : {} }, result.ok, result.ok ? { result: String(result.value) } : { error: result.exception })
|
|
2470
|
+
return
|
|
2471
|
+
}
|
|
2472
|
+
default:
|
|
2473
|
+
throw new BrowserError(`browser: history seq ${seq} action "${entry.action}" is not replayable`, 'BROWSER_HISTORY_NOT_REPLAYABLE')
|
|
2474
|
+
}
|
|
2475
|
+
}
|
|
2476
|
+
|
|
2477
|
+
/** Close the session and destroy all its views. Idempotent. */
|
|
2478
|
+
close(session: BrowserSessionId): Promise<void> {
|
|
2479
|
+
const existing = this.sessions.get(session)
|
|
2480
|
+
if (existing !== undefined) {
|
|
2481
|
+
this.sessions.delete(session)
|
|
2482
|
+
for (const tab of existing.tabs) this.ignoreHostFailure(this.host.destroyView(tab.handle))
|
|
2483
|
+
const replacement = [...this.sessions.values()].find(candidate => candidate.taskKey === existing.taskKey)
|
|
2484
|
+
if (this.sessionsByTask.get(existing.taskKey) === session) {
|
|
2485
|
+
if (replacement === undefined) this.sessionsByTask.delete(existing.taskKey)
|
|
2486
|
+
else this.sessionsByTask.set(existing.taskKey, replacement.id)
|
|
2487
|
+
}
|
|
2488
|
+
if (replacement === undefined) {
|
|
2489
|
+
this.taskStates.delete(existing.taskKey)
|
|
2490
|
+
}
|
|
2491
|
+
}
|
|
2492
|
+
return Promise.resolve()
|
|
2493
|
+
}
|
|
2494
|
+
|
|
2495
|
+
/** Recover the live session associated with a stable task key. */
|
|
2496
|
+
private sessionForTask(taskKey: string): Session | undefined {
|
|
2497
|
+
const indexed = this.sessionsByTask.get(taskKey)
|
|
2498
|
+
if (indexed !== undefined) {
|
|
2499
|
+
const session = this.sessions.get(indexed)
|
|
2500
|
+
if (session !== undefined) return session
|
|
2501
|
+
this.sessionsByTask.delete(taskKey)
|
|
2502
|
+
}
|
|
2503
|
+
for (const session of this.sessions.values()) {
|
|
2504
|
+
if (session.taskKey === taskKey) {
|
|
2505
|
+
this.sessionsByTask.set(taskKey, session.id)
|
|
2506
|
+
return session
|
|
2507
|
+
}
|
|
2508
|
+
}
|
|
2509
|
+
return undefined
|
|
2510
|
+
}
|
|
2511
|
+
|
|
2512
|
+
/** Look up a session or throw the unknown-session error. */
|
|
2513
|
+
private session(session: BrowserSessionId): Session {
|
|
2514
|
+
const existing = this.sessions.get(session)
|
|
2515
|
+
if (existing === undefined) {
|
|
2516
|
+
throw new BrowserError(`browser: session "${session}" is not open`, 'BROWSER_SESSION_UNKNOWN')
|
|
2517
|
+
}
|
|
2518
|
+
return existing
|
|
2519
|
+
}
|
|
2520
|
+
|
|
2521
|
+
/** The active tab of a session. */
|
|
2522
|
+
private activeTab(s: Session): Tab {
|
|
2523
|
+
const tab = s.tabs[s.activeIndex]
|
|
2524
|
+
if (tab === undefined) throw new BrowserError('browser: session has no active tab', 'BROWSER_TAB_UNKNOWN')
|
|
2525
|
+
return tab
|
|
2526
|
+
}
|
|
2527
|
+
|
|
2528
|
+
/** Navigate through the browser history while preserving page readiness behavior. */
|
|
2529
|
+
private async navigateHistory(
|
|
2530
|
+
session: BrowserSessionId,
|
|
2531
|
+
direction: -1 | 1,
|
|
2532
|
+
action: 'back' | 'forward',
|
|
2533
|
+
signal?: AbortSignal,
|
|
2534
|
+
): Promise<boolean> {
|
|
2535
|
+
const s = this.session(session)
|
|
2536
|
+
const tab = this.activeTab(s)
|
|
2537
|
+
signal?.throwIfAborted()
|
|
2538
|
+
const history = await withTimeout(
|
|
2539
|
+
tab.handle.sendCommand(CDP_PAGE_GET_NAVIGATION_HISTORY, {}),
|
|
2540
|
+
15_000,
|
|
2541
|
+
signal,
|
|
2542
|
+
'browser: history lookup timed out',
|
|
2543
|
+
)
|
|
2544
|
+
const currentIndex = typeof history.currentIndex === 'number' ? history.currentIndex : -1
|
|
2545
|
+
const entries = Array.isArray(history.entries) ? history.entries as Array<{ id?: unknown }> : []
|
|
2546
|
+
const target = entries[currentIndex + direction]
|
|
2547
|
+
if (target === undefined || typeof target.id !== 'number') {
|
|
2548
|
+
this.record(s, action, { navigated: false }, true)
|
|
2549
|
+
return false
|
|
2550
|
+
}
|
|
2551
|
+
await withTimeout(
|
|
2552
|
+
tab.handle.sendCommand(CDP_PAGE_NAVIGATE_TO_HISTORY_ENTRY, { entryId: target.id }),
|
|
2553
|
+
30_000,
|
|
2554
|
+
signal,
|
|
2555
|
+
`browser: ${action} timed out after 30000ms`,
|
|
2556
|
+
)
|
|
2557
|
+
this.invalidateSnapshots(tab)
|
|
2558
|
+
this.record(s, action, { navigated: true }, true)
|
|
2559
|
+
this.showActive(s)
|
|
2560
|
+
await waitForDocumentReady(tab.handle, signal)
|
|
2561
|
+
void reinstallPageChrome(tab.handle)
|
|
2562
|
+
return true
|
|
2563
|
+
}
|
|
2564
|
+
|
|
2565
|
+
/** Drop every reference that was captured before a document transition. */
|
|
2566
|
+
private invalidateSnapshots(tab: Tab): void {
|
|
2567
|
+
tab.navigationEpoch += 1
|
|
2568
|
+
tab.snapshots.clear()
|
|
2569
|
+
}
|
|
2570
|
+
|
|
2571
|
+
/** Resolve one exact snapshot reference, rejecting any changed or missing target. */
|
|
2572
|
+
private async resolveSnapshotTarget(
|
|
2573
|
+
tab: Tab,
|
|
2574
|
+
request: BrowserRefRequest,
|
|
2575
|
+
block: 'start' | 'center' | 'end' | 'nearest',
|
|
2576
|
+
signal?: AbortSignal,
|
|
2577
|
+
): Promise<BrowserScrollResult & { readonly x: number; readonly y: number }> {
|
|
2578
|
+
const record = tab.snapshots.get(request.snapshotId)
|
|
2579
|
+
if (record === undefined) {
|
|
2580
|
+
throw new BrowserError(`browser: snapshot "${request.snapshotId}" is not available in this tab`, 'BROWSER_SNAPSHOT_UNKNOWN')
|
|
2581
|
+
}
|
|
2582
|
+
if (record.tabId !== tab.id || record.epoch !== tab.navigationEpoch) {
|
|
2583
|
+
throw new BrowserError(`browser: snapshot "${request.snapshotId}" is stale`, 'BROWSER_SNAPSHOT_STALE')
|
|
2584
|
+
}
|
|
2585
|
+
const target = record.targets.get(request.ref)
|
|
2586
|
+
if (target === undefined) {
|
|
2587
|
+
throw new BrowserError(`browser: snapshot "${request.snapshotId}" has no element ref ${request.ref}`, 'BROWSER_REF_UNKNOWN')
|
|
2588
|
+
}
|
|
2589
|
+
const script = `(() => {
|
|
2590
|
+
if (location.href !== ${JSON.stringify(record.url)}) return { stale: 'url changed' }
|
|
2591
|
+
let el
|
|
2592
|
+
try { el = document.querySelector(${JSON.stringify(target.path)}) } catch { return { stale: 'selector invalid' } }
|
|
2593
|
+
if (!el || el.closest('[data-dsh-browser-chrome]')) return { stale: 'element missing' }
|
|
2594
|
+
const fingerprint = [
|
|
2595
|
+
el.tagName,
|
|
2596
|
+
el.getAttribute('type') || '',
|
|
2597
|
+
el.id || '',
|
|
2598
|
+
el.getAttribute('name') || '',
|
|
2599
|
+
el.getAttribute('aria-label') || '',
|
|
2600
|
+
(el.textContent || el.value || '').toString().replace(/\s+/g, ' ').trim().slice(0, 120),
|
|
2601
|
+
].join('\u001f')
|
|
2602
|
+
if (fingerprint !== ${JSON.stringify(target.fingerprint)}) return { stale: 'element changed' }
|
|
2603
|
+
el.scrollIntoView({ block: ${JSON.stringify(block)}, inline: 'nearest', behavior: 'auto' })
|
|
2604
|
+
const rect = el.getBoundingClientRect()
|
|
2605
|
+
const style = getComputedStyle(el)
|
|
2606
|
+
if (rect.width < 4 || rect.height < 4 || style.visibility === 'hidden' || style.display === 'none') return { stale: 'element hidden' }
|
|
2607
|
+
const root = document.documentElement
|
|
2608
|
+
return {
|
|
2609
|
+
x: Math.round(rect.x + rect.width / 2),
|
|
2610
|
+
y: Math.round(rect.y + rect.height / 2),
|
|
2611
|
+
scrollX: window.scrollX,
|
|
2612
|
+
scrollY: window.scrollY,
|
|
2613
|
+
maxX: Math.max(0, root.scrollWidth - window.innerWidth),
|
|
2614
|
+
maxY: Math.max(0, root.scrollHeight - window.innerHeight),
|
|
2615
|
+
}
|
|
2616
|
+
})()`
|
|
2617
|
+
const result = await withTimeout(
|
|
2618
|
+
handleSendEvaluate(tab.handle, script),
|
|
2619
|
+
15_000,
|
|
2620
|
+
signal,
|
|
2621
|
+
'browser: snapshot reference resolution timed out',
|
|
2622
|
+
)
|
|
2623
|
+
if (!result.ok) {
|
|
2624
|
+
throw new BrowserError(`browser: snapshot reference resolution failed: ${result.exception}`, 'BROWSER_REF_RESOLVE_FAILED')
|
|
2625
|
+
}
|
|
2626
|
+
const value = result.value as { stale?: string; x?: number; y?: number; scrollX?: number; scrollY?: number; maxX?: number; maxY?: number }
|
|
2627
|
+
if (typeof value.stale === 'string'
|
|
2628
|
+
|| typeof value.x !== 'number'
|
|
2629
|
+
|| typeof value.y !== 'number'
|
|
2630
|
+
|| typeof value.scrollX !== 'number'
|
|
2631
|
+
|| typeof value.scrollY !== 'number'
|
|
2632
|
+
|| typeof value.maxX !== 'number'
|
|
2633
|
+
|| typeof value.maxY !== 'number') {
|
|
2634
|
+
throw new BrowserError(`browser: snapshot "${request.snapshotId}" is stale${typeof value.stale === 'string' ? `: ${value.stale}` : ''}`, 'BROWSER_SNAPSHOT_STALE')
|
|
2635
|
+
}
|
|
2636
|
+
return { x: value.x, y: value.y, maxX: value.maxX, maxY: value.maxY }
|
|
2637
|
+
}
|
|
2638
|
+
|
|
2639
|
+
/** Sync provider fallback cache from the host's authoritative workspace state. */
|
|
2640
|
+
private rememberHostedTask(task: BrowserTaskInfo): void {
|
|
2641
|
+
this.taskStates.set(task.key, {
|
|
2642
|
+
status: task.status,
|
|
2643
|
+
control: task.control,
|
|
2644
|
+
...task.latestAction !== undefined ? { latestAction: task.latestAction } : {},
|
|
2645
|
+
...task.error !== undefined ? { error: task.error } : {},
|
|
2646
|
+
updatedAt: task.updatedAt,
|
|
2647
|
+
})
|
|
2648
|
+
}
|
|
2649
|
+
|
|
2650
|
+
/** Build the provider-side task summary when a host has no richer workspace. */
|
|
2651
|
+
private localTaskInfo(s: Session): BrowserTaskInfo {
|
|
2652
|
+
const state = this.taskStates.get(s.taskKey) ?? { status: 'idle' as const, control: 'agent' as const, updatedAt: Date.now() }
|
|
2653
|
+
return {
|
|
2654
|
+
key: s.taskKey,
|
|
2655
|
+
label: s.taskLabel,
|
|
2656
|
+
active: this.sessions.size === 1,
|
|
2657
|
+
tabs: s.tabs.length,
|
|
2658
|
+
status: state.status,
|
|
2659
|
+
control: state.control,
|
|
2660
|
+
...state.latestAction !== undefined ? { latestAction: state.latestAction } : {},
|
|
2661
|
+
updatedAt: state.updatedAt,
|
|
2662
|
+
...state.error !== undefined ? { error: state.error } : {},
|
|
2663
|
+
}
|
|
2664
|
+
}
|
|
2665
|
+
|
|
2666
|
+
/** Create a tab with its short-lived snapshot reference store. */
|
|
2667
|
+
private createTab(handle: ElectronViewHandle): Tab {
|
|
2668
|
+
return { id: `tab:${randomUUID()}`, handle, navigationEpoch: 0, snapshots: new Map() }
|
|
2669
|
+
}
|
|
2670
|
+
|
|
2671
|
+
/** Append a fresh tab and make it active. */
|
|
2672
|
+
private newTab(s: Session): void {
|
|
2673
|
+
const handle = this.host.createView(s.taskKey, s.taskLabel === '' ? undefined : s.taskLabel)
|
|
2674
|
+
s.tabs.push(this.createTab(handle))
|
|
2675
|
+
s.activeIndex = s.tabs.length - 1
|
|
2676
|
+
this.showActive(s)
|
|
2677
|
+
}
|
|
2678
|
+
|
|
2679
|
+
/** Notify the host of the active tab; it preserves the human-selected task view. */
|
|
2680
|
+
/**
|
|
2681
|
+
* Fire-and-forget host call: these run while the provider keeps going, so a rejection
|
|
2682
|
+
* must never escape. Electron's host answers `unknown view` for a handle it no longer
|
|
2683
|
+
* knows (a host restart leaves the provider holding stale ones), and an unhandled
|
|
2684
|
+
* rejection surfaces as *some other* tool call failing - Ctrl+W did exactly that.
|
|
2685
|
+
* The desired end state (view gone) holds either way, so swallowing is right here.
|
|
2686
|
+
*/
|
|
2687
|
+
private ignoreHostFailure(promise: unknown): void {
|
|
2688
|
+
void Promise.resolve(promise).catch(() => undefined)
|
|
2689
|
+
}
|
|
2690
|
+
private showActive(s: Session): void {
|
|
2691
|
+
// 让陈旧的一次显示安静地失败:紧接着的操作/切换会把它纠正回来,而一条没人接的
|
|
2692
|
+
// rejection 会以「别的工具调用失败了」的形式冒出来(实测 Ctrl+W 关最后一个标签就是这样)。
|
|
2693
|
+
void Promise.resolve(this.host.showView?.(this.activeTab(s).handle)).catch(() => undefined)
|
|
2694
|
+
}
|
|
2695
|
+
|
|
2696
|
+
/** Read the current URL of a view through CDP. */
|
|
2697
|
+
private async currentUrl(handle: ElectronViewHandle): Promise<string> {
|
|
2698
|
+
// Bound the read: a wedged renderer would otherwise hang listTabs.
|
|
2699
|
+
const timeoutMs = 10_000
|
|
2700
|
+
const result = await withTimeout(
|
|
2701
|
+
handleSendEvaluate(handle, 'location.href'),
|
|
2702
|
+
timeoutMs,
|
|
2703
|
+
undefined,
|
|
2704
|
+
`browser: url read timed out after ${timeoutMs}ms`,
|
|
2705
|
+
)
|
|
2706
|
+
return result.ok && typeof result.value === 'string' ? result.value : ''
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
|
|
2710
|
+
/**
|
|
2711
|
+
* Bound a promise so a wedged CDP call surfaces as an error instead of
|
|
2712
|
+
* hanging the tool call forever. The caller's signal, when provided, wins
|
|
2713
|
+
* over the timeout if it fires first.
|
|
2714
|
+
* @param promise - the operation to bound.
|
|
2715
|
+
* @param ms - the timeout budget.
|
|
2716
|
+
* @param signal - optional caller signal.
|
|
2717
|
+
* @param message - the timeout error message.
|
|
2718
|
+
* @returns the promise's value, or a rejected promise on timeout/abort.
|
|
2719
|
+
*/
|
|
2720
|
+
function withTimeout<T>(
|
|
2721
|
+
promise: Promise<T>,
|
|
2722
|
+
ms: number,
|
|
2723
|
+
signal: AbortSignal | undefined,
|
|
2724
|
+
message: string,
|
|
2725
|
+
): Promise<T> {
|
|
2726
|
+
return new Promise<T>((resolve, reject) => {
|
|
2727
|
+
let done = false
|
|
2728
|
+
const timer = setTimeout(() => {
|
|
2729
|
+
if (done) return
|
|
2730
|
+
done = true
|
|
2731
|
+
// A fired timeout must also release the abort listener; { once: true }
|
|
2732
|
+
// only releases it on the next abort, which may never come.
|
|
2733
|
+
if (signal !== undefined) signal.removeEventListener('abort', onAbort)
|
|
2734
|
+
// A stable code lets callers branch on a timeout; the name is preserved for
|
|
2735
|
+
// the one call site that already matched on it.
|
|
2736
|
+
const error = new BrowserError(message, 'BROWSER_OPERATION_TIMEOUT')
|
|
2737
|
+
error.name = 'TimeoutError'
|
|
2738
|
+
reject(error)
|
|
2739
|
+
}, ms)
|
|
2740
|
+
const finish = (fn: () => void): void => {
|
|
2741
|
+
if (done) return
|
|
2742
|
+
done = true
|
|
2743
|
+
clearTimeout(timer)
|
|
2744
|
+
if (signal !== undefined) signal.removeEventListener('abort', onAbort)
|
|
2745
|
+
fn()
|
|
2746
|
+
}
|
|
2747
|
+
const onAbort = (): void => {
|
|
2748
|
+
if (done) return
|
|
2749
|
+
done = true
|
|
2750
|
+
clearTimeout(timer)
|
|
2751
|
+
reject(signal?.reason instanceof Error ? signal.reason : new Error('aborted'))
|
|
2752
|
+
}
|
|
2753
|
+
if (signal !== undefined) signal.addEventListener('abort', onAbort, { once: true })
|
|
2754
|
+
promise.then(
|
|
2755
|
+
value => finish(() => resolve(value)),
|
|
2756
|
+
error => finish(() => reject(error)),
|
|
2757
|
+
)
|
|
2758
|
+
})
|
|
2759
|
+
}
|
|
2760
|
+
|
|
2761
|
+
/**
|
|
2762
|
+
* Run a `Runtime.evaluate` through a view handle and normalize the result.
|
|
2763
|
+
* Shared by execute, snapshot, content, and internal URL reads.
|
|
2764
|
+
* @param handle - the view handle to evaluate in.
|
|
2765
|
+
* @param expression - the JS expression.
|
|
2766
|
+
* @param signal - optional abort signal; a fired signal rejects the call.
|
|
2767
|
+
*/
|
|
2768
|
+
/** Best-effort injection of the human chrome into the current document. */
|
|
2769
|
+
async function reinstallPageChrome(handle: ElectronViewHandle): Promise<void> {
|
|
2770
|
+
// Prefer the host's own injection: only it holds the per-view binding token, so
|
|
2771
|
+
// its copy can still authenticate actions the human triggers. The tokenless
|
|
2772
|
+
// script below is a fallback for hosts that do not own the chrome.
|
|
2773
|
+
if (typeof handle.reinstallChrome === 'function') {
|
|
2774
|
+
try {
|
|
2775
|
+
await handle.reinstallChrome()
|
|
2776
|
+
return
|
|
2777
|
+
} catch {
|
|
2778
|
+
// Fall through rather than leaving the document without any chrome.
|
|
2779
|
+
}
|
|
2780
|
+
}
|
|
2781
|
+
try {
|
|
2782
|
+
await handle.sendCommand(CDP_RUNTIME_EVALUATE, {
|
|
2783
|
+
expression: PAGE_CHROME_SCRIPT,
|
|
2784
|
+
returnByValue: true,
|
|
2785
|
+
} satisfies CdpEvaluateParams)
|
|
2786
|
+
} catch {
|
|
2787
|
+
// Chrome is cosmetic; never fail navigation for it.
|
|
2788
|
+
}
|
|
2789
|
+
}
|
|
2790
|
+
|
|
2791
|
+
/**
|
|
2792
|
+
* The in-page half of {@link resolvePointerTarget}: resolve the element, scroll
|
|
2793
|
+
* it into view, and return its centre.
|
|
2794
|
+
*
|
|
2795
|
+
* Exported so it can be exercised. The matching rule — the innermost visible
|
|
2796
|
+
* element whose label contains the text wins — is the part most likely to be
|
|
2797
|
+
* wrong, and Node has no DOM to check it against.
|
|
2798
|
+
*/
|
|
2799
|
+
export function pointerTargetScript(selector: string | undefined, text: string | undefined): string {
|
|
2800
|
+
return `(() => {
|
|
2801
|
+
const selector = ${JSON.stringify(selector ?? null)}
|
|
2802
|
+
const needle = ${JSON.stringify(text ?? null)}
|
|
2803
|
+
const visible = (el) => {
|
|
2804
|
+
const r = el.getBoundingClientRect()
|
|
2805
|
+
const cs = getComputedStyle(el)
|
|
2806
|
+
return r.width >= 4 && r.height >= 4 && cs.visibility !== 'hidden' && cs.display !== 'none'
|
|
2807
|
+
}
|
|
2808
|
+
const usable = (el) => !el.closest('[data-dsh-browser-chrome]') && visible(el)
|
|
2809
|
+
const labelOf = (el) => (el.getAttribute('aria-label') || el.textContent || el.value || '').toString().replace(/\\s+/g, ' ').trim()
|
|
2810
|
+
const describe = (el) => labelOf(el).slice(0, 80)
|
|
2811
|
+
let el = null
|
|
2812
|
+
if (selector !== null) {
|
|
2813
|
+
let matches
|
|
2814
|
+
try { matches = [...document.querySelectorAll(selector)] } catch (e) { return { error: 'invalid selector: ' + String(e) } }
|
|
2815
|
+
el = matches.find(usable) ?? null
|
|
2816
|
+
} else {
|
|
2817
|
+
const lower = needle.toLowerCase()
|
|
2818
|
+
// One bottom-up pass gives every element its own text. Matching a list of
|
|
2819
|
+
// tag names is not enough: plenty of text lives in tags nobody thinks to
|
|
2820
|
+
// list (p, h1, dd, figcaption, legend, option...), and a site that splits a
|
|
2821
|
+
// label into one span per character leaves every leaf holding a single
|
|
2822
|
+
// character while its container holds the whole phrase. Reading
|
|
2823
|
+
// textContent per element instead would re-walk each subtree.
|
|
2824
|
+
const texts = new Map()
|
|
2825
|
+
const walk = (node) => {
|
|
2826
|
+
let text = ''
|
|
2827
|
+
for (const child of node.childNodes) {
|
|
2828
|
+
if (child.nodeType === 3) text += child.nodeValue ?? ''
|
|
2829
|
+
else if (child.nodeType === 1) text += walk(child)
|
|
2830
|
+
}
|
|
2831
|
+
if (node.nodeType === 1) texts.set(node, text)
|
|
2832
|
+
return text
|
|
2833
|
+
}
|
|
2834
|
+
if (document.body !== null) walk(document.body)
|
|
2835
|
+
let best = null
|
|
2836
|
+
for (const [candidate, own] of texts) {
|
|
2837
|
+
// Cheapest rejection first: a big page has far more elements than matches.
|
|
2838
|
+
const label = (candidate.getAttribute('aria-label') || own || candidate.value || '').replace(/\\s+/g, ' ').trim()
|
|
2839
|
+
if (label === '' || !label.toLowerCase().includes(lower)) continue
|
|
2840
|
+
if (!usable(candidate)) continue
|
|
2841
|
+
let depth = 0
|
|
2842
|
+
for (let node = candidate; node !== null; node = node.parentElement) depth += 1
|
|
2843
|
+
// The shortest label is the innermost element still containing the text;
|
|
2844
|
+
// on a tie the deeper one wins, so a <button> beats its wrapper.
|
|
2845
|
+
if (best === null || label.length < best.label.length || (label.length === best.label.length && depth > best.depth)) {
|
|
2846
|
+
best = { el: candidate, label, depth }
|
|
2847
|
+
}
|
|
2848
|
+
}
|
|
2849
|
+
el = best?.el ?? null
|
|
2850
|
+
}
|
|
2851
|
+
if (el === null) return { missing: true }
|
|
2852
|
+
el.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'auto' })
|
|
2853
|
+
const r = el.getBoundingClientRect()
|
|
2854
|
+
if (r.width < 4 || r.height < 4) return { missing: true }
|
|
2855
|
+
return { x: r.left + r.width / 2, y: r.top + r.height / 2, target: describe(el) }
|
|
2856
|
+
})()`
|
|
2857
|
+
}
|
|
2858
|
+
/**
|
|
2859
|
+
* Resolve a pointer target to a viewport point.
|
|
2860
|
+
*
|
|
2861
|
+
* Coordinates pass straight through. A selector or text is resolved inside the
|
|
2862
|
+
* page and scrolled into view first, and the element is described in the result
|
|
2863
|
+
* so the caller can confirm what it actually hit — a bare coordinate click
|
|
2864
|
+
* cannot tell you that.
|
|
2865
|
+
*/
|
|
2866
|
+
async function resolvePointerTarget(
|
|
2867
|
+
handle: ElectronViewHandle,
|
|
2868
|
+
target: BrowserPointerTarget,
|
|
2869
|
+
signal?: AbortSignal,
|
|
2870
|
+
): Promise<BrowserPointerResult> {
|
|
2871
|
+
if (target.selector === undefined && target.text === undefined) {
|
|
2872
|
+
if (typeof target.x !== 'number' || typeof target.y !== 'number') {
|
|
2873
|
+
throw new BrowserError('browser: a pointer action needs x and y, a selector, or text', 'BROWSER_TARGET_MISSING')
|
|
2874
|
+
}
|
|
2875
|
+
// Coordinates are NOT scrolled into view (only selector/text are), so a point
|
|
2876
|
+
// below the fold is dropped by the renderer and the call looks like it worked.
|
|
2877
|
+
// Measured on a real page: the same centre that hits at 100% is off-screen at
|
|
2878
|
+
// 110% because the page reflows taller. Refuse, and say what is actually there.
|
|
2879
|
+
const view = await withTimeout(
|
|
2880
|
+
handleSendEvaluate(handle, `(function () {
|
|
2881
|
+
var x = ${JSON.stringify(target.x)}, y = ${JSON.stringify(target.y)};
|
|
2882
|
+
var el = document.elementFromPoint(x, y);
|
|
2883
|
+
return { iw: window.innerWidth, ih: window.innerHeight,
|
|
2884
|
+
hit: el === null ? '' : (el.tagName + (el.id ? '#' + el.id : '') + (el.textContent ? ' ' + el.textContent.replace(/\\s+/g, ' ').trim().slice(0, 40) : '')) };
|
|
2885
|
+
})()`, signal),
|
|
2886
|
+
5_000,
|
|
2887
|
+
signal,
|
|
2888
|
+
'browser: coordinate probe timed out after 5000ms',
|
|
2889
|
+
)
|
|
2890
|
+
const info = view.ok ? view.value as { iw?: number; ih?: number; hit?: string } | null : null
|
|
2891
|
+
// A 0x0 viewport means "not laid out yet" (hidden view), not "the point is off
|
|
2892
|
+
// screen": clicks still land there (the smoke's input checks prove it), so only a
|
|
2893
|
+
// real, non-zero viewport may refuse.
|
|
2894
|
+
if (info !== null && typeof info?.iw === 'number' && typeof info?.ih === 'number' && info.iw > 0 && info.ih > 0) {
|
|
2895
|
+
if (target.x >= info.iw || target.y >= info.ih || target.x < 0 || target.y < 0) {
|
|
2896
|
+
throw new BrowserError(
|
|
2897
|
+
`browser: (${target.x}, ${target.y}) is outside the visible viewport (${info.iw}x${info.ih}) - a coordinate click does not scroll, so nothing would be clicked; scroll it into view first, or address the element with a selector/text (those scroll automatically)`,
|
|
2898
|
+
'BROWSER_TARGET_OFFSCREEN',
|
|
2899
|
+
)
|
|
2900
|
+
}
|
|
2901
|
+
}
|
|
2902
|
+
return {
|
|
2903
|
+
x: target.x,
|
|
2904
|
+
y: target.y,
|
|
2905
|
+
...info?.hit !== undefined && info.hit !== '' ? { target: info.hit } : {},
|
|
2906
|
+
}
|
|
2907
|
+
}
|
|
2908
|
+
const script = pointerTargetScript(target.selector, target.text)
|
|
2909
|
+
const result = await withTimeout(
|
|
2910
|
+
handleSendEvaluate(handle, script, signal),
|
|
2911
|
+
10_000,
|
|
2912
|
+
signal,
|
|
2913
|
+
'browser: target resolution timed out after 10000ms',
|
|
2914
|
+
)
|
|
2915
|
+
if (!result.ok) {
|
|
2916
|
+
throw new BrowserError(`browser: could not resolve the target: ${result.exception}`, 'BROWSER_TARGET_FAILED')
|
|
2917
|
+
}
|
|
2918
|
+
const value = result.value as { x?: number; y?: number; target?: string; missing?: boolean; error?: string } | null
|
|
2919
|
+
if (value?.error !== undefined) {
|
|
2920
|
+
throw new BrowserError(`browser: ${value.error}`, 'BROWSER_TARGET_FAILED')
|
|
2921
|
+
}
|
|
2922
|
+
if (value?.missing === true || typeof value?.x !== 'number' || typeof value?.y !== 'number') {
|
|
2923
|
+
const what = target.selector !== undefined ? `selector "${target.selector}"` : `text "${target.text ?? ''}"`
|
|
2924
|
+
throw new BrowserError(`browser: no visible element matches ${what}`, 'BROWSER_TARGET_NOT_FOUND')
|
|
2925
|
+
}
|
|
2926
|
+
return { x: value.x, y: value.y, ...value.target === undefined ? {} : { target: value.target } }
|
|
2927
|
+
}
|
|
2928
|
+
|
|
2929
|
+
/**
|
|
2930
|
+
* Wait until the main document reports complete, without turning an otherwise
|
|
2931
|
+
* successful navigation into a failure when a page is slow or never settles.
|
|
2932
|
+
*/
|
|
2933
|
+
async function waitForDocumentReady(
|
|
2934
|
+
handle: ElectronViewHandle,
|
|
2935
|
+
signal?: AbortSignal,
|
|
2936
|
+
settleMs = 250,
|
|
2937
|
+
): Promise<void> {
|
|
2938
|
+
const deadline = Date.now() + 12_000
|
|
2939
|
+
try {
|
|
2940
|
+
await withTimeout((async () => {
|
|
2941
|
+
while (Date.now() <= deadline) {
|
|
2942
|
+
const result = await handleSendEvaluate(handle, 'document.readyState', signal).catch(() => undefined)
|
|
2943
|
+
if (result?.ok && result.value === 'complete') {
|
|
2944
|
+
// The settle delay exists so a screenshot or snapshot does not catch a
|
|
2945
|
+
// still-blank renderer. A scrape reads the DOM, not pixels, so it
|
|
2946
|
+
// passes 0 — otherwise a thousand-page batch would idle 250s.
|
|
2947
|
+
if (settleMs > 0) await new Promise(resolve => setTimeout(resolve, settleMs))
|
|
2948
|
+
return
|
|
2949
|
+
}
|
|
2950
|
+
await new Promise(resolve => setTimeout(resolve, 150))
|
|
2951
|
+
}
|
|
2952
|
+
})(), 13_000, signal, 'readiness wait exceeded')
|
|
2953
|
+
} catch {
|
|
2954
|
+
// Readiness is an optimization: never fail a valid navigation for it.
|
|
2955
|
+
}
|
|
2956
|
+
}
|
|
2957
|
+
|
|
2958
|
+
/** Mark a short window in which CDP input must not transfer control to the user. */
|
|
2959
|
+
async function suppressAutoUserControl(handle: ElectronViewHandle, signal?: AbortSignal): Promise<void> {
|
|
2960
|
+
const expression = '(() => { const host = document.getElementById(' + JSON.stringify(PAGE_CHROME_HOST_ID)
|
|
2961
|
+
+ '); if (!host) return false; host.setAttribute("data-dsh-agent-input-until", String(Date.now() + ' + String(AGENT_INPUT_SUPPRESSION_MS) + ')); return true })()'
|
|
2962
|
+
await withTimeout(
|
|
2963
|
+
handleSendEvaluate(handle, expression, signal),
|
|
2964
|
+
2_000,
|
|
2965
|
+
signal,
|
|
2966
|
+
'browser: agent input suppression timed out',
|
|
2967
|
+
).catch(() => undefined)
|
|
2968
|
+
}
|
|
2969
|
+
|
|
2970
|
+
/**
|
|
2971
|
+
* Minimal DOM shape {@link renderMarkdown} reads; a real DOM node fits it.
|
|
2972
|
+
*/
|
|
2973
|
+
export interface MarkdownNode {
|
|
2974
|
+
readonly nodeType?: number
|
|
2975
|
+
readonly tagName?: string | null
|
|
2976
|
+
readonly textContent?: string | null
|
|
2977
|
+
readonly childNodes?: ArrayLike<MarkdownNode> | null
|
|
2978
|
+
readonly href?: string | null
|
|
2979
|
+
readonly src?: string | null
|
|
2980
|
+
readonly alt?: string | null
|
|
2981
|
+
}
|
|
2982
|
+
|
|
2983
|
+
/**
|
|
2984
|
+
* Best-effort markdown rendering of a DOM subtree, used by
|
|
2985
|
+
* {@link ElectronBrowserProvider.content} for `format: 'markdown'`.
|
|
2986
|
+
*
|
|
2987
|
+
* Deliberately self-contained (no closures over module state, no imports):
|
|
2988
|
+
* the provider embeds this function's source in the page with
|
|
2989
|
+
* `Function.prototype.toString`, so the tests exercise the very code the page
|
|
2990
|
+
* runs.
|
|
2991
|
+
*
|
|
2992
|
+
* Block containers (div/p/section/article/li/headings/...) recurse into their
|
|
2993
|
+
* children and are joined with newlines, while adjacent inline runs are
|
|
2994
|
+
* concatenated — text split by <b>/<span> stays one paragraph, and a container
|
|
2995
|
+
* never emits its own `textContent` on top of its children (the old walker did,
|
|
2996
|
+
* which flattened real pages — everything is wrapped in divs — to plain text).
|
|
2997
|
+
* @param root - the subtree root (an element, or a text node).
|
|
2998
|
+
* @returns the markdown text.
|
|
2999
|
+
*/
|
|
3000
|
+
export function renderMarkdown(root: MarkdownNode): string {
|
|
3001
|
+
const BLOCK_TAGS = new Set([
|
|
3002
|
+
'address', 'article', 'aside', 'blockquote', 'dd', 'details', 'dialog',
|
|
3003
|
+
'div', 'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form',
|
|
3004
|
+
'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li',
|
|
3005
|
+
'main', 'nav', 'ol', 'p', 'pre', 'section', 'summary', 'table',
|
|
3006
|
+
'tbody', 'td', 'tfoot', 'th', 'thead', 'tr', 'ul',
|
|
3007
|
+
])
|
|
3008
|
+
const SKIP_TAGS = new Set(['script', 'style', 'noscript', 'template'])
|
|
3009
|
+
const collapse = (text: string): string => text.replace(/\s+/g, ' ').trim()
|
|
3010
|
+
const childrenOf = (node: MarkdownNode): MarkdownNode[] => {
|
|
3011
|
+
const list = node.childNodes
|
|
3012
|
+
if (list === undefined || list === null) return []
|
|
3013
|
+
const out: MarkdownNode[] = []
|
|
3014
|
+
for (let index = 0; index < list.length; index++) out.push(list[index])
|
|
3015
|
+
return out
|
|
3016
|
+
}
|
|
3017
|
+
/** Inline rendering: concatenates descendants, keeping links/images inline. */
|
|
3018
|
+
const inline = (node: MarkdownNode): string => {
|
|
3019
|
+
// Whitespace is collapsed but NOT trimmed: trimming here would eat the space
|
|
3020
|
+
// that separates two inline runs ('Hello ' + <b>world</b>).
|
|
3021
|
+
if (node.nodeType === 3) return (node.textContent ?? '').replace(/\s+/g, ' ')
|
|
3022
|
+
if (node.nodeType !== 1) return ''
|
|
3023
|
+
const tag = (node.tagName ?? '').toLowerCase()
|
|
3024
|
+
if (SKIP_TAGS.has(tag)) return ''
|
|
3025
|
+
if (tag === 'br') return '\n'
|
|
3026
|
+
if (tag === 'img') return node.src ? '' : ''
|
|
3027
|
+
if (tag === 'a') {
|
|
3028
|
+
const text = collapse(inlineChildren(node))
|
|
3029
|
+
return text === '' ? '' : '[' + text + '](' + (node.href ?? '') + ')'
|
|
3030
|
+
}
|
|
3031
|
+
return inlineChildren(node)
|
|
3032
|
+
}
|
|
3033
|
+
const inlineChildren = (node: MarkdownNode): string => childrenOf(node).map(inline).join('')
|
|
3034
|
+
/** Render one child as a block piece (own line) or an inline piece (merged). */
|
|
3035
|
+
const renderBlock = (node: MarkdownNode): { text: string; block: boolean } => {
|
|
3036
|
+
if (node.nodeType === 3) return { text: inline(node), block: false }
|
|
3037
|
+
if (node.nodeType !== 1) return { text: '', block: false }
|
|
3038
|
+
const tag = (node.tagName ?? '').toLowerCase()
|
|
3039
|
+
if (SKIP_TAGS.has(tag)) return { text: '', block: false }
|
|
3040
|
+
if (tag === 'br') return { text: '\n', block: false }
|
|
3041
|
+
if (tag === 'hr') return { text: '---', block: true }
|
|
3042
|
+
const heading = /^h([1-6])$/.exec(tag)
|
|
3043
|
+
if (heading !== null) {
|
|
3044
|
+
const text = collapse(inline(node))
|
|
3045
|
+
return { text: text === '' ? '' : '#'.repeat(Number(heading[1])) + ' ' + text, block: true }
|
|
3046
|
+
}
|
|
3047
|
+
if (tag === 'li') {
|
|
3048
|
+
const text = collapse(inline(node))
|
|
3049
|
+
return { text: text === '' ? '' : '- ' + text, block: true }
|
|
3050
|
+
}
|
|
3051
|
+
if (BLOCK_TAGS.has(tag)) return { text: renderChildren(node), block: true }
|
|
3052
|
+
return { text: inline(node), block: false }
|
|
3053
|
+
}
|
|
3054
|
+
/** Join a node's children: inline neighbours merge, block boundaries newline. */
|
|
3055
|
+
const renderChildren = (node: MarkdownNode): string => {
|
|
3056
|
+
const out: Array<{ text: string; block: boolean }> = []
|
|
3057
|
+
for (const child of childrenOf(node)) {
|
|
3058
|
+
const piece = renderBlock(child)
|
|
3059
|
+
if (collapse(piece.text) === '') continue
|
|
3060
|
+
const previous = out[out.length - 1]
|
|
3061
|
+
if (previous !== undefined && !previous.block && !piece.block) previous.text += piece.text
|
|
3062
|
+
else out.push({ text: piece.text, block: piece.block })
|
|
3063
|
+
}
|
|
3064
|
+
// Inline runs are trimmed once, after merging, so their inner spacing stays.
|
|
3065
|
+
return out.map(piece => piece.block ? piece.text : collapse(piece.text)).join('\n')
|
|
3066
|
+
}
|
|
3067
|
+
if (root.nodeType === 3) return collapse(root.textContent ?? '')
|
|
3068
|
+
return renderChildren(root).replace(/\n{3,}/g, '\n\n').trim()
|
|
3069
|
+
}
|
|
3070
|
+
|
|
3071
|
+
async function handleSendEvaluate(
|
|
3072
|
+
handle: ElectronViewHandle,
|
|
3073
|
+
expression: string,
|
|
3074
|
+
signal?: AbortSignal,
|
|
3075
|
+
): Promise<BrowserExecuteResult> {
|
|
3076
|
+
signal?.throwIfAborted()
|
|
3077
|
+
const result = await handle.sendCommand(CDP_RUNTIME_EVALUATE, {
|
|
3078
|
+
expression,
|
|
3079
|
+
returnByValue: true,
|
|
3080
|
+
awaitPromise: true,
|
|
3081
|
+
} satisfies CdpEvaluateParams)
|
|
3082
|
+
if (result.exceptionDetails !== undefined) {
|
|
3083
|
+
const detail = result.exceptionDetails as { text?: string; exception?: { description?: string } }
|
|
3084
|
+
return { ok: false, exception: detail.exception?.description ?? detail.text ?? 'unknown exception' }
|
|
3085
|
+
}
|
|
3086
|
+
return { ok: true, value: (result.result as { value?: unknown } | undefined)?.value ?? null }
|
|
3087
|
+
}
|
|
3088
|
+
|