dsh-builtin-browser 0.1.23 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +91 -3
- package/README.md +98 -4
- package/client.js +334 -0
- package/cordis.patch.yml +24 -5
- package/desktop-bridge/install.mjs +98 -0
- package/desktop-bridge/plugin-browser-bridge.js +385 -0
- package/docs/README.md +1 -1
- package/docs/architecture.md +1 -1
- package/docs/tool-reference.md +2 -1
- package/lib/browser/runtime.d.ts +18 -1
- package/lib/browser/runtime.js +19 -0
- package/lib/browser/types.d.ts +33 -0
- package/lib/browser-electron/bridge-connection.d.ts +58 -0
- package/lib/browser-electron/bridge-connection.js +133 -0
- package/lib/browser-electron/desktop-bridge-host.d.ts +124 -0
- package/lib/browser-electron/desktop-bridge-host.js +297 -0
- package/lib/browser-electron/entry.js +195 -11
- package/lib/browser-electron/history-store.d.ts +75 -0
- package/lib/browser-electron/history-store.js +162 -0
- package/lib/browser-electron/host-main.js +28 -1
- package/lib/browser-electron/provider.d.ts +122 -1
- package/lib/browser-electron/provider.js +233 -21
- package/lib/browser-electron/remote-host.d.ts +17 -1
- package/lib/browser-electron/remote-host.js +81 -9
- package/lib/browser-electron/settings-store.d.ts +92 -0
- package/lib/browser-electron/settings-store.js +183 -0
- package/lib/browser-electron/system-browser.d.ts +100 -0
- package/lib/browser-electron/system-browser.js +322 -0
- package/lib/browser-electron/virtual-cursor.d.ts +67 -0
- package/lib/browser-electron/virtual-cursor.js +141 -0
- package/lib/index.d.ts +16 -2
- package/lib/index.js +18 -2
- package/lib/tool-browser/index.js +122 -16
- package/package.json +14 -1
- package/src/browser/runtime.ts +22 -0
- package/src/browser/types.ts +31 -0
- package/src/browser-electron/bridge-connection.ts +145 -0
- package/src/browser-electron/desktop-bridge-host.ts +305 -0
- package/src/browser-electron/entry.ts +217 -11
- package/src/browser-electron/history-store.ts +171 -0
- package/src/browser-electron/host-main.ts +26 -1
- package/src/browser-electron/provider.ts +284 -21
- package/src/browser-electron/remote-host.ts +84 -7
- package/src/browser-electron/settings-store.ts +227 -0
- package/src/browser-electron/system-browser.ts +318 -0
- package/src/browser-electron/virtual-cursor.ts +152 -0
- package/src/index.ts +20 -2
- package/src/tool-browser/index.ts +121 -15
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drive the browser the user already has (Chrome / Edge / any Chromium) instead of
|
|
3
|
+
* the Electron we ship — the same approach Codex Browser Use takes.
|
|
4
|
+
*
|
|
5
|
+
* WHY
|
|
6
|
+
* The bundled Electron is a private copy with its own profile, so it can never see
|
|
7
|
+
* the logins the user already has in Chrome, and on a managed machine the user may
|
|
8
|
+
* simply prefer their own browser. Launching their browser with a *separate*
|
|
9
|
+
* `--user-data-dir` gives us a real Chromium we can drive over CDP without touching
|
|
10
|
+
* their everyday profile, bookmarks or windows: nothing of theirs is opened, locked
|
|
11
|
+
* or modified, and closing us never closes them.
|
|
12
|
+
*
|
|
13
|
+
* HOW
|
|
14
|
+
* Chromium exposes CDP on `--remote-debugging-port=0`, and writes the port it chose
|
|
15
|
+
* into `<profile>/DevToolsActivePort`. That file is the discovery mechanism — no
|
|
16
|
+
* port guessing, no collisions. From there everything is CDP over one WebSocket:
|
|
17
|
+
* `Target.createTarget` makes a page, `Target.attachToTarget` (flattened) gives a
|
|
18
|
+
* session for it, and the plugin's commands ride that session. Node 22 ships a global
|
|
19
|
+
* `WebSocket`, so this needs no dependency at all.
|
|
20
|
+
*
|
|
21
|
+
* WHAT IT IMPLEMENTS
|
|
22
|
+
* The same `ElectronBrowserViewHost` seam as the self-hosted and sidebar hosts, so
|
|
23
|
+
* the provider, the tools, history, the cursor and teardown rules are unchanged —
|
|
24
|
+
* only the carrier differs.
|
|
25
|
+
* @module dsh-browser/browser-electron/system-browser
|
|
26
|
+
*/
|
|
27
|
+
import { readFileSync, rmSync } from 'node:fs';
|
|
28
|
+
import { existsSync } from 'node:fs';
|
|
29
|
+
import { join } from 'node:path';
|
|
30
|
+
import { spawn } from 'node:child_process';
|
|
31
|
+
import { randomUUID } from 'node:crypto';
|
|
32
|
+
/**
|
|
33
|
+
* Where each Chromium product installs on Windows, plus the registry paths that
|
|
34
|
+
* survive a non-default install location. Only paths that exist are returned, so a
|
|
35
|
+
* caller can tell "not installed" from "installed elsewhere".
|
|
36
|
+
*/
|
|
37
|
+
const WINDOWS_LOCATIONS = {
|
|
38
|
+
chrome: [
|
|
39
|
+
String.raw `C:\Program Files\Google\Chrome\Application\chrome.exe`,
|
|
40
|
+
String.raw `C:\Program Files (x86)\Google\Chrome\Application\chrome.exe`,
|
|
41
|
+
],
|
|
42
|
+
edge: [
|
|
43
|
+
String.raw `C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe`,
|
|
44
|
+
String.raw `C:\Program Files\Microsoft\Edge\Application\msedge.exe`,
|
|
45
|
+
],
|
|
46
|
+
brave: [
|
|
47
|
+
String.raw `C:\Program Files\BraveSoftware\Brave-Browser\Application\brave.exe`,
|
|
48
|
+
String.raw `C:\Program Files (x86)\BraveSoftware\Brave-Browser\Application\brave.exe`,
|
|
49
|
+
],
|
|
50
|
+
};
|
|
51
|
+
/**
|
|
52
|
+
* Find an installed Chromium browser.
|
|
53
|
+
*
|
|
54
|
+
* `auto` prefers Chrome, then Edge, then Brave — the order reflects how likely each
|
|
55
|
+
* is to be the browser a user actually chose rather than one Windows shipped.
|
|
56
|
+
* @param channel - the configured choice; `bundled` never resolves to a system browser.
|
|
57
|
+
* @param env - environment lookup, injected so tests need no real machine.
|
|
58
|
+
* @returns the detected browser, or undefined when the choice is unavailable.
|
|
59
|
+
*/
|
|
60
|
+
export function detectBrowser(channel, env = process.env) {
|
|
61
|
+
if (channel === 'bundled')
|
|
62
|
+
return undefined;
|
|
63
|
+
const order = channel === 'auto' ? ['chrome', 'edge', 'brave'] : [channel];
|
|
64
|
+
for (const kind of order) {
|
|
65
|
+
const fromEnv = (kind === 'chrome' ? env.DSH_BROWSER_CHROME_PATH : kind === 'edge' ? env.DSH_BROWSER_EDGE_PATH : undefined);
|
|
66
|
+
if (fromEnv !== undefined && fromEnv !== '' && existsSync(fromEnv))
|
|
67
|
+
return { kind, path: fromEnv };
|
|
68
|
+
for (const candidate of WINDOWS_LOCATIONS[kind] ?? []) {
|
|
69
|
+
if (existsSync(candidate))
|
|
70
|
+
return { kind, path: candidate };
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return undefined;
|
|
74
|
+
}
|
|
75
|
+
/** One CDP connection to a browser-level WebSocket, with flattened sessions. */
|
|
76
|
+
class CdpClient {
|
|
77
|
+
url;
|
|
78
|
+
nextId = 1;
|
|
79
|
+
pending = new Map();
|
|
80
|
+
ready;
|
|
81
|
+
/**
|
|
82
|
+
* @param url - the browser's `webSocketDebuggerUrl`.
|
|
83
|
+
*/
|
|
84
|
+
constructor(url) {
|
|
85
|
+
this.url = url;
|
|
86
|
+
// Node 22 provides WebSocket globally; no dependency is added for this.
|
|
87
|
+
const socket = new WebSocket(url);
|
|
88
|
+
this.socket = socket;
|
|
89
|
+
this.ready = new Promise((resolve, reject) => {
|
|
90
|
+
socket.addEventListener('open', () => resolve());
|
|
91
|
+
socket.addEventListener('error', () => reject(new Error('dsh-builtin-browser: could not reach the browser over CDP')));
|
|
92
|
+
});
|
|
93
|
+
socket.addEventListener('message', event => this.onMessage(String(event.data)));
|
|
94
|
+
socket.addEventListener('close', () => this.fail(new Error('dsh-builtin-browser: the browser closed the CDP connection')));
|
|
95
|
+
}
|
|
96
|
+
socket;
|
|
97
|
+
/** Resolve once the socket is open. */
|
|
98
|
+
async whenReady() {
|
|
99
|
+
await this.ready;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Send one CDP command, optionally on an attached session.
|
|
103
|
+
* @param method - the CDP method.
|
|
104
|
+
* @param params - its parameters.
|
|
105
|
+
* @param sessionId - the flattened session to target, when this is a page command.
|
|
106
|
+
* @returns the CDP result.
|
|
107
|
+
*/
|
|
108
|
+
async send(method, params, sessionId) {
|
|
109
|
+
await this.ready;
|
|
110
|
+
const id = this.nextId++;
|
|
111
|
+
const message = { id, method, ...params !== undefined ? { params } : {}, ...sessionId !== undefined ? { sessionId } : {} };
|
|
112
|
+
return await new Promise((resolve, reject) => {
|
|
113
|
+
this.pending.set(id, { resolve, reject });
|
|
114
|
+
this.socket.send(JSON.stringify(message));
|
|
115
|
+
setTimeout(() => {
|
|
116
|
+
if (this.pending.delete(id))
|
|
117
|
+
reject(new Error(`dsh-builtin-browser: ${method} timed out`));
|
|
118
|
+
}, 30_000).unref?.();
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
/** Close the socket. */
|
|
122
|
+
close() {
|
|
123
|
+
try {
|
|
124
|
+
this.socket.close();
|
|
125
|
+
}
|
|
126
|
+
catch { /* already closed */ }
|
|
127
|
+
}
|
|
128
|
+
/** Route one CDP message to its waiter. */
|
|
129
|
+
onMessage(raw) {
|
|
130
|
+
let message;
|
|
131
|
+
try {
|
|
132
|
+
message = JSON.parse(raw);
|
|
133
|
+
}
|
|
134
|
+
catch {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
if (message.id === undefined)
|
|
138
|
+
return;
|
|
139
|
+
const entry = this.pending.get(message.id);
|
|
140
|
+
if (entry === undefined)
|
|
141
|
+
return;
|
|
142
|
+
this.pending.delete(message.id);
|
|
143
|
+
if (message.error !== undefined)
|
|
144
|
+
entry.reject(new Error(`dsh-builtin-browser: CDP error: ${message.error.message ?? 'unknown'}`));
|
|
145
|
+
else
|
|
146
|
+
entry.resolve(message.result);
|
|
147
|
+
}
|
|
148
|
+
/** Fail everything in flight. */
|
|
149
|
+
fail(error) {
|
|
150
|
+
const pending = [...this.pending.values()];
|
|
151
|
+
this.pending.clear();
|
|
152
|
+
for (const entry of pending)
|
|
153
|
+
entry.reject(error);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
/** A system browser driven over CDP, presented as a browser view host. */
|
|
157
|
+
export class SystemBrowserViewHost {
|
|
158
|
+
child;
|
|
159
|
+
client;
|
|
160
|
+
kind;
|
|
161
|
+
ephemeralDir;
|
|
162
|
+
views = new Map();
|
|
163
|
+
disposed = false;
|
|
164
|
+
/**
|
|
165
|
+
* @param child - the launched browser process.
|
|
166
|
+
* @param client - the CDP connection to it.
|
|
167
|
+
* @param kind - which product this is (for diagnostics).
|
|
168
|
+
* @param ephemeralDir - a throwaway profile to remove on release, when the user
|
|
169
|
+
* has turned persistence off and no login state should outlive the session.
|
|
170
|
+
*/
|
|
171
|
+
constructor(child, client, kind, ephemeralDir) {
|
|
172
|
+
this.child = child;
|
|
173
|
+
this.client = client;
|
|
174
|
+
this.kind = kind;
|
|
175
|
+
this.ephemeralDir = ephemeralDir;
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* @param browser - the detected installation to launch.
|
|
179
|
+
* @param profileDir - a plugin-owned directory; the user's own profile is never touched.
|
|
180
|
+
* @param extraArgs - additional Chromium switches.
|
|
181
|
+
* @param ephemeralDir - a directory to delete when the browser is released, used
|
|
182
|
+
* when persistence is switched off so no login state outlives the session.
|
|
183
|
+
* @returns the host, or undefined when the browser refuses to come up.
|
|
184
|
+
*/
|
|
185
|
+
static async launch(browser, profileDir, extraArgs = [], ephemeralDir) {
|
|
186
|
+
const args = [
|
|
187
|
+
'--remote-debugging-port=0',
|
|
188
|
+
`--user-data-dir=${profileDir}`,
|
|
189
|
+
// Behave like a fresh, unattended browser: no first-run UI, no default-browser
|
|
190
|
+
// prompt, no restore bubble, and no "Chrome is being controlled" infobar.
|
|
191
|
+
'--no-first-run',
|
|
192
|
+
'--no-default-browser-check',
|
|
193
|
+
'--disable-features=Translate,MediaRouter',
|
|
194
|
+
'--disable-session-crashed-bubble',
|
|
195
|
+
'--hide-crash-restore-bubble',
|
|
196
|
+
...extraArgs,
|
|
197
|
+
'about:blank',
|
|
198
|
+
];
|
|
199
|
+
const child = spawn(browser.path, args, { stdio: ['ignore', 'ignore', 'ignore'], windowsHide: false });
|
|
200
|
+
const portFile = join(profileDir, 'DevToolsActivePort');
|
|
201
|
+
const deadline = Date.now() + 30_000;
|
|
202
|
+
for (;;) {
|
|
203
|
+
if (Date.now() > deadline) {
|
|
204
|
+
child.kill();
|
|
205
|
+
return undefined;
|
|
206
|
+
}
|
|
207
|
+
if (child.exitCode !== null)
|
|
208
|
+
return undefined;
|
|
209
|
+
if (existsSync(portFile)) {
|
|
210
|
+
try {
|
|
211
|
+
const port = readFileSync(portFile, 'utf8').split('\n')[0]?.trim() ?? '';
|
|
212
|
+
if (port !== '') {
|
|
213
|
+
const version = await (await fetch(`http://127.0.0.1:${port}/json/version`)).json();
|
|
214
|
+
if (typeof version.webSocketDebuggerUrl === 'string') {
|
|
215
|
+
const client = new CdpClient(version.webSocketDebuggerUrl);
|
|
216
|
+
await client.whenReady();
|
|
217
|
+
return new SystemBrowserViewHost(child, client, browser.kind, ephemeralDir);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
catch {
|
|
222
|
+
// The file can exist a moment before the port answers; poll again.
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
await new Promise(resolve => setTimeout(resolve, 250));
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
/** Whether this host can back views: the CDP connection is live. */
|
|
229
|
+
available() {
|
|
230
|
+
return !this.disposed;
|
|
231
|
+
}
|
|
232
|
+
createView() {
|
|
233
|
+
const viewId = randomUUID();
|
|
234
|
+
return {
|
|
235
|
+
id: viewId,
|
|
236
|
+
sendCommand: async (method, params) => {
|
|
237
|
+
const session = this.views.get(viewId) ?? await this.ensureSession(viewId);
|
|
238
|
+
const result = await this.client.send(method, params, session);
|
|
239
|
+
return (result ?? {});
|
|
240
|
+
},
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
destroyView(handle) {
|
|
244
|
+
const session = this.views.get(handle.id);
|
|
245
|
+
this.views.delete(handle.id);
|
|
246
|
+
if (session === undefined)
|
|
247
|
+
return;
|
|
248
|
+
// A session id is not a target id: closing the page needs the target the session
|
|
249
|
+
// was attached to, which is why they are tracked separately.
|
|
250
|
+
const targetId = this.sessions.get(session);
|
|
251
|
+
this.sessions.delete(session);
|
|
252
|
+
if (targetId !== undefined) {
|
|
253
|
+
// Best-effort: a page that is already gone is not an error.
|
|
254
|
+
void this.client.send('Target.closeTarget', { targetId }).catch(() => undefined);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
/** Nothing to show: the browser owns its own windows. */
|
|
258
|
+
showView() { }
|
|
259
|
+
/** Same as {@link showView}. */
|
|
260
|
+
async presentView() { }
|
|
261
|
+
/** The browser owns window grouping. */
|
|
262
|
+
groupView() { }
|
|
263
|
+
/**
|
|
264
|
+
* Bring the browser's window forward.
|
|
265
|
+
*
|
|
266
|
+
* The browser is a separate application, so there is no view to raise; focusing
|
|
267
|
+
* the process is the closest honest equivalent, and failing to do so (a minimized
|
|
268
|
+
* window, a locked session) is never worth an error.
|
|
269
|
+
*/
|
|
270
|
+
async focus() {
|
|
271
|
+
try {
|
|
272
|
+
this.child.kill('SIGCONT');
|
|
273
|
+
}
|
|
274
|
+
catch { /* not supported on Windows; harmless */ }
|
|
275
|
+
}
|
|
276
|
+
/** The browser reports its own window lifecycle. */
|
|
277
|
+
onUserAction() { }
|
|
278
|
+
/** The browser reports its own window lifecycle. */
|
|
279
|
+
onViewClosed() { }
|
|
280
|
+
/** Close the browser we launched. The user's own windows are a different process. */
|
|
281
|
+
dispose() {
|
|
282
|
+
if (this.disposed)
|
|
283
|
+
return;
|
|
284
|
+
this.disposed = true;
|
|
285
|
+
this.views.clear();
|
|
286
|
+
this.sessions.clear();
|
|
287
|
+
this.client.close();
|
|
288
|
+
try {
|
|
289
|
+
this.child.kill();
|
|
290
|
+
}
|
|
291
|
+
catch { /* already gone */ }
|
|
292
|
+
const ephemeral = this.ephemeralDir;
|
|
293
|
+
if (ephemeral !== undefined) {
|
|
294
|
+
// The browser flushes its profile while shutting down, so deleting later avoids
|
|
295
|
+
// racing it. Leftovers are only a stray temp directory, never a live credential.
|
|
296
|
+
setTimeout(() => { try {
|
|
297
|
+
rmSync(ephemeral, { recursive: true, force: true });
|
|
298
|
+
}
|
|
299
|
+
catch { /* best effort */ } }, 3_000).unref?.();
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
/** The flattened session for a view, creating its page on first use. */
|
|
303
|
+
async ensureSession(viewId) {
|
|
304
|
+
const existing = this.views.get(viewId);
|
|
305
|
+
if (existing !== undefined)
|
|
306
|
+
return existing;
|
|
307
|
+
const created = await this.client.send('Target.createTarget', { url: 'about:blank' });
|
|
308
|
+
const targetId = created?.targetId;
|
|
309
|
+
if (typeof targetId !== 'string')
|
|
310
|
+
throw new Error('dsh-builtin-browser: the browser did not create a page');
|
|
311
|
+
const attached = await this.client.send('Target.attachToTarget', { targetId, flatten: true });
|
|
312
|
+
const sessionId = attached?.sessionId;
|
|
313
|
+
if (typeof sessionId !== 'string')
|
|
314
|
+
throw new Error('dsh-builtin-browser: the browser did not attach to the new page');
|
|
315
|
+
// The session id is what page commands ride; the target id is what closes it.
|
|
316
|
+
this.sessions.set(sessionId, targetId);
|
|
317
|
+
this.views.set(viewId, sessionId);
|
|
318
|
+
return sessionId;
|
|
319
|
+
}
|
|
320
|
+
/** sessionId -> targetId, so a destroyed view can close the right page. */
|
|
321
|
+
sessions = new Map();
|
|
322
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Synthetic pointer feedback for a human watching the tab: an overlay drawn
|
|
3
|
+
* INSIDE the page, never the real system pointer. Its presence is the "the agent
|
|
4
|
+
* has taken over this tab" signal, so it also covers DOM-level operations that
|
|
5
|
+
* move no pointer at all.
|
|
6
|
+
*
|
|
7
|
+
* Implementation notes that matter:
|
|
8
|
+
* - The overlay is `position: fixed` and positioned from viewport coordinates,
|
|
9
|
+
* which is exactly what the element locator already computes and what
|
|
10
|
+
* `Input.dispatchMouseEvent` consumes.
|
|
11
|
+
* - All styling is inline and the ripple uses the Web Animations API: a
|
|
12
|
+
* `<style>` element would be subject to the page's `style-src` CSP, an
|
|
13
|
+
* injected script is not. Injected page feedback must never be silently
|
|
14
|
+
* dropped by a strict CSP.
|
|
15
|
+
* - Every call is best-effort: feedback is never allowed to fail an operation.
|
|
16
|
+
* @module dsh-browser/browser-electron/virtual-cursor
|
|
17
|
+
*/
|
|
18
|
+
/** What the pointer is doing at the reported point. */
|
|
19
|
+
export type CursorAction = 'move' | 'click';
|
|
20
|
+
/**
|
|
21
|
+
* Whether a paint at this position would change anything on screen.
|
|
22
|
+
* @param handle - the view the overlay belongs to.
|
|
23
|
+
* @param x - viewport x in CSS pixels.
|
|
24
|
+
* @param y - viewport y in CSS pixels.
|
|
25
|
+
* @param action - the action being reported.
|
|
26
|
+
* @param force - paint even when unchanged (a click ripple must always fire).
|
|
27
|
+
* @returns true when the overlay should be painted.
|
|
28
|
+
*/
|
|
29
|
+
export declare function cursorNeedsPaint<TView>(handle: TView, x: number, y: number, action: CursorAction, force?: boolean): boolean;
|
|
30
|
+
/**
|
|
31
|
+
* The in-page expression that materializes or moves the overlay. Idempotent: the
|
|
32
|
+
* first call creates the nodes, later calls only move them (and pulse).
|
|
33
|
+
* @param x - viewport x in CSS pixels.
|
|
34
|
+
* @param y - viewport y in CSS pixels.
|
|
35
|
+
* @param action - whether to pulse a click ripple at the point.
|
|
36
|
+
* @param label - optional short description of the operation, shown beside the
|
|
37
|
+
* pointer. The overlay is the only "the agent has taken over this tab" signal, so
|
|
38
|
+
* saying *what* it is doing is what turns it from a decoration into an account of
|
|
39
|
+
* the agent's actions that a human can follow without reading the tool log.
|
|
40
|
+
* @returns a self-contained expression that evaluates to `true`.
|
|
41
|
+
*/
|
|
42
|
+
export declare function cursorScript(x: number, y: number, action: CursorAction, label?: string): string;
|
|
43
|
+
/**
|
|
44
|
+
* Forget the remembered position for a view.
|
|
45
|
+
*
|
|
46
|
+
* Called when the document is replaced: the overlay lived in the old document and
|
|
47
|
+
* went with it, so the cache would otherwise claim the pointer is already where it
|
|
48
|
+
* needs to be and skip every later paint — the pointer would simply never return
|
|
49
|
+
* after a navigation.
|
|
50
|
+
* @param handle - the view whose cache should be dropped.
|
|
51
|
+
*/
|
|
52
|
+
export declare function forgetCursor<TView>(handle: TView): void;
|
|
53
|
+
/**
|
|
54
|
+
* Draw (or move) the pointer overlay on one view's page.
|
|
55
|
+
*
|
|
56
|
+
* Fire-and-forget: a page that is mid-navigation, cross-origin-blocked, or
|
|
57
|
+
* otherwise unable to answer must not turn a successful operation into a
|
|
58
|
+
* failure, and the navigation that just committed may still be replacing the
|
|
59
|
+
* document this call would have painted into.
|
|
60
|
+
* @param handle - the view to paint into.
|
|
61
|
+
* @param x - viewport x in CSS pixels.
|
|
62
|
+
* @param y - viewport y in CSS pixels.
|
|
63
|
+
* @param action - whether to pulse a click ripple.
|
|
64
|
+
* @param evaluate - the provider's evaluate bridge (keeps the CDP coupling in one place).
|
|
65
|
+
* @param label - optional short description of the operation, shown beside the pointer.
|
|
66
|
+
*/
|
|
67
|
+
export declare function paintCursor<TView>(handle: TView, x: number, y: number, action: CursorAction, evaluate: (handle: TView, expression: string) => Promise<unknown>, label?: string, force?: boolean): void;
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Synthetic pointer feedback for a human watching the tab: an overlay drawn
|
|
3
|
+
* INSIDE the page, never the real system pointer. Its presence is the "the agent
|
|
4
|
+
* has taken over this tab" signal, so it also covers DOM-level operations that
|
|
5
|
+
* move no pointer at all.
|
|
6
|
+
*
|
|
7
|
+
* Implementation notes that matter:
|
|
8
|
+
* - The overlay is `position: fixed` and positioned from viewport coordinates,
|
|
9
|
+
* which is exactly what the element locator already computes and what
|
|
10
|
+
* `Input.dispatchMouseEvent` consumes.
|
|
11
|
+
* - All styling is inline and the ripple uses the Web Animations API: a
|
|
12
|
+
* `<style>` element would be subject to the page's `style-src` CSP, an
|
|
13
|
+
* injected script is not. Injected page feedback must never be silently
|
|
14
|
+
* dropped by a strict CSP.
|
|
15
|
+
* - Every call is best-effort: feedback is never allowed to fail an operation.
|
|
16
|
+
* @module dsh-browser/browser-electron/virtual-cursor
|
|
17
|
+
*/
|
|
18
|
+
/** Element id of the overlay root, stable so repeat calls update one node. */
|
|
19
|
+
const CURSOR_ID = '__dsh_agent_cursor__';
|
|
20
|
+
/**
|
|
21
|
+
* Where the overlay was last placed, so an unchanged position is not repainted.
|
|
22
|
+
*
|
|
23
|
+
* A single tool call issues several CDP commands (locate, click, cursor), and the
|
|
24
|
+
* cursor is repainted for each one that moves it. Repainting to the same pixel is a
|
|
25
|
+
* wasted round-trip *and* a visible flicker: the transition restarts from the same
|
|
26
|
+
* place, which reads as a stutter rather than as movement.
|
|
27
|
+
*/
|
|
28
|
+
const lastPlaced = new WeakMap();
|
|
29
|
+
/**
|
|
30
|
+
* Whether a paint at this position would change anything on screen.
|
|
31
|
+
* @param handle - the view the overlay belongs to.
|
|
32
|
+
* @param x - viewport x in CSS pixels.
|
|
33
|
+
* @param y - viewport y in CSS pixels.
|
|
34
|
+
* @param action - the action being reported.
|
|
35
|
+
* @param force - paint even when unchanged (a click ripple must always fire).
|
|
36
|
+
* @returns true when the overlay should be painted.
|
|
37
|
+
*/
|
|
38
|
+
export function cursorNeedsPaint(handle, x, y, action, force = false) {
|
|
39
|
+
if (force)
|
|
40
|
+
return true;
|
|
41
|
+
const key = handle;
|
|
42
|
+
if (key === null || typeof key !== 'object')
|
|
43
|
+
return true;
|
|
44
|
+
const previous = lastPlaced.get(key);
|
|
45
|
+
return previous === undefined || previous.x !== x || previous.y !== y || previous.action !== action;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* The in-page expression that materializes or moves the overlay. Idempotent: the
|
|
49
|
+
* first call creates the nodes, later calls only move them (and pulse).
|
|
50
|
+
* @param x - viewport x in CSS pixels.
|
|
51
|
+
* @param y - viewport y in CSS pixels.
|
|
52
|
+
* @param action - whether to pulse a click ripple at the point.
|
|
53
|
+
* @param label - optional short description of the operation, shown beside the
|
|
54
|
+
* pointer. The overlay is the only "the agent has taken over this tab" signal, so
|
|
55
|
+
* saying *what* it is doing is what turns it from a decoration into an account of
|
|
56
|
+
* the agent's actions that a human can follow without reading the tool log.
|
|
57
|
+
* @returns a self-contained expression that evaluates to `true`.
|
|
58
|
+
*/
|
|
59
|
+
export function cursorScript(x, y, action, label) {
|
|
60
|
+
return `(() => {
|
|
61
|
+
const id = ${JSON.stringify(CURSOR_ID)}
|
|
62
|
+
let root = document.getElementById(id)
|
|
63
|
+
if (!root) {
|
|
64
|
+
root = document.createElement('div')
|
|
65
|
+
root.id = id
|
|
66
|
+
root.setAttribute('aria-hidden', 'true')
|
|
67
|
+
root.style.cssText = 'position:fixed;left:0;top:0;width:0;height:0;z-index:2147483647;pointer-events:none;transition:transform 190ms cubic-bezier(.22,.85,.24,1);will-change:transform'
|
|
68
|
+
const svg = '<svg width="22" height="22" viewBox="0 0 22 22" style="position:absolute;left:-2px;top:-2px;filter:drop-shadow(0 1px 2px rgba(0,0,0,.45))">'
|
|
69
|
+
+ '<path d="M4 2 L4 18 L8.4 13.8 L11.2 19.6 L13.8 18.4 L11 12.8 L17 12.6 Z" fill="#ffffff" stroke="#111111" stroke-width="1.4" stroke-linejoin="round"/></svg>'
|
|
70
|
+
const ripple = '<span data-dsh-role="ripple" style="position:absolute;left:0;top:0;width:34px;height:34px;margin:-17px 0 0 -17px;border-radius:50%;border:2px solid rgba(64,160,255,.9);opacity:0"></span>'
|
|
71
|
+
// The bubble sits below-right of the pointer tip and is capped in width so a long
|
|
72
|
+
// label cannot cover the page it is describing.
|
|
73
|
+
const bubble = '<span data-dsh-role="bubble" style="position:absolute;left:18px;top:16px;max-width:280px;padding:3px 8px;border-radius:5px;background:rgba(17,17,17,.92);color:#fff;font:12px/1.45 -apple-system,Segoe UI,Roboto,sans-serif;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;opacity:0;box-shadow:0 1px 3px rgba(0,0,0,.4)"></span>'
|
|
74
|
+
root.innerHTML = svg + ripple + bubble
|
|
75
|
+
;(document.body || document.documentElement).appendChild(root)
|
|
76
|
+
}
|
|
77
|
+
root.style.transform = 'translate(${x}px, ${y}px)'
|
|
78
|
+
${action === 'click' ? `const ripple = root.querySelector('[data-dsh-role="ripple"]')
|
|
79
|
+
if (ripple && typeof ripple.animate === 'function') {
|
|
80
|
+
ripple.animate(
|
|
81
|
+
[{ transform: 'scale(.25)', opacity: 0.95 }, { transform: 'scale(1.6)', opacity: 0 }],
|
|
82
|
+
{ duration: 550, easing: 'ease-out' },
|
|
83
|
+
)
|
|
84
|
+
}` : ''}
|
|
85
|
+
${label !== undefined && label !== '' ? `const bubble = root.querySelector('[data-dsh-role="bubble"]')
|
|
86
|
+
if (bubble) {
|
|
87
|
+
bubble.textContent = ${JSON.stringify(label)}
|
|
88
|
+
if (typeof bubble.animate === 'function') {
|
|
89
|
+
// Fade in, hold, fade out: the label is a narration of the action, not a
|
|
90
|
+
// permanent caption, so a stale one never lingers over later work.
|
|
91
|
+
bubble.animate(
|
|
92
|
+
[{ opacity: 0, transform: 'translateY(-3px)' }, { opacity: 1, transform: 'translateY(0)' }, { opacity: 1, offset: 0.6 }, { opacity: 0 }],
|
|
93
|
+
{ duration: 2200, easing: 'ease-out', fill: 'forwards' },
|
|
94
|
+
)
|
|
95
|
+
} else {
|
|
96
|
+
bubble.style.opacity = '1'
|
|
97
|
+
}
|
|
98
|
+
}` : ''}
|
|
99
|
+
return true
|
|
100
|
+
})()`;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Forget the remembered position for a view.
|
|
104
|
+
*
|
|
105
|
+
* Called when the document is replaced: the overlay lived in the old document and
|
|
106
|
+
* went with it, so the cache would otherwise claim the pointer is already where it
|
|
107
|
+
* needs to be and skip every later paint — the pointer would simply never return
|
|
108
|
+
* after a navigation.
|
|
109
|
+
* @param handle - the view whose cache should be dropped.
|
|
110
|
+
*/
|
|
111
|
+
export function forgetCursor(handle) {
|
|
112
|
+
const key = handle;
|
|
113
|
+
if (key !== null && typeof key === 'object')
|
|
114
|
+
lastPlaced.delete(key);
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Draw (or move) the pointer overlay on one view's page.
|
|
118
|
+
*
|
|
119
|
+
* Fire-and-forget: a page that is mid-navigation, cross-origin-blocked, or
|
|
120
|
+
* otherwise unable to answer must not turn a successful operation into a
|
|
121
|
+
* failure, and the navigation that just committed may still be replacing the
|
|
122
|
+
* document this call would have painted into.
|
|
123
|
+
* @param handle - the view to paint into.
|
|
124
|
+
* @param x - viewport x in CSS pixels.
|
|
125
|
+
* @param y - viewport y in CSS pixels.
|
|
126
|
+
* @param action - whether to pulse a click ripple.
|
|
127
|
+
* @param evaluate - the provider's evaluate bridge (keeps the CDP coupling in one place).
|
|
128
|
+
* @param label - optional short description of the operation, shown beside the pointer.
|
|
129
|
+
*/
|
|
130
|
+
export function paintCursor(handle, x, y, action, evaluate, label, force = false) {
|
|
131
|
+
if (!Number.isFinite(x) || !Number.isFinite(y))
|
|
132
|
+
return;
|
|
133
|
+
const roundedX = Math.round(x);
|
|
134
|
+
const roundedY = Math.round(y);
|
|
135
|
+
if (!cursorNeedsPaint(handle, roundedX, roundedY, action, force))
|
|
136
|
+
return;
|
|
137
|
+
const key = handle;
|
|
138
|
+
if (key !== null && typeof key === 'object')
|
|
139
|
+
lastPlaced.set(key, { x: roundedX, y: roundedY, action });
|
|
140
|
+
void evaluate(handle, cursorScript(roundedX, roundedY, action, label)).catch(() => { });
|
|
141
|
+
}
|
package/lib/index.d.ts
CHANGED
|
@@ -4,10 +4,24 @@
|
|
|
4
4
|
* - `dsh-builtin-browser/browser` -> the ctx.browser seam (Service)
|
|
5
5
|
* - `dsh-builtin-browser/browser-electron` -> the Electron CDP provider
|
|
6
6
|
* - `dsh-builtin-browser/tool-browser` -> the model-facing browser_* tools
|
|
7
|
-
*
|
|
8
|
-
* the
|
|
7
|
+
* The capability itself is composed by those rows; this root entry exists for
|
|
8
|
+
* programmatic imports AND because the host's client-module scan identifies a
|
|
9
|
+
* package's browser half through a loader row whose specifier resolves to the
|
|
10
|
+
* PACKAGE ROOT. A row named after a subpath (`dsh-builtin-browser/browser`) is
|
|
11
|
+
* not a package specifier and is skipped by design, which is why the patch also
|
|
12
|
+
* registers a row under the bare package name — see cordis.patch.yml.
|
|
9
13
|
* @module dsh-builtin-browser
|
|
10
14
|
*/
|
|
15
|
+
/** Plugin identity for the root row (the composition surface is the subpath rows). */
|
|
16
|
+
export declare const name = "dsh-builtin-browser";
|
|
17
|
+
/**
|
|
18
|
+
* Inert application for the root row: the browser seam, the Electron provider and
|
|
19
|
+
* the tools are mounted by the subpath rows, so this one only exists to give the
|
|
20
|
+
* package a loader row named after the package itself. Without it the host's
|
|
21
|
+
* client scan finds no row to resolve `dsh.client` from, and the settings panel
|
|
22
|
+
* never reaches the browser (the plugin works, its client half is invisible).
|
|
23
|
+
*/
|
|
24
|
+
export declare function apply(): void;
|
|
11
25
|
export { BrowserError } from './browser/types.js';
|
|
12
26
|
export type { BrowserA11yNode, BrowserA11yRequest, BrowserA11yResult, BrowserChallenge, BrowserCheckRequest, BrowserClearRequest, BrowserClickRequest, BrowserContentFormat, BrowserContentRequest, BrowserContentResult, BrowserElementTarget, BrowserExecuteRequest, BrowserExecuteResult, BrowserFillField, BrowserFillRequest, BrowserFillResult, BrowserGetValueRequest, BrowserGetValueResult, BrowserNavigateRequest, BrowserOpenRequest, BrowserProvider, BrowserScrapeField, BrowserScrapeRequest, BrowserScrapeResult, BrowserScreenshotRequest, BrowserScreenshotResult, BrowserSelectRequest, BrowserSelectResult, BrowserSessionId, BrowserSetValueRequest, BrowserSetValueResult, BrowserSnapshotElement, BrowserSnapshotResult, BrowserTab, BrowserTypeRequest, ExportedCookie, } from './browser/types.js';
|
|
13
27
|
export { BrowserRuntime } from './browser/runtime.js';
|
package/lib/index.js
CHANGED
|
@@ -4,10 +4,26 @@
|
|
|
4
4
|
* - `dsh-builtin-browser/browser` -> the ctx.browser seam (Service)
|
|
5
5
|
* - `dsh-builtin-browser/browser-electron` -> the Electron CDP provider
|
|
6
6
|
* - `dsh-builtin-browser/tool-browser` -> the model-facing browser_* tools
|
|
7
|
-
*
|
|
8
|
-
* the
|
|
7
|
+
* The capability itself is composed by those rows; this root entry exists for
|
|
8
|
+
* programmatic imports AND because the host's client-module scan identifies a
|
|
9
|
+
* package's browser half through a loader row whose specifier resolves to the
|
|
10
|
+
* PACKAGE ROOT. A row named after a subpath (`dsh-builtin-browser/browser`) is
|
|
11
|
+
* not a package specifier and is skipped by design, which is why the patch also
|
|
12
|
+
* registers a row under the bare package name — see cordis.patch.yml.
|
|
9
13
|
* @module dsh-builtin-browser
|
|
10
14
|
*/
|
|
15
|
+
/** Plugin identity for the root row (the composition surface is the subpath rows). */
|
|
16
|
+
export const name = 'dsh-builtin-browser';
|
|
17
|
+
/**
|
|
18
|
+
* Inert application for the root row: the browser seam, the Electron provider and
|
|
19
|
+
* the tools are mounted by the subpath rows, so this one only exists to give the
|
|
20
|
+
* package a loader row named after the package itself. Without it the host's
|
|
21
|
+
* client scan finds no row to resolve `dsh.client` from, and the settings panel
|
|
22
|
+
* never reaches the browser (the plugin works, its client half is invisible).
|
|
23
|
+
*/
|
|
24
|
+
export function apply() {
|
|
25
|
+
// Intentionally empty: see the module comment above.
|
|
26
|
+
}
|
|
11
27
|
export { BrowserError } from './browser/types.js';
|
|
12
28
|
export { BrowserRuntime } from './browser/runtime.js';
|
|
13
29
|
export { ElectronBrowserProvider } from './browser-electron/provider.js';
|