@vellumai/assistant 0.4.21 → 0.4.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/ipc/check-swift-decoder-drift.ts +55 -44
- package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +0 -75
- package/src/__tests__/headless-browser-interactions.test.ts +0 -4
- package/src/__tests__/ipc-snapshot.test.ts +0 -54
- package/src/__tests__/resolve-guardian-trust-class.test.ts +61 -0
- package/src/__tests__/session-init.benchmark.test.ts +0 -4
- package/src/config/system-prompt.ts +1 -0
- package/src/config/templates/BOOTSTRAP.md +21 -31
- package/src/config/templates/SOUL.md +19 -9
- package/src/daemon/computer-use-session.ts +5 -3
- package/src/daemon/daemon-control.ts +3 -0
- package/src/daemon/handlers/browser.ts +2 -48
- package/src/daemon/handlers/config-voice.ts +155 -33
- package/src/daemon/handlers/dictation.ts +361 -214
- package/src/daemon/ipc-contract/browser.ts +4 -74
- package/src/daemon/ipc-contract/surfaces.ts +51 -48
- package/src/daemon/ipc-contract-inventory.json +0 -7
- package/src/daemon/session-agent-loop.ts +2 -1
- package/src/daemon/session-runtime-assembly.ts +477 -247
- package/src/daemon/session-surfaces.ts +5 -3
- package/src/daemon/session-tool-setup.ts +27 -13
- package/src/memory/migrations/102-alter-table-columns.ts +254 -37
- package/src/memory/schema.ts +1227 -1035
- package/src/tools/browser/browser-execution.ts +314 -331
- package/src/tools/browser/browser-handoff.ts +11 -37
- package/src/tools/browser/browser-manager.ts +271 -264
- package/src/tools/browser/browser-screencast.ts +19 -75
|
@@ -1,31 +1,15 @@
|
|
|
1
|
-
import { mkdirSync } from
|
|
2
|
-
import { join } from
|
|
3
|
-
|
|
4
|
-
import { getLogger } from
|
|
5
|
-
import { getDataDir } from
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import type { ExtractedCredential } from './network-recording-types.js';
|
|
9
|
-
import { checkBrowserRuntime } from './runtime-check.js';
|
|
10
|
-
|
|
11
|
-
const log = getLogger('browser-manager');
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* Returns true when the host has a GUI capable of displaying a browser window.
|
|
15
|
-
* macOS and Windows always have a display; Linux requires DISPLAY or WAYLAND_DISPLAY.
|
|
16
|
-
*/
|
|
17
|
-
function canDisplayGui(): boolean {
|
|
18
|
-
if (process.platform === 'darwin' || process.platform === 'win32') return true;
|
|
19
|
-
return !!(process.env.DISPLAY || process.env.WAYLAND_DISPLAY);
|
|
20
|
-
}
|
|
1
|
+
import { mkdirSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
|
|
4
|
+
import { getLogger } from "../../util/logger.js";
|
|
5
|
+
import { getDataDir } from "../../util/platform.js";
|
|
6
|
+
import { authSessionCache } from "./auth-cache.js";
|
|
7
|
+
import type { ExtractedCredential } from "./network-recording-types.js";
|
|
21
8
|
|
|
22
|
-
|
|
23
|
-
// to map between page coordinates and screencast-frame coordinates.
|
|
24
|
-
export const SCREENCAST_WIDTH = 1280;
|
|
25
|
-
export const SCREENCAST_HEIGHT = 800;
|
|
9
|
+
const log = getLogger("browser-manager");
|
|
26
10
|
|
|
27
11
|
function getDownloadsDir(): string {
|
|
28
|
-
const dir = join(getDataDir(),
|
|
12
|
+
const dir = join(getDataDir(), "browser-downloads");
|
|
29
13
|
mkdirSync(dir, { recursive: true });
|
|
30
14
|
return dir;
|
|
31
15
|
}
|
|
@@ -43,7 +27,10 @@ export type PageResponse = {
|
|
|
43
27
|
url(): string;
|
|
44
28
|
};
|
|
45
29
|
|
|
46
|
-
export type RouteHandler = (
|
|
30
|
+
export type RouteHandler = (
|
|
31
|
+
route: PageRoute,
|
|
32
|
+
request: PageRequest,
|
|
33
|
+
) => Promise<void> | void;
|
|
47
34
|
|
|
48
35
|
export type PageRoute = {
|
|
49
36
|
abort(errorCode?: string): Promise<void>;
|
|
@@ -57,38 +44,59 @@ export type PageRequest = {
|
|
|
57
44
|
export type Page = {
|
|
58
45
|
close(): Promise<void>;
|
|
59
46
|
isClosed(): boolean;
|
|
60
|
-
goto(
|
|
47
|
+
goto(
|
|
48
|
+
url: string,
|
|
49
|
+
options?: { waitUntil?: string; timeout?: number },
|
|
50
|
+
): Promise<PageResponse | null>;
|
|
61
51
|
title(): Promise<string>;
|
|
62
52
|
url(): string;
|
|
63
53
|
evaluate(expression: string): Promise<unknown>;
|
|
64
54
|
click(selector: string, options?: { timeout?: number }): Promise<void>;
|
|
65
|
-
fill(
|
|
66
|
-
|
|
67
|
-
|
|
55
|
+
fill(
|
|
56
|
+
selector: string,
|
|
57
|
+
value: string,
|
|
58
|
+
options?: { timeout?: number },
|
|
59
|
+
): Promise<void>;
|
|
60
|
+
press(
|
|
61
|
+
selector: string,
|
|
62
|
+
key: string,
|
|
63
|
+
options?: { timeout?: number },
|
|
64
|
+
): Promise<void>;
|
|
65
|
+
selectOption(
|
|
66
|
+
selector: string,
|
|
67
|
+
values: Record<string, string | number>,
|
|
68
|
+
options?: { timeout?: number },
|
|
69
|
+
): Promise<string[]>;
|
|
68
70
|
hover(selector: string, options?: { timeout?: number }): Promise<void>;
|
|
69
|
-
waitForSelector(
|
|
70
|
-
|
|
71
|
+
waitForSelector(
|
|
72
|
+
selector: string,
|
|
73
|
+
options?: { timeout?: number },
|
|
74
|
+
): Promise<unknown>;
|
|
75
|
+
waitForFunction(
|
|
76
|
+
expression: string,
|
|
77
|
+
options?: { timeout?: number },
|
|
78
|
+
): Promise<unknown>;
|
|
71
79
|
route(pattern: string, handler: RouteHandler): Promise<void>;
|
|
72
80
|
unroute(pattern: string, handler?: RouteHandler): Promise<void>;
|
|
73
81
|
bringToFront(): Promise<void>;
|
|
74
|
-
screenshot(options?: {
|
|
82
|
+
screenshot(options?: {
|
|
83
|
+
type?: string;
|
|
84
|
+
quality?: number;
|
|
85
|
+
fullPage?: boolean;
|
|
86
|
+
}): Promise<Buffer>;
|
|
75
87
|
keyboard: { press(key: string): Promise<void> };
|
|
76
88
|
mouse: {
|
|
77
|
-
click(
|
|
89
|
+
click(
|
|
90
|
+
x: number,
|
|
91
|
+
y: number,
|
|
92
|
+
options?: { button?: string; clickCount?: number },
|
|
93
|
+
): Promise<void>;
|
|
78
94
|
move(x: number, y: number): Promise<void>;
|
|
79
95
|
wheel(deltaX: number, deltaY: number): Promise<void>;
|
|
80
96
|
};
|
|
81
97
|
on(event: string, handler: (...args: unknown[]) => void): void;
|
|
82
98
|
};
|
|
83
99
|
|
|
84
|
-
type ScreencastFrameMetadata = {
|
|
85
|
-
offsetTop: number;
|
|
86
|
-
pageScaleFactor: number;
|
|
87
|
-
scrollOffsetX: number;
|
|
88
|
-
scrollOffsetY: number;
|
|
89
|
-
timestamp: number;
|
|
90
|
-
};
|
|
91
|
-
|
|
92
100
|
type CDPSession = {
|
|
93
101
|
send(method: string, params?: Record<string, unknown>): Promise<unknown>;
|
|
94
102
|
on(event: string, handler: (params: Record<string, unknown>) => void): void;
|
|
@@ -99,7 +107,10 @@ type RawPlaywrightPage = {
|
|
|
99
107
|
context(): { newCDPSession(page: unknown): Promise<CDPSession> };
|
|
100
108
|
};
|
|
101
109
|
|
|
102
|
-
type LaunchFn = (
|
|
110
|
+
type LaunchFn = (
|
|
111
|
+
userDataDir: string,
|
|
112
|
+
options: { headless: boolean },
|
|
113
|
+
) => Promise<BrowserContext>;
|
|
103
114
|
|
|
104
115
|
let launchPersistentContext: LaunchFn | null = null;
|
|
105
116
|
|
|
@@ -107,13 +118,8 @@ export function setLaunchFn(fn: LaunchFn | null): void {
|
|
|
107
118
|
launchPersistentContext = fn;
|
|
108
119
|
}
|
|
109
120
|
|
|
110
|
-
async function getDefaultLaunchFn(): Promise<LaunchFn> {
|
|
111
|
-
const pw = await import('playwright');
|
|
112
|
-
return pw.chromium.launchPersistentContext.bind(pw.chromium);
|
|
113
|
-
}
|
|
114
|
-
|
|
115
121
|
function getProfileDir(): string {
|
|
116
|
-
return join(getDataDir(),
|
|
122
|
+
return join(getDataDir(), "browser-profile");
|
|
117
123
|
}
|
|
118
124
|
|
|
119
125
|
class BrowserManager {
|
|
@@ -123,31 +129,38 @@ class BrowserManager {
|
|
|
123
129
|
private pages = new Map<string, Page>();
|
|
124
130
|
private rawPages = new Map<string, unknown>();
|
|
125
131
|
private cdpSessions = new Map<string, CDPSession>();
|
|
126
|
-
private screencastCallbacks = new Map<string, (frame: { data: string; metadata: ScreencastFrameMetadata }) => void>();
|
|
127
132
|
private snapshotMaps = new Map<string, Map<string, string>>();
|
|
128
|
-
private
|
|
129
|
-
private cdpUrl: string = 'http://localhost:9222';
|
|
133
|
+
private cdpUrl: string = "http://localhost:9222";
|
|
130
134
|
private cdpBrowser: unknown = null; // Store CDP browser reference separately
|
|
131
|
-
private _browserLaunched = false; // true when browser was launched (vs connected via CDP)
|
|
135
|
+
private _browserLaunched = false; // true when browser was launched via test injection (vs connected via CDP)
|
|
132
136
|
private browserCdpSession: CDPSession | null = null;
|
|
133
137
|
private browserWindowId: number | null = null;
|
|
134
|
-
private cdpRequestResolvers = new Map<
|
|
138
|
+
private cdpRequestResolvers = new Map<
|
|
139
|
+
string,
|
|
140
|
+
(response: { success: boolean; declined?: boolean }) => void
|
|
141
|
+
>();
|
|
135
142
|
private interactiveModeSessions = new Set<string>();
|
|
136
143
|
private handoffResolvers = new Map<string, () => void>();
|
|
137
|
-
private sessionSenders = new Map<
|
|
144
|
+
private sessionSenders = new Map<
|
|
145
|
+
string,
|
|
146
|
+
(msg: { type: string; sessionId: string }) => void
|
|
147
|
+
>();
|
|
138
148
|
private downloads = new Map<string, DownloadInfo[]>();
|
|
139
|
-
private pendingDownloads = new Map<
|
|
149
|
+
private pendingDownloads = new Map<
|
|
150
|
+
string,
|
|
151
|
+
{ resolve: (info: DownloadInfo) => void; reject: (err: Error) => void }[]
|
|
152
|
+
>();
|
|
140
153
|
|
|
141
|
-
|
|
142
|
-
return this._browserMode;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
/** Whether page.route() is supported. False only for connectOverCDP browsers. */
|
|
154
|
+
/** Whether page.route() is supported. False for connectOverCDP browsers. */
|
|
146
155
|
get supportsRouteInterception(): boolean {
|
|
147
|
-
|
|
156
|
+
// page.route() only works with launched browsers (test injection), not CDP-connected ones
|
|
157
|
+
return this._browserLaunched;
|
|
148
158
|
}
|
|
149
159
|
|
|
150
|
-
registerSender(
|
|
160
|
+
registerSender(
|
|
161
|
+
sessionId: string,
|
|
162
|
+
sendToClient: (msg: { type: string; sessionId: string }) => void,
|
|
163
|
+
): void {
|
|
151
164
|
this.sessionSenders.set(sessionId, sendToClient);
|
|
152
165
|
}
|
|
153
166
|
|
|
@@ -155,16 +168,12 @@ class BrowserManager {
|
|
|
155
168
|
this.sessionSenders.delete(sessionId);
|
|
156
169
|
}
|
|
157
170
|
|
|
158
|
-
setBrowserMode(mode: 'headless' | 'cdp', cdpUrl?: string): void {
|
|
159
|
-
this._browserMode = mode;
|
|
160
|
-
if (cdpUrl) this.cdpUrl = cdpUrl;
|
|
161
|
-
log.info({ mode, cdpUrl: this.cdpUrl }, 'Browser mode set');
|
|
162
|
-
}
|
|
163
|
-
|
|
164
171
|
async detectCDP(url?: string): Promise<boolean> {
|
|
165
172
|
const target = url || this.cdpUrl;
|
|
166
173
|
try {
|
|
167
|
-
const response = await fetch(`${target}/json/version`, {
|
|
174
|
+
const response = await fetch(`${target}/json/version`, {
|
|
175
|
+
signal: AbortSignal.timeout(3000),
|
|
176
|
+
});
|
|
168
177
|
return response.ok;
|
|
169
178
|
} catch {
|
|
170
179
|
return false;
|
|
@@ -175,7 +184,10 @@ class BrowserManager {
|
|
|
175
184
|
* Request Chrome restart from client via IPC. Returns true if client confirmed and CDP is now available.
|
|
176
185
|
* The sendToClient callback sends the request, and resolveCDPResponse() is called when the response arrives.
|
|
177
186
|
*/
|
|
178
|
-
async requestCDPFromClient(
|
|
187
|
+
async requestCDPFromClient(
|
|
188
|
+
sessionId: string,
|
|
189
|
+
sendToClient: (msg: { type: string; sessionId: string }) => void,
|
|
190
|
+
): Promise<boolean> {
|
|
179
191
|
// Cancel any existing pending request for this session to avoid leaked promises
|
|
180
192
|
const existing = this.cdpRequestResolvers.get(sessionId);
|
|
181
193
|
if (existing) {
|
|
@@ -201,126 +213,97 @@ class BrowserManager {
|
|
|
201
213
|
}, 15_000);
|
|
202
214
|
|
|
203
215
|
this.cdpRequestResolvers.set(sessionId, resolver);
|
|
204
|
-
sendToClient({ type:
|
|
216
|
+
sendToClient({ type: "browser_cdp_request", sessionId });
|
|
205
217
|
});
|
|
206
218
|
}
|
|
207
219
|
|
|
208
220
|
/**
|
|
209
221
|
* Called when a browser_cdp_response message arrives from the client.
|
|
210
222
|
*/
|
|
211
|
-
resolveCDPResponse(
|
|
223
|
+
resolveCDPResponse(
|
|
224
|
+
sessionId: string,
|
|
225
|
+
success: boolean,
|
|
226
|
+
declined?: boolean,
|
|
227
|
+
): void {
|
|
212
228
|
const resolver = this.cdpRequestResolvers.get(sessionId);
|
|
213
229
|
if (resolver) {
|
|
214
230
|
resolver({ success, declined });
|
|
215
231
|
}
|
|
216
232
|
}
|
|
217
233
|
|
|
218
|
-
private async ensureContext(
|
|
234
|
+
private async ensureContext(
|
|
235
|
+
invokingSessionId?: string,
|
|
236
|
+
): Promise<BrowserContext> {
|
|
219
237
|
if (this.context) return this.context;
|
|
220
238
|
if (this.contextCreating) return this.contextCreating;
|
|
221
239
|
|
|
222
240
|
this.contextCreating = (async () => {
|
|
223
241
|
// Deterministic test mode: when launch is injected via setLaunchFn,
|
|
224
242
|
// bypass ambient CDP probing/negotiation and use the injected launcher.
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
useCdp = true;
|
|
237
|
-
} else if (invokingSessionId && sender) {
|
|
238
|
-
log.info({ sessionId: invokingSessionId }, 'Requesting CDP from client');
|
|
239
|
-
const accepted = await this.requestCDPFromClient(invokingSessionId, sender);
|
|
240
|
-
if (accepted) {
|
|
241
|
-
const nowAvailable = await this.detectCDP();
|
|
242
|
-
if (nowAvailable) {
|
|
243
|
-
useCdp = true;
|
|
244
|
-
} else {
|
|
245
|
-
log.warn('Client accepted CDP request but CDP not detected');
|
|
246
|
-
}
|
|
247
|
-
} else {
|
|
248
|
-
log.info('Client declined CDP request');
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
}
|
|
243
|
+
if (launchPersistentContext != null) {
|
|
244
|
+
const profileDir = getProfileDir();
|
|
245
|
+
mkdirSync(profileDir, { recursive: true });
|
|
246
|
+
await authSessionCache.load();
|
|
247
|
+
const ctx = await launchPersistentContext(profileDir, {
|
|
248
|
+
headless: false,
|
|
249
|
+
});
|
|
250
|
+
this._browserLaunched = true;
|
|
251
|
+
log.info({ profileDir }, "Browser context created (test injection)");
|
|
252
|
+
return ctx;
|
|
253
|
+
}
|
|
252
254
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
255
|
+
// Try to detect an existing Chrome with --remote-debugging-port,
|
|
256
|
+
// or ask the client to launch Chrome with CDP enabled.
|
|
257
|
+
const sender = invokingSessionId
|
|
258
|
+
? this.sessionSenders.get(invokingSessionId)
|
|
259
|
+
: undefined;
|
|
260
|
+
let cdpAvailable = await this.detectCDP();
|
|
261
|
+
|
|
262
|
+
if (!cdpAvailable && invokingSessionId && sender) {
|
|
263
|
+
log.info(
|
|
264
|
+
{ sessionId: invokingSessionId },
|
|
265
|
+
"Requesting CDP from client",
|
|
266
|
+
);
|
|
267
|
+
const accepted = await this.requestCDPFromClient(
|
|
268
|
+
invokingSessionId,
|
|
269
|
+
sender,
|
|
270
|
+
);
|
|
271
|
+
if (accepted) {
|
|
272
|
+
cdpAvailable = await this.detectCDP();
|
|
273
|
+
if (!cdpAvailable) {
|
|
274
|
+
log.warn("Client accepted CDP request but CDP not detected");
|
|
268
275
|
}
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
const willBeHeaded = canDisplayGui();
|
|
273
|
-
log.info(
|
|
274
|
-
{ sessionId: invokingSessionId, willBeHeaded },
|
|
275
|
-
willBeHeaded
|
|
276
|
-
? 'CDP unavailable/declined; launching visible browser (display available)'
|
|
277
|
-
: 'CDP unavailable/declined; staying in headless mode (no display available)',
|
|
278
|
-
);
|
|
276
|
+
} else {
|
|
277
|
+
log.info("Client declined CDP request");
|
|
278
|
+
throw new Error("Browser access was declined by user");
|
|
279
279
|
}
|
|
280
280
|
}
|
|
281
281
|
|
|
282
|
-
|
|
283
|
-
|
|
282
|
+
if (!cdpAvailable) {
|
|
283
|
+
throw new Error(
|
|
284
|
+
"Chrome with remote debugging is not available. Please launch Chrome with --remote-debugging-port=9222.",
|
|
285
|
+
);
|
|
286
|
+
}
|
|
284
287
|
|
|
285
288
|
// Initialize auth session cache alongside browser context
|
|
286
289
|
await authSessionCache.load();
|
|
287
290
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
const
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
proc.kill();
|
|
304
|
-
reject(new Error(`Chromium install timed out after ${timeoutMs / 1000}s`));
|
|
305
|
-
}, timeoutMs),
|
|
306
|
-
),
|
|
307
|
-
]);
|
|
308
|
-
if (exitCode === 0) {
|
|
309
|
-
log.info('Chromium installed successfully');
|
|
310
|
-
} else {
|
|
311
|
-
const stderr = await new Response(proc.stderr).text();
|
|
312
|
-
const msg = stderr.trim() || `exited with code ${exitCode}`;
|
|
313
|
-
throw new Error(`Failed to install Chromium: ${msg}`);
|
|
314
|
-
}
|
|
315
|
-
}
|
|
291
|
+
try {
|
|
292
|
+
const pw = await import("playwright");
|
|
293
|
+
const browser = await pw.chromium.connectOverCDP(this.cdpUrl, {
|
|
294
|
+
timeout: 10_000,
|
|
295
|
+
});
|
|
296
|
+
this.cdpBrowser = browser;
|
|
297
|
+
this._browserLaunched = false;
|
|
298
|
+
const contexts = browser.contexts();
|
|
299
|
+
const ctx = contexts[0] || (await browser.newContext());
|
|
300
|
+
await this.initBrowserCdpSession();
|
|
301
|
+
log.info({ cdpUrl: this.cdpUrl }, "Connected to Chrome via CDP");
|
|
302
|
+
return ctx as unknown as BrowserContext;
|
|
303
|
+
} catch (err) {
|
|
304
|
+
log.warn({ err }, "CDP connectOverCDP failed");
|
|
305
|
+
throw new Error(`Failed to connect to Chrome via CDP: ${err}`);
|
|
316
306
|
}
|
|
317
|
-
|
|
318
|
-
const launch = launchPersistentContext ?? await getDefaultLaunchFn();
|
|
319
|
-
const headless = !canDisplayGui();
|
|
320
|
-
const ctx = await launch(profileDir, { headless });
|
|
321
|
-
this._browserLaunched = true;
|
|
322
|
-
log.info({ profileDir, headless }, headless ? 'Browser context created (headless)' : 'Browser context created (visible)');
|
|
323
|
-
return ctx;
|
|
324
307
|
})();
|
|
325
308
|
|
|
326
309
|
try {
|
|
@@ -332,9 +315,9 @@ class BrowserManager {
|
|
|
332
315
|
on?: (event: string, handler: (...args: unknown[]) => void) => void;
|
|
333
316
|
off?: (event: string, handler: (...args: unknown[]) => void) => void;
|
|
334
317
|
};
|
|
335
|
-
if (typeof rawCtx.on ===
|
|
318
|
+
if (typeof rawCtx.on === "function") {
|
|
336
319
|
this.contextCloseHandler = () => {
|
|
337
|
-
log.warn(
|
|
320
|
+
log.warn("Browser context closed unexpectedly, resetting state");
|
|
338
321
|
this.context = null;
|
|
339
322
|
this.contextCloseHandler = null;
|
|
340
323
|
this.browserCdpSession = null;
|
|
@@ -350,15 +333,16 @@ class BrowserManager {
|
|
|
350
333
|
this.pages.clear();
|
|
351
334
|
this.rawPages.clear();
|
|
352
335
|
this.cdpSessions.clear();
|
|
353
|
-
|
|
336
|
+
|
|
354
337
|
this.snapshotMaps.clear();
|
|
355
338
|
this.downloads.clear();
|
|
356
339
|
for (const pending of this.pendingDownloads.values()) {
|
|
357
|
-
for (const waiter of pending)
|
|
340
|
+
for (const waiter of pending)
|
|
341
|
+
waiter.reject(new Error("Browser closed"));
|
|
358
342
|
}
|
|
359
343
|
this.pendingDownloads.clear();
|
|
360
344
|
};
|
|
361
|
-
rawCtx.on(
|
|
345
|
+
rawCtx.on("close", this.contextCloseHandler);
|
|
362
346
|
}
|
|
363
347
|
|
|
364
348
|
return this.context;
|
|
@@ -384,17 +368,24 @@ class BrowserManager {
|
|
|
384
368
|
// In connectOverCDP mode, Chrome often starts with a pre-opened blank tab.
|
|
385
369
|
// Only reuse blank/new-tab pages to avoid hijacking active user tabs, which
|
|
386
370
|
// could cause user-visible disruption or data loss when the session closes.
|
|
387
|
-
if (
|
|
388
|
-
const BLANK_TAB_URLS = new Set([
|
|
371
|
+
if (!this._browserLaunched && typeof context.pages === "function") {
|
|
372
|
+
const BLANK_TAB_URLS = new Set([
|
|
373
|
+
"about:blank",
|
|
374
|
+
"chrome://newtab/",
|
|
375
|
+
"chrome://new-tab-page/",
|
|
376
|
+
]);
|
|
389
377
|
const claimedPages = new Set(this.pages.values());
|
|
390
378
|
const reusable = context.pages().find((p) => {
|
|
391
379
|
if (p.isClosed() || claimedPages.has(p)) return false;
|
|
392
380
|
const url = p.url();
|
|
393
|
-
return BLANK_TAB_URLS.has(url) || url ===
|
|
381
|
+
return BLANK_TAB_URLS.has(url) || url === "";
|
|
394
382
|
});
|
|
395
383
|
if (reusable) {
|
|
396
384
|
page = reusable;
|
|
397
|
-
log.debug(
|
|
385
|
+
log.debug(
|
|
386
|
+
{ sessionId, url: reusable.url() },
|
|
387
|
+
"Reusing blank CDP tab instead of creating a new page",
|
|
388
|
+
);
|
|
398
389
|
}
|
|
399
390
|
}
|
|
400
391
|
|
|
@@ -408,22 +399,28 @@ class BrowserManager {
|
|
|
408
399
|
// For launched browsers (not CDP-connected), create a page-level CDP session
|
|
409
400
|
// so we can position the browser window. Browser domain commands (setWindowBounds,
|
|
410
401
|
// getWindowForTarget) are accessible from page-level CDP sessions.
|
|
411
|
-
if (!this.browserCdpSession && this._browserLaunched
|
|
402
|
+
if (!this.browserCdpSession && this._browserLaunched) {
|
|
412
403
|
try {
|
|
413
404
|
const rawPage = page as unknown as RawPlaywrightPage;
|
|
414
405
|
this.browserCdpSession = await rawPage.context().newCDPSession(rawPage);
|
|
415
406
|
await this.ensureBrowserWindowId();
|
|
416
407
|
} catch (err) {
|
|
417
|
-
log.warn(
|
|
408
|
+
log.warn(
|
|
409
|
+
{ err },
|
|
410
|
+
"Failed to create CDP session for window positioning",
|
|
411
|
+
);
|
|
418
412
|
}
|
|
419
413
|
}
|
|
420
414
|
|
|
421
415
|
// Position the browser window so the user can watch.
|
|
422
|
-
if (
|
|
416
|
+
if (
|
|
417
|
+
this.browserCdpSession &&
|
|
418
|
+
!this.interactiveModeSessions.has(sessionId)
|
|
419
|
+
) {
|
|
423
420
|
await this.positionWindowSidebar();
|
|
424
421
|
}
|
|
425
422
|
|
|
426
|
-
log.debug({ sessionId },
|
|
423
|
+
log.debug({ sessionId }, "Session page created");
|
|
427
424
|
return page;
|
|
428
425
|
}
|
|
429
426
|
|
|
@@ -447,10 +444,10 @@ class BrowserManager {
|
|
|
447
444
|
// Reject any pending download waiters
|
|
448
445
|
const pending = this.pendingDownloads.get(sessionId);
|
|
449
446
|
if (pending) {
|
|
450
|
-
for (const waiter of pending) waiter.reject(new Error(
|
|
447
|
+
for (const waiter of pending) waiter.reject(new Error("Session closed"));
|
|
451
448
|
this.pendingDownloads.delete(sessionId);
|
|
452
449
|
}
|
|
453
|
-
log.debug({ sessionId },
|
|
450
|
+
log.debug({ sessionId }, "Session page closed");
|
|
454
451
|
}
|
|
455
452
|
|
|
456
453
|
async closeAllPages(): Promise<void> {
|
|
@@ -459,7 +456,7 @@ class BrowserManager {
|
|
|
459
456
|
try {
|
|
460
457
|
await this.stopScreencast(sessionId);
|
|
461
458
|
} catch (err) {
|
|
462
|
-
log.warn({ err, sessionId },
|
|
459
|
+
log.warn({ err, sessionId }, "Failed to stop screencast");
|
|
463
460
|
}
|
|
464
461
|
}
|
|
465
462
|
|
|
@@ -468,7 +465,7 @@ class BrowserManager {
|
|
|
468
465
|
try {
|
|
469
466
|
await page.close();
|
|
470
467
|
} catch (err) {
|
|
471
|
-
log.warn({ err, sessionId },
|
|
468
|
+
log.warn({ err, sessionId }, "Failed to close page");
|
|
472
469
|
}
|
|
473
470
|
}
|
|
474
471
|
}
|
|
@@ -477,7 +474,7 @@ class BrowserManager {
|
|
|
477
474
|
this.snapshotMaps.clear();
|
|
478
475
|
this.downloads.clear();
|
|
479
476
|
for (const pending of this.pendingDownloads.values()) {
|
|
480
|
-
for (const waiter of pending) waiter.reject(new Error(
|
|
477
|
+
for (const waiter of pending) waiter.reject(new Error("Browser closed"));
|
|
481
478
|
}
|
|
482
479
|
this.pendingDownloads.clear();
|
|
483
480
|
|
|
@@ -485,33 +482,40 @@ class BrowserManager {
|
|
|
485
482
|
// Remove the close listener before intentional close to avoid
|
|
486
483
|
// the handler firing and clearing state we're already cleaning up.
|
|
487
484
|
if (this.contextCloseHandler) {
|
|
488
|
-
const rawCtx = this.context as unknown as {
|
|
489
|
-
|
|
490
|
-
|
|
485
|
+
const rawCtx = this.context as unknown as {
|
|
486
|
+
off?: (event: string, handler: (...args: unknown[]) => void) => void;
|
|
487
|
+
};
|
|
488
|
+
if (typeof rawCtx.off === "function") {
|
|
489
|
+
rawCtx.off("close", this.contextCloseHandler);
|
|
491
490
|
}
|
|
492
491
|
this.contextCloseHandler = null;
|
|
493
492
|
}
|
|
494
493
|
try {
|
|
495
494
|
await this.context.close();
|
|
496
495
|
} catch (err) {
|
|
497
|
-
log.warn({ err },
|
|
496
|
+
log.warn({ err }, "Failed to close browser context");
|
|
498
497
|
}
|
|
499
498
|
this.context = null;
|
|
500
|
-
log.info(
|
|
499
|
+
log.info("Browser context closed");
|
|
501
500
|
}
|
|
502
501
|
|
|
503
502
|
// Detach browser-level CDP session used for window management
|
|
504
503
|
if (this.browserCdpSession) {
|
|
505
504
|
try {
|
|
506
505
|
await this.browserCdpSession.detach();
|
|
507
|
-
} catch (e) {
|
|
506
|
+
} catch (e) {
|
|
507
|
+
log.debug({ err: e }, "CDP session detach failed during shutdown");
|
|
508
|
+
}
|
|
508
509
|
this.browserCdpSession = null;
|
|
509
510
|
this.browserWindowId = null;
|
|
510
511
|
}
|
|
511
512
|
|
|
512
513
|
// Close or disconnect CDP browser connection if present
|
|
513
514
|
if (this.cdpBrowser) {
|
|
514
|
-
const b = this.cdpBrowser as {
|
|
515
|
+
const b = this.cdpBrowser as {
|
|
516
|
+
close?: () => Promise<void>;
|
|
517
|
+
disconnect?: () => Promise<void>;
|
|
518
|
+
};
|
|
515
519
|
const wasLaunched = this._browserLaunched;
|
|
516
520
|
this.cdpBrowser = null;
|
|
517
521
|
this._browserLaunched = false;
|
|
@@ -524,62 +528,27 @@ class BrowserManager {
|
|
|
524
528
|
await b.disconnect?.();
|
|
525
529
|
}
|
|
526
530
|
} catch (err) {
|
|
527
|
-
log.warn({ err },
|
|
531
|
+
log.warn({ err }, "Failed to close/disconnect CDP browser");
|
|
528
532
|
}
|
|
529
533
|
}
|
|
530
534
|
}
|
|
531
535
|
|
|
532
|
-
async startScreencast(sessionId: string, onFrame: (frame: { data: string; metadata: ScreencastFrameMetadata }) => void): Promise<void> {
|
|
533
|
-
const rawPage = this.rawPages.get(sessionId) as RawPlaywrightPage | undefined;
|
|
534
|
-
if (!rawPage) throw new Error('No page for session');
|
|
535
|
-
|
|
536
|
-
// Stop any existing screencast before creating a new CDP session
|
|
537
|
-
await this.stopScreencast(sessionId);
|
|
538
|
-
|
|
539
|
-
const cdp = await rawPage.context().newCDPSession(rawPage);
|
|
540
|
-
this.cdpSessions.set(sessionId, cdp);
|
|
541
|
-
this.screencastCallbacks.set(sessionId, onFrame);
|
|
542
|
-
|
|
543
|
-
// Keep screencast intentionally low-frequency to avoid Chrome renderer /
|
|
544
|
-
// WindowServer spikes while users type in interactive auth flows.
|
|
545
|
-
const MIN_FRAME_INTERVAL_MS = 1000;
|
|
546
|
-
let lastFrameTime = 0;
|
|
547
|
-
|
|
548
|
-
cdp.on('Page.screencastFrame', (params) => {
|
|
549
|
-
const now = Date.now();
|
|
550
|
-
if (now - lastFrameTime >= MIN_FRAME_INTERVAL_MS) {
|
|
551
|
-
lastFrameTime = now;
|
|
552
|
-
onFrame({ data: params.data as string, metadata: params.metadata as ScreencastFrameMetadata });
|
|
553
|
-
}
|
|
554
|
-
// Always ack so CDP continues delivering frames (otherwise it stalls)
|
|
555
|
-
silentlyWithLog(cdp.send('Page.screencastFrameAck', { sessionId: params.sessionId }), 'screencast frame ack');
|
|
556
|
-
});
|
|
557
|
-
|
|
558
|
-
await cdp.send('Page.startScreencast', {
|
|
559
|
-
format: 'jpeg',
|
|
560
|
-
quality: 45,
|
|
561
|
-
maxWidth: SCREENCAST_WIDTH,
|
|
562
|
-
maxHeight: SCREENCAST_HEIGHT,
|
|
563
|
-
everyNthFrame: 4,
|
|
564
|
-
});
|
|
565
|
-
}
|
|
566
|
-
|
|
567
536
|
async stopScreencast(sessionId: string): Promise<void> {
|
|
568
537
|
const cdp = this.cdpSessions.get(sessionId);
|
|
569
538
|
if (cdp) {
|
|
570
539
|
try {
|
|
571
|
-
await cdp.send(
|
|
540
|
+
await cdp.send("Page.stopScreencast");
|
|
572
541
|
await cdp.detach();
|
|
573
|
-
} catch (e) {
|
|
542
|
+
} catch (e) {
|
|
543
|
+
log.debug(
|
|
544
|
+
{ err: e },
|
|
545
|
+
"Screencast stop / CDP detach failed during cleanup",
|
|
546
|
+
);
|
|
547
|
+
}
|
|
574
548
|
this.cdpSessions.delete(sessionId);
|
|
575
|
-
this.screencastCallbacks.delete(sessionId);
|
|
576
549
|
}
|
|
577
550
|
}
|
|
578
551
|
|
|
579
|
-
isScreencasting(sessionId: string): boolean {
|
|
580
|
-
return this.cdpSessions.has(sessionId);
|
|
581
|
-
}
|
|
582
|
-
|
|
583
552
|
storeSnapshotMap(sessionId: string, map: Map<string, string>): void {
|
|
584
553
|
this.snapshotMaps.set(sessionId, map);
|
|
585
554
|
}
|
|
@@ -601,14 +570,16 @@ class BrowserManager {
|
|
|
601
570
|
private async initBrowserCdpSession(): Promise<void> {
|
|
602
571
|
if (!this.cdpBrowser) return;
|
|
603
572
|
try {
|
|
604
|
-
const browser = this.cdpBrowser as {
|
|
605
|
-
|
|
573
|
+
const browser = this.cdpBrowser as {
|
|
574
|
+
newBrowserCDPSession?: () => Promise<CDPSession>;
|
|
575
|
+
};
|
|
576
|
+
if (typeof browser.newBrowserCDPSession !== "function") return;
|
|
606
577
|
|
|
607
578
|
this.browserCdpSession = await browser.newBrowserCDPSession();
|
|
608
579
|
this.browserWindowId = null;
|
|
609
580
|
await this.ensureBrowserWindowId();
|
|
610
581
|
} catch (err) {
|
|
611
|
-
log.warn({ err },
|
|
582
|
+
log.warn({ err }, "Failed to init browser CDP session");
|
|
612
583
|
}
|
|
613
584
|
}
|
|
614
585
|
|
|
@@ -616,19 +587,29 @@ class BrowserManager {
|
|
|
616
587
|
if (!this.browserCdpSession) return null;
|
|
617
588
|
if (this.browserWindowId != null) return this.browserWindowId;
|
|
618
589
|
try {
|
|
619
|
-
const targets = await this.browserCdpSession.send(
|
|
590
|
+
const targets = (await this.browserCdpSession.send(
|
|
591
|
+
"Target.getTargets",
|
|
592
|
+
)) as {
|
|
620
593
|
targetInfos: Array<{ targetId: string; type: string }>;
|
|
621
594
|
};
|
|
622
|
-
const pageTarget = targets.targetInfos.find(
|
|
595
|
+
const pageTarget = targets.targetInfos.find(
|
|
596
|
+
(t: { type: string }) => t.type === "page",
|
|
597
|
+
);
|
|
623
598
|
if (!pageTarget) return null;
|
|
624
|
-
const result = await this.browserCdpSession.send(
|
|
625
|
-
|
|
626
|
-
|
|
599
|
+
const result = (await this.browserCdpSession.send(
|
|
600
|
+
"Browser.getWindowForTarget",
|
|
601
|
+
{
|
|
602
|
+
targetId: pageTarget.targetId,
|
|
603
|
+
},
|
|
604
|
+
)) as { windowId: number };
|
|
627
605
|
this.browserWindowId = result.windowId;
|
|
628
|
-
log.debug(
|
|
606
|
+
log.debug(
|
|
607
|
+
{ windowId: this.browserWindowId },
|
|
608
|
+
"Got browser window ID via CDP",
|
|
609
|
+
);
|
|
629
610
|
return this.browserWindowId;
|
|
630
611
|
} catch (err) {
|
|
631
|
-
log.warn({ err },
|
|
612
|
+
log.warn({ err }, "Failed to resolve browser window ID");
|
|
632
613
|
return null;
|
|
633
614
|
}
|
|
634
615
|
}
|
|
@@ -642,13 +623,19 @@ class BrowserManager {
|
|
|
642
623
|
const windowId = await this.ensureBrowserWindowId();
|
|
643
624
|
if (windowId == null) return;
|
|
644
625
|
try {
|
|
645
|
-
await this.browserCdpSession.send(
|
|
626
|
+
await this.browserCdpSession.send("Browser.setWindowBounds", {
|
|
646
627
|
windowId,
|
|
647
|
-
bounds: {
|
|
628
|
+
bounds: {
|
|
629
|
+
left: 480,
|
|
630
|
+
top: 40,
|
|
631
|
+
width: 940,
|
|
632
|
+
height: 700,
|
|
633
|
+
windowState: "normal",
|
|
634
|
+
},
|
|
648
635
|
});
|
|
649
|
-
log.debug(
|
|
636
|
+
log.debug("positionWindowSidebar: placed browser window in top-right");
|
|
650
637
|
} catch (err) {
|
|
651
|
-
log.warn({ err },
|
|
638
|
+
log.warn({ err }, "positionWindowSidebar: failed to position window");
|
|
652
639
|
// CDP session may be stale (e.g. page closed) — clear it so it gets recreated
|
|
653
640
|
this.browserCdpSession = null;
|
|
654
641
|
this.browserWindowId = null;
|
|
@@ -663,13 +650,19 @@ class BrowserManager {
|
|
|
663
650
|
const windowId = await this.ensureBrowserWindowId();
|
|
664
651
|
if (windowId == null) return;
|
|
665
652
|
try {
|
|
666
|
-
await this.browserCdpSession.send(
|
|
653
|
+
await this.browserCdpSession.send("Browser.setWindowBounds", {
|
|
667
654
|
windowId,
|
|
668
|
-
bounds: {
|
|
655
|
+
bounds: {
|
|
656
|
+
left: 200,
|
|
657
|
+
top: 40,
|
|
658
|
+
width: 1100,
|
|
659
|
+
height: 820,
|
|
660
|
+
windowState: "normal",
|
|
661
|
+
},
|
|
669
662
|
});
|
|
670
|
-
log.debug(
|
|
663
|
+
log.debug("moveWindowOnscreen: moved window onscreen via CDP");
|
|
671
664
|
} catch (err) {
|
|
672
|
-
log.warn({ err },
|
|
665
|
+
log.warn({ err }, "moveWindowOnscreen: CDP setWindowBounds failed");
|
|
673
666
|
}
|
|
674
667
|
}
|
|
675
668
|
|
|
@@ -690,7 +683,10 @@ class BrowserManager {
|
|
|
690
683
|
}
|
|
691
684
|
}
|
|
692
685
|
|
|
693
|
-
async waitForHandoffComplete(
|
|
686
|
+
async waitForHandoffComplete(
|
|
687
|
+
sessionId: string,
|
|
688
|
+
timeoutMs: number = 300_000,
|
|
689
|
+
): Promise<void> {
|
|
694
690
|
if (!this.interactiveModeSessions.has(sessionId)) return;
|
|
695
691
|
|
|
696
692
|
// Cancel any existing pending handoff for this session
|
|
@@ -738,7 +734,10 @@ class BrowserManager {
|
|
|
738
734
|
}
|
|
739
735
|
const currentUrl = page.url();
|
|
740
736
|
if (currentUrl !== initialUrl) {
|
|
741
|
-
log.info(
|
|
737
|
+
log.info(
|
|
738
|
+
{ sessionId, from: initialUrl, to: currentUrl },
|
|
739
|
+
"Handoff auto-resolved: URL changed",
|
|
740
|
+
);
|
|
742
741
|
this.interactiveModeSessions.delete(sessionId);
|
|
743
742
|
resolver();
|
|
744
743
|
}
|
|
@@ -776,7 +775,9 @@ class BrowserManager {
|
|
|
776
775
|
async extractCookies(domain?: string): Promise<ExtractedCredential[]> {
|
|
777
776
|
if (!this.browserCdpSession) return [];
|
|
778
777
|
try {
|
|
779
|
-
const result = await this.browserCdpSession.send(
|
|
778
|
+
const result = (await this.browserCdpSession.send(
|
|
779
|
+
"Network.getAllCookies",
|
|
780
|
+
)) as {
|
|
780
781
|
cookies: Array<{
|
|
781
782
|
name: string;
|
|
782
783
|
value: string;
|
|
@@ -790,14 +791,15 @@ class BrowserManager {
|
|
|
790
791
|
|
|
791
792
|
let cookies = result.cookies ?? [];
|
|
792
793
|
if (domain) {
|
|
793
|
-
cookies = cookies.filter(
|
|
794
|
-
c
|
|
795
|
-
|
|
796
|
-
|
|
794
|
+
cookies = cookies.filter(
|
|
795
|
+
(c) =>
|
|
796
|
+
c.domain === domain ||
|
|
797
|
+
c.domain === `.${domain}` ||
|
|
798
|
+
c.domain.endsWith(`.${domain}`),
|
|
797
799
|
);
|
|
798
800
|
}
|
|
799
801
|
|
|
800
|
-
return cookies.map(c => ({
|
|
802
|
+
return cookies.map((c) => ({
|
|
801
803
|
name: c.name,
|
|
802
804
|
value: c.value,
|
|
803
805
|
domain: c.domain,
|
|
@@ -807,13 +809,13 @@ class BrowserManager {
|
|
|
807
809
|
expires: c.expires > 0 ? c.expires : undefined,
|
|
808
810
|
}));
|
|
809
811
|
} catch (err) {
|
|
810
|
-
log.warn({ err },
|
|
812
|
+
log.warn({ err }, "Failed to extract cookies via CDP");
|
|
811
813
|
return [];
|
|
812
814
|
}
|
|
813
815
|
}
|
|
814
816
|
|
|
815
817
|
private setupDownloadTracking(sessionId: string, page: Page): void {
|
|
816
|
-
page.on(
|
|
818
|
+
page.on("download", async (download: unknown) => {
|
|
817
819
|
const dl = download as {
|
|
818
820
|
suggestedFilename(): string;
|
|
819
821
|
path(): Promise<string | null>;
|
|
@@ -838,16 +840,18 @@ class BrowserManager {
|
|
|
838
840
|
this.downloads.set(sessionId, list);
|
|
839
841
|
}
|
|
840
842
|
|
|
841
|
-
log.info({ sessionId, filename, path: destPath },
|
|
843
|
+
log.info({ sessionId, filename, path: destPath }, "Download completed");
|
|
842
844
|
} catch (err) {
|
|
843
845
|
const failure = await dl.failure();
|
|
844
|
-
log.warn({ err, failure, sessionId },
|
|
846
|
+
log.warn({ err, failure, sessionId }, "Download failed");
|
|
845
847
|
|
|
846
848
|
// Reject any pending waiters
|
|
847
849
|
const pending = this.pendingDownloads.get(sessionId);
|
|
848
850
|
if (pending && pending.length > 0) {
|
|
849
851
|
const waiter = pending.shift()!;
|
|
850
|
-
waiter.reject(
|
|
852
|
+
waiter.reject(
|
|
853
|
+
new Error(`Download failed: ${failure ?? String(err)}`),
|
|
854
|
+
);
|
|
851
855
|
if (pending.length === 0) this.pendingDownloads.delete(sessionId);
|
|
852
856
|
}
|
|
853
857
|
}
|
|
@@ -860,7 +864,10 @@ class BrowserManager {
|
|
|
860
864
|
return list[list.length - 1];
|
|
861
865
|
}
|
|
862
866
|
|
|
863
|
-
waitForDownload(
|
|
867
|
+
waitForDownload(
|
|
868
|
+
sessionId: string,
|
|
869
|
+
timeoutMs: number = 30_000,
|
|
870
|
+
): Promise<DownloadInfo> {
|
|
864
871
|
// Check if an unconsumed download already completed for this session
|
|
865
872
|
const existing = this.downloads.get(sessionId);
|
|
866
873
|
if (existing && existing.length > 0) {
|
|
@@ -874,7 +881,7 @@ class BrowserManager {
|
|
|
874
881
|
// Remove this waiter from the pending list
|
|
875
882
|
const pending = this.pendingDownloads.get(sessionId);
|
|
876
883
|
if (pending) {
|
|
877
|
-
const idx = pending.findIndex(w => w.resolve === wrappedResolve);
|
|
884
|
+
const idx = pending.findIndex((w) => w.resolve === wrappedResolve);
|
|
878
885
|
if (idx >= 0) pending.splice(idx, 1);
|
|
879
886
|
if (pending.length === 0) this.pendingDownloads.delete(sessionId);
|
|
880
887
|
}
|