@omercnet/paseo-shared-browser 1.2.2-next.232.1 → 1.2.2-next.239.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +312 -25
- package/REMOTE_BROWSER_RESEARCH.md +395 -0
- package/STREAMING_DESIGN.md +191 -0
- package/client/browser-auto-capture-density.test.ts +102 -0
- package/client/browser-auto-capture-density.ts +30 -0
- package/client/browser-canvas-input.test.ts +198 -0
- package/client/browser-canvas-input.ts +202 -0
- package/client/browser-canvas-keyboard-web.test.ts +113 -0
- package/client/browser-canvas-keyboard-web.ts +230 -0
- package/client/browser-canvas-keyboard.test.ts +123 -0
- package/client/browser-canvas-keyboard.ts +126 -0
- package/client/browser-canvas-layout.test.ts +109 -0
- package/client/browser-canvas-layout.ts +51 -0
- package/client/browser-canvas-viewport-web.test.ts +77 -0
- package/client/browser-canvas-viewport-web.ts +63 -0
- package/client/browser-canvas-viewport.tsx +152 -0
- package/client/browser-capture-cadence.test.ts +9 -0
- package/client/browser-capture-cadence.ts +6 -0
- package/client/browser-capture-density-controls.tsx +75 -0
- package/client/browser-chrome.tsx +234 -0
- package/client/browser-client-presence.ts +184 -0
- package/client/browser-compose-controls.test.tsx +275 -0
- package/client/browser-compose-controls.tsx +174 -0
- package/client/browser-control-button.tsx +109 -0
- package/client/browser-control-dialog.tsx +71 -0
- package/client/browser-control-request.test.ts +27 -0
- package/client/browser-control-request.ts +57 -0
- package/client/browser-control-tooltip-web.ts +10 -0
- package/client/browser-dialog-body.tsx +22 -0
- package/client/browser-emulation-mode.test.ts +75 -0
- package/client/browser-emulation-mode.ts +98 -0
- package/client/browser-fill-viewport.test.ts +45 -0
- package/client/browser-fill-viewport.ts +28 -0
- package/client/browser-frame-buffer.test.ts +157 -0
- package/client/browser-frame-buffer.ts +148 -0
- package/client/browser-frame-image.tsx +62 -0
- package/client/browser-image-load.test.ts +23 -0
- package/client/browser-image-load.ts +26 -0
- package/client/browser-input-queue.test.ts +1201 -0
- package/client/browser-input-queue.ts +505 -0
- package/client/browser-menu-placement.test.ts +90 -0
- package/client/browser-menu-placement.ts +78 -0
- package/client/browser-panel-styles.ts +384 -0
- package/client/browser-quality-controls.test.ts +26 -0
- package/client/browser-quality-controls.tsx +139 -0
- package/client/browser-resolution-picker.tsx +189 -0
- package/client/browser-tabs.tsx +117 -0
- package/client/browser-toolbar-menu-web.test.ts +185 -0
- package/client/browser-toolbar-menu-web.ts +159 -0
- package/client/browser-toolbar-menu.tsx +255 -0
- package/client/browser-toolbar-submenu.tsx +93 -0
- package/client/browser-video-bytes.test.ts +44 -0
- package/client/browser-video-decoder.test.ts +473 -0
- package/client/browser-video-decoder.ts +311 -0
- package/client/browser-video-surface.tsx +21 -0
- package/client/browser-video-visibility.test.ts +45 -0
- package/client/browser-viewer-recovery.test.ts +111 -0
- package/client/browser-viewer-recovery.ts +78 -0
- package/client/browser.tsx +1642 -1569
- package/client/favorite-star.tsx +23 -0
- package/client/frame-lifecycle.ts +62 -59
- package/client/use-browser-auto-capture-density.test.ts +104 -0
- package/client/use-browser-auto-capture-density.ts +43 -0
- package/client/use-browser-canvas-input.ts +370 -0
- package/client/use-browser-capture-density.ts +87 -0
- package/client/use-browser-emulation-mode.ts +56 -0
- package/client/use-browser-fill-viewport.test.ts +108 -0
- package/client/use-browser-fill-viewport.ts +62 -0
- package/client/use-browser-frame-buffer.ts +66 -0
- package/client/use-browser-image-capture.ts +139 -0
- package/client/use-browser-scoped-mutation.test.ts +118 -0
- package/client/use-browser-scoped-mutation.ts +59 -0
- package/client/use-browser-video.test.ts +991 -0
- package/client/use-browser-video.ts +474 -0
- package/client/use-browser-viewer-recovery.ts +26 -0
- package/client/use-resolution-favorites.ts +90 -0
- package/client/web.test.ts +125 -0
- package/client/web.ts +405 -0
- package/index.client.tsx +0 -10
- package/index.server.ts +32 -0
- package/npm-shrinkwrap.json +7 -7
- package/package.json +9 -5
- package/paseo-plugin.json +3 -1
- package/scripts/prepare-runtime.mjs +1 -1
- package/server/agent-browser-runtime.ts +1598 -314
- package/server/browser-cursor.test.ts +64 -0
- package/server/browser-cursor.ts +69 -0
- package/server/browser-frame-context.ts +22 -0
- package/server/browser-frame-epoch.test.ts +95 -0
- package/server/browser-frame-rejection.ts +17 -0
- package/server/browser-gesture.ts +120 -0
- package/server/browser-operation-diagnostics.test.ts +79 -0
- package/server/browser-operation-diagnostics.ts +103 -0
- package/server/browser-policy.ts +1313 -151
- package/server/browser-video-input-authority.test.ts +208 -0
- package/server/browser-video-policy.test.ts +564 -0
- package/server/browser-video-socket.test.ts +368 -0
- package/server/browser.ts +151 -16
- package/server/capture-transport-policy.test.ts +105 -0
- package/server/capture-transport-policy.ts +154 -0
- package/server/capture-transport-runtime.test.ts +213 -0
- package/server/daemon-launch-chain.test.ts +70 -0
- package/server/daemon-shutdown.test.ts +289 -0
- package/server/discrete-input-freshness.test.ts +364 -0
- package/server/discrete-input-native-lifetime.test.ts +150 -0
- package/server/display-loss-recovery.test.ts +294 -0
- package/server/emulation-restoration.test.ts +262 -0
- package/server/input-cleanup.ts +23 -0
- package/server/input-generation.test.ts +31 -0
- package/server/input-generation.ts +36 -0
- package/server/jpeg-capture-demand.test.ts +76 -0
- package/server/jpeg-capture-demand.ts +98 -0
- package/server/jpeg-frame.ts +29 -0
- package/server/keyboard-input.test.ts +61 -0
- package/server/keyboard-input.ts +104 -0
- package/server/live-input-policy.test.ts +958 -0
- package/server/live-input-runtime.test.ts +732 -0
- package/server/live-input-supervisor.test.ts +147 -0
- package/server/mcp-entry.ts +44 -2
- package/server/native-video-backpressure.test.ts +228 -0
- package/server/native-video-buffer.test.ts +141 -0
- package/server/native-video-buffer.ts +114 -0
- package/server/native-video-capture.test.ts +599 -0
- package/server/native-video-capture.ts +497 -0
- package/server/native-video-extension-source.test.ts +201 -0
- package/server/native-video-extension-source.ts +364 -0
- package/server/native-video-extension.test.ts +41 -0
- package/server/native-video-extension.ts +83 -0
- package/server/native-video-packet.ts +9 -0
- package/server/native-video-settings.test.ts +245 -0
- package/server/navigation-budget.ts +8 -0
- package/server/navigation-commit.ts +125 -0
- package/server/navigation-policy.test.ts +153 -0
- package/server/process-identity.ts +58 -0
- package/server/reattach-held-input.test.ts +143 -0
- package/server/runtime-frame-reuse.test.ts +295 -0
- package/server/runtime-jpeg-demand.test.ts +200 -0
- package/server/runtime-jpeg-shutdown.test.ts +210 -0
- package/server/runtime-jpeg-source-order.test.ts +201 -0
- package/server/runtime-launch-environment.test.ts +111 -0
- package/server/runtime-navigation.test.ts +327 -0
- package/server/runtime-owner-display.test.ts +152 -0
- package/server/runtime-owner.ts +305 -31
- package/server/runtime-page-state.test.ts +249 -0
- package/server/runtime-protocol.ts +31 -1
- package/server/runtime-target-retention.test.ts +181 -0
- package/server/runtime-video-opt-in.test.ts +82 -0
- package/server/runtime-video.test.ts +444 -0
- package/server/screencast-source-time.ts +46 -0
- package/server/stale-wheel-admission.test.ts +136 -0
- package/server/supervisor-client.ts +28 -4
- package/server/supervisor-heartbeat.test.ts +80 -0
- package/server/supervisor.ts +553 -55
- package/server/video-source-recovery.test.ts +20 -0
- package/server/video-source-recovery.ts +64 -0
- package/server/viewer-capture-lifetime.ts +51 -0
- package/server/virtual-display.test.ts +184 -0
- package/server/virtual-display.ts +236 -0
- package/shared/browser-display-preferences.ts +46 -0
- package/shared/browser-video.ts +80 -0
- package/shared/browser.ts +286 -65
- package/shared/capture-density.ts +16 -0
- package/shared/capture-settings.ts +17 -0
- package/shared/device-presets.ts +90 -0
- package/shared/resolution-menu.test.ts +120 -0
- package/shared/resolution-menu.ts +72 -0
- package/shared/video-settings.test.ts +38 -0
- package/shared/video-settings.ts +34 -0
- package/shared/viewport-limits.ts +5 -0
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
2
3
|
import { chmod, mkdir, readFile, rm } from "node:fs/promises";
|
|
3
4
|
import { isAbsolute, resolve } from "node:path";
|
|
4
5
|
import { promisify } from "node:util";
|
|
6
|
+
import { canUseCaptureDensity, captureDensitySchema } from "../shared/capture-density";
|
|
7
|
+
|
|
8
|
+
import { captureDimensions, DEFAULT_JPEG_QUALITY } from "../shared/capture-settings";
|
|
9
|
+
import { MAX_VIEWPORT } from "../shared/viewport-limits";
|
|
5
10
|
import {
|
|
6
11
|
attachToTarget,
|
|
7
12
|
CdpConnection,
|
|
@@ -12,13 +17,56 @@ import {
|
|
|
12
17
|
CdpUnknownOutcomeError,
|
|
13
18
|
listPageTargets,
|
|
14
19
|
} from "./cdp";
|
|
20
|
+
import { readJpegFrameDimensions } from "./jpeg-frame";
|
|
21
|
+
|
|
22
|
+
export { readJpegFrameDimensions as jpegDimensions } from "./jpeg-frame";
|
|
23
|
+
|
|
24
|
+
import {
|
|
25
|
+
type BrowserCursor,
|
|
26
|
+
type BrowserGestureKeyEvent,
|
|
27
|
+
browserCursorSchema,
|
|
28
|
+
browserGestureKeySchema,
|
|
29
|
+
MAX_HELD_BROWSER_KEYS,
|
|
30
|
+
} from "../shared/browser";
|
|
31
|
+
import { resolveVideoEncoderSettings } from "../shared/video-settings";
|
|
32
|
+
import { browserCursorExpression } from "./browser-cursor";
|
|
33
|
+
import { GESTURE_IDLE_MS, GESTURE_LIFETIME_MS } from "./browser-gesture";
|
|
34
|
+
import { CAPTURE_MAX_AGE_MS, createCaptureTransportPolicy } from "./capture-transport-policy";
|
|
35
|
+
import { formatRuntimeInputGeneration } from "./input-generation";
|
|
36
|
+
import { createJpegCaptureDemand } from "./jpeg-capture-demand";
|
|
37
|
+
import { heldKeyModifiers, nativeKeyEvent } from "./keyboard-input";
|
|
38
|
+
import {
|
|
39
|
+
NativeVideoCapture,
|
|
40
|
+
type NativeVideoQuality,
|
|
41
|
+
type NativeVideoRead,
|
|
42
|
+
} from "./native-video-capture";
|
|
43
|
+
import { NATIVE_VIDEO_EXTENSION_ID, prepareNativeVideoExtension } from "./native-video-extension";
|
|
44
|
+
import {
|
|
45
|
+
NAVIGATION_ACK_TIMEOUT_MS,
|
|
46
|
+
NAVIGATION_COMMIT_TIMEOUT_MS,
|
|
47
|
+
NAVIGATION_METADATA_TIMEOUT_MS,
|
|
48
|
+
} from "./navigation-budget";
|
|
49
|
+
import { waitForNavigationCommit } from "./navigation-commit";
|
|
50
|
+
import {
|
|
51
|
+
type ProcessIdentity,
|
|
52
|
+
probeProcess,
|
|
53
|
+
processExists,
|
|
54
|
+
RUNTIME_OWNER_VARIABLE,
|
|
55
|
+
} from "./process-identity";
|
|
56
|
+
import { createScreencastSourceTime } from "./screencast-source-time";
|
|
57
|
+
import { createVideoSourceRecovery } from "./video-source-recovery";
|
|
15
58
|
|
|
16
59
|
const execFileAsync = promisify(execFile);
|
|
17
|
-
export const AGENT_BROWSER_VERSION = "0.
|
|
60
|
+
export const AGENT_BROWSER_VERSION = "0.38.2";
|
|
18
61
|
const PRIVATE_DIRECTORY_MODE = 0o700;
|
|
19
62
|
const PRIVATE_FILE_MODE = 0o600;
|
|
20
63
|
const DEFAULT_TIMEOUT_MS = 15_000;
|
|
21
64
|
const SCREENCAST_SETUP_RETRY_MS = 25;
|
|
65
|
+
/** Bound for the agent-browser session daemon to exit after close or SIGKILL. */
|
|
66
|
+
const DAEMON_EXIT_WAIT_MS = 5_000;
|
|
67
|
+
// Cursor decoration must not hold the serialized input/video fence while page
|
|
68
|
+
// scripts are busy. A missed sample leaves input and its cleanup unchanged.
|
|
69
|
+
const CURSOR_SAMPLE_TIMEOUT_MS = 250;
|
|
22
70
|
|
|
23
71
|
export class AgentBrowserUnavailableError extends Error {
|
|
24
72
|
override readonly name = "AgentBrowserUnavailableError";
|
|
@@ -34,6 +82,8 @@ export interface BrowserViewport {
|
|
|
34
82
|
width: number;
|
|
35
83
|
height: number;
|
|
36
84
|
deviceScaleFactor: number;
|
|
85
|
+
/** Capture density does not change the emulated layout or pointer coordinate system. */
|
|
86
|
+
captureScale?: number;
|
|
37
87
|
mobile: boolean;
|
|
38
88
|
touch: boolean;
|
|
39
89
|
userAgent?: string;
|
|
@@ -68,6 +118,8 @@ export interface RuntimePageState {
|
|
|
68
118
|
title: string;
|
|
69
119
|
canGoBack: boolean;
|
|
70
120
|
canGoForward: boolean;
|
|
121
|
+
/** Process-local attachment/document revision, including same-URL reloads. */
|
|
122
|
+
inputGeneration: string;
|
|
71
123
|
}
|
|
72
124
|
|
|
73
125
|
export interface RuntimeFrame {
|
|
@@ -94,7 +146,13 @@ export interface AgentBrowserRuntimeOptions {
|
|
|
94
146
|
session: string;
|
|
95
147
|
initialUrl?: string;
|
|
96
148
|
headed?: boolean;
|
|
149
|
+
/** Trusted per-instance private X11 context, only DISPLAY/XAUTHORITY; never mutates host environment. */
|
|
150
|
+
launchEnvironment?: Readonly<Record<string, string>>;
|
|
151
|
+
/** Trusted daemon-side opt-in. Only then is the capture extension materialized and loaded. */
|
|
152
|
+
nativeVideo?: boolean;
|
|
97
153
|
timeoutMs?: number;
|
|
154
|
+
/** Bound for the session daemon to exit after close or SIGKILL. */
|
|
155
|
+
daemonExitWaitMs?: number;
|
|
98
156
|
}
|
|
99
157
|
|
|
100
158
|
interface VersionResult {
|
|
@@ -108,14 +166,10 @@ interface NavigationHistory {
|
|
|
108
166
|
currentIndex: number;
|
|
109
167
|
entries: Array<{ id: number; url: string; title: string }>;
|
|
110
168
|
}
|
|
111
|
-
interface PageLifecycleEvent {
|
|
112
|
-
name: string;
|
|
113
|
-
loaderId: string;
|
|
114
|
-
}
|
|
115
169
|
|
|
116
170
|
interface ScreencastFrame {
|
|
117
171
|
data: string;
|
|
118
|
-
metadata: { deviceWidth: number; deviceHeight: number };
|
|
172
|
+
metadata: { deviceWidth: number; deviceHeight: number; timestamp?: number };
|
|
119
173
|
sessionId: number;
|
|
120
174
|
}
|
|
121
175
|
|
|
@@ -124,7 +178,11 @@ function requireAbsolute(path: string, label: string): string {
|
|
|
124
178
|
return resolve(path);
|
|
125
179
|
}
|
|
126
180
|
|
|
127
|
-
function runtimeEnvironment(
|
|
181
|
+
function runtimeEnvironment(
|
|
182
|
+
ipcDirectory: string,
|
|
183
|
+
launchEnvironment: Readonly<Record<string, string>> | undefined,
|
|
184
|
+
ownerNonce: string,
|
|
185
|
+
): NodeJS.ProcessEnv {
|
|
128
186
|
const environment: NodeJS.ProcessEnv = {};
|
|
129
187
|
for (const key of [
|
|
130
188
|
"PATH",
|
|
@@ -153,6 +211,27 @@ function runtimeEnvironment(ipcDirectory: string): NodeJS.ProcessEnv {
|
|
|
153
211
|
const value = process.env[key];
|
|
154
212
|
if (value !== undefined) environment[key] = value;
|
|
155
213
|
}
|
|
214
|
+
if (launchEnvironment !== undefined) {
|
|
215
|
+
const keys = Object.keys(launchEnvironment);
|
|
216
|
+
const display = launchEnvironment.DISPLAY;
|
|
217
|
+
const authority = launchEnvironment.XAUTHORITY;
|
|
218
|
+
if (
|
|
219
|
+
keys.length !== 2 ||
|
|
220
|
+
keys.some((key) => key !== "DISPLAY" && key !== "XAUTHORITY") ||
|
|
221
|
+
typeof display !== "string" ||
|
|
222
|
+
!/^:[0-9]{1,5}(?:\.[0-9]{1,2})?$/.test(display) ||
|
|
223
|
+
typeof authority !== "string" ||
|
|
224
|
+
!isAbsolute(authority)
|
|
225
|
+
) {
|
|
226
|
+
// Do not include display/authentication values in errors or child arguments.
|
|
227
|
+
throw new AgentBrowserIncompatibleError("Private browser display environment is invalid");
|
|
228
|
+
}
|
|
229
|
+
environment.DISPLAY = display;
|
|
230
|
+
environment.XAUTHORITY = authority;
|
|
231
|
+
// A private X11 display must not inherit an unrelated host Wayland session.
|
|
232
|
+
delete environment.WAYLAND_DISPLAY;
|
|
233
|
+
}
|
|
234
|
+
environment[RUNTIME_OWNER_VARIABLE] = ownerNonce;
|
|
156
235
|
environment.AGENT_BROWSER_SOCKET_DIR = ipcDirectory;
|
|
157
236
|
environment.AGENT_BROWSER_IDLE_TIMEOUT_MS = "0";
|
|
158
237
|
environment.AGENT_BROWSER_STREAM_PORT = "0";
|
|
@@ -183,26 +262,6 @@ function findString(value: unknown, keys: readonly string[]): string | null {
|
|
|
183
262
|
return null;
|
|
184
263
|
}
|
|
185
264
|
|
|
186
|
-
export function jpegDimensions(base64: string): { width: number; height: number } | null {
|
|
187
|
-
const bytes = Buffer.from(base64, "base64");
|
|
188
|
-
let offset = 2;
|
|
189
|
-
while (bytes[0] === 0xff && bytes[1] === 0xd8 && offset + 9 <= bytes.length) {
|
|
190
|
-
if (bytes[offset] !== 0xff) return null;
|
|
191
|
-
const marker = bytes[offset + 1];
|
|
192
|
-
// SOFn carries the frame size; DHT (c4), JPG (c8) and DAC (cc) share the range but do not.
|
|
193
|
-
if (
|
|
194
|
-
marker !== undefined &&
|
|
195
|
-
marker >= 0xc0 &&
|
|
196
|
-
marker <= 0xcf &&
|
|
197
|
-
![0xc4, 0xc8, 0xcc].includes(marker)
|
|
198
|
-
)
|
|
199
|
-
return { width: bytes.readUInt16BE(offset + 7), height: bytes.readUInt16BE(offset + 5) };
|
|
200
|
-
if (marker === 0xda) return null;
|
|
201
|
-
offset += 2 + bytes.readUInt16BE(offset + 2);
|
|
202
|
-
}
|
|
203
|
-
return null;
|
|
204
|
-
}
|
|
205
|
-
|
|
206
265
|
export class AgentBrowserRuntime {
|
|
207
266
|
readonly binaryPath: string;
|
|
208
267
|
readonly executablePath: string;
|
|
@@ -213,17 +272,58 @@ export class AgentBrowserRuntime {
|
|
|
213
272
|
private readonly headed: boolean;
|
|
214
273
|
private readonly timeoutMs: number;
|
|
215
274
|
private readonly environment: NodeJS.ProcessEnv;
|
|
275
|
+
private daemon: { pid: number; identity: ProcessIdentity | null } | null = null;
|
|
276
|
+
private readonly ownerNonce = randomUUID();
|
|
277
|
+
private readonly daemonExitWaitMs: number;
|
|
216
278
|
private connection: CdpConnection | null = null;
|
|
217
279
|
private page: CdpSession | null = null;
|
|
218
280
|
private targetId: string | null = null;
|
|
219
|
-
private viewport:
|
|
281
|
+
private viewport: DeviceEmulation | null = null;
|
|
282
|
+
private emulationAppliedPage: CdpSession | null = null;
|
|
283
|
+
private attachmentGeneration = 0;
|
|
284
|
+
private readonly capturePolicy = createCaptureTransportPolicy();
|
|
285
|
+
private readonly screencastSourceTime = createScreencastSourceTime();
|
|
220
286
|
private screencastFrame: RuntimeFrame | null = null;
|
|
287
|
+
private screencastFrameReceivedAt: number | null = null;
|
|
221
288
|
private screencastWaiters = new Set<(frame: RuntimeFrame | null) => void>();
|
|
222
289
|
private screencastActive = false;
|
|
223
|
-
private screencastQuality =
|
|
290
|
+
private screencastQuality: number = DEFAULT_JPEG_QUALITY;
|
|
291
|
+
private screencastStartFailed = false;
|
|
292
|
+
private readonly jpegDemand = createJpegCaptureDemand({
|
|
293
|
+
now: () => performance.now(),
|
|
294
|
+
retire: () => this.stopScreencastSession(),
|
|
295
|
+
onError: () => console.warn("Shared Browser could not retire unused JPEG capture"),
|
|
296
|
+
});
|
|
224
297
|
private heldButtons = new Set<MouseButton>();
|
|
225
|
-
|
|
298
|
+
/** Last native pointer position sent, so cancelled drags release where they were. */
|
|
299
|
+
private lastMousePoint = { x: 0, y: 0 };
|
|
300
|
+
private heldKeys = new Map<string, { key: string; code: string }>();
|
|
301
|
+
private heldTouches = false;
|
|
302
|
+
/** Attachment on which the current held state was published; cleanup never targets another. */
|
|
303
|
+
private heldOwner: CdpSession | null = null;
|
|
304
|
+
private activeTouches = new Map<number, { x: number; y: number; id: number }>();
|
|
305
|
+
private documentGeneration = 0;
|
|
306
|
+
private liveInput: {
|
|
307
|
+
id: string;
|
|
308
|
+
page: CdpSession;
|
|
309
|
+
generation: number;
|
|
310
|
+
documentGeneration: number;
|
|
311
|
+
idleUntil: number;
|
|
312
|
+
expiresAt: number;
|
|
313
|
+
timer: ReturnType<typeof setTimeout> | null;
|
|
314
|
+
} | null = null;
|
|
226
315
|
private stopping = false;
|
|
316
|
+
private shutdownInFlight: Promise<void> | null = null;
|
|
317
|
+
private shutDown = false;
|
|
318
|
+
/** A daemon was launched for this session, so absence of evidence is not proof of exit. */
|
|
319
|
+
private daemonLaunched = false;
|
|
320
|
+
private readonly nativeVideo: boolean;
|
|
321
|
+
private video: NativeVideoCapture | null = null;
|
|
322
|
+
private videoRetirement: Promise<void> | null = null;
|
|
323
|
+
private ownsBrowser = true;
|
|
324
|
+
private readonly videoRecovery = createVideoSourceRecovery();
|
|
325
|
+
private videoTransitionDepth = 0;
|
|
326
|
+
private videoTransitionEpoch = 0;
|
|
227
327
|
|
|
228
328
|
constructor(options: AgentBrowserRuntimeOptions) {
|
|
229
329
|
this.binaryPath = requireAbsolute(options.binaryPath, "agent-browser binary path");
|
|
@@ -236,22 +336,46 @@ export class AgentBrowserRuntime {
|
|
|
236
336
|
this.session = options.session;
|
|
237
337
|
this.headed = options.headed ?? false;
|
|
238
338
|
this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
239
|
-
this.
|
|
339
|
+
this.daemonExitWaitMs = options.daemonExitWaitMs ?? DAEMON_EXIT_WAIT_MS;
|
|
340
|
+
this.environment = runtimeEnvironment(
|
|
341
|
+
this.ipcDirectory,
|
|
342
|
+
options.launchEnvironment,
|
|
343
|
+
this.ownerNonce,
|
|
344
|
+
);
|
|
240
345
|
this.initialUrl = options.initialUrl ?? "about:blank";
|
|
346
|
+
this.nativeVideo = options.nativeVideo === true;
|
|
241
347
|
}
|
|
242
348
|
|
|
243
349
|
async launch(): Promise<void> {
|
|
350
|
+
this.assertRuntimeOpen();
|
|
244
351
|
if (this.connection?.isOpen) return;
|
|
245
352
|
this.invalidateScreencastFrame();
|
|
246
|
-
this.stopping = false;
|
|
247
353
|
await mkdir(this.profilePath, { recursive: true, mode: PRIVATE_DIRECTORY_MODE });
|
|
248
354
|
await mkdir(this.ipcDirectory, { recursive: true, mode: PRIVATE_DIRECTORY_MODE });
|
|
249
355
|
await Promise.all([
|
|
250
356
|
chmod(this.profilePath, PRIVATE_DIRECTORY_MODE),
|
|
251
357
|
chmod(this.ipcDirectory, PRIVATE_DIRECTORY_MODE),
|
|
252
358
|
]);
|
|
359
|
+
this.assertRuntimeOpen();
|
|
253
360
|
await this.assertVersion();
|
|
254
|
-
|
|
361
|
+
this.assertRuntimeOpen();
|
|
362
|
+
// Image-only sessions never materialize, load or allowlist the capture
|
|
363
|
+
// extension, so its tabCapture/debugger permissions do not exist in them.
|
|
364
|
+
const videoExtension = this.nativeVideo
|
|
365
|
+
? await prepareNativeVideoExtension(this.ipcDirectory)
|
|
366
|
+
: null;
|
|
367
|
+
this.assertRuntimeOpen();
|
|
368
|
+
const existingArguments = process.env.PASEO_SHARED_BROWSER_CHROMIUM_ARGS;
|
|
369
|
+
const chromiumArguments = [
|
|
370
|
+
existingArguments,
|
|
371
|
+
videoExtension ? `--allowlisted-extension-id=${NATIVE_VIDEO_EXTENSION_ID}` : undefined,
|
|
372
|
+
]
|
|
373
|
+
.filter(Boolean)
|
|
374
|
+
.join(",");
|
|
375
|
+
this.ownsBrowser = true;
|
|
376
|
+
// From here a nonce-owned daemon may exist even if open rejects (e.g. navigation failure).
|
|
377
|
+
this.daemonLaunched = true;
|
|
378
|
+
this.daemon = null;
|
|
255
379
|
const opened = await this.invoke([
|
|
256
380
|
"--session",
|
|
257
381
|
this.session,
|
|
@@ -259,25 +383,63 @@ export class AgentBrowserRuntime {
|
|
|
259
383
|
this.profilePath,
|
|
260
384
|
"--executable-path",
|
|
261
385
|
this.executablePath,
|
|
386
|
+
...(videoExtension ? ["--extension", videoExtension] : []),
|
|
387
|
+
"--hide-scrollbars",
|
|
388
|
+
"false",
|
|
262
389
|
...(this.headed ? ["--headed"] : []),
|
|
263
390
|
...(chromiumArguments ? ["--args", chromiumArguments] : []),
|
|
264
391
|
"--json",
|
|
265
392
|
"open",
|
|
266
393
|
this.initialUrl,
|
|
267
394
|
]);
|
|
395
|
+
this.assertRuntimeOpen();
|
|
268
396
|
const targetId = findString(opened, ["targetId", "target_id"]);
|
|
269
397
|
await this.protectIpcMetadata();
|
|
398
|
+
await this.captureDaemon();
|
|
399
|
+
this.assertRuntimeOpen();
|
|
270
400
|
await this.connectCdp(targetId);
|
|
271
401
|
}
|
|
272
402
|
|
|
403
|
+
/** Attach an independent page controller to an existing browser session. */
|
|
404
|
+
async attachTarget(targetId: string): Promise<void> {
|
|
405
|
+
this.assertRuntimeOpen();
|
|
406
|
+
this.ownsBrowser = false;
|
|
407
|
+
await this.connectCdp(targetId, false);
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/** Create a page in the existing profile without changing another controller's selected page. */
|
|
411
|
+
async createTarget(url = "about:blank"): Promise<string> {
|
|
412
|
+
const result = await this.requireConnection().send<{ targetId: string }>(
|
|
413
|
+
"Target.createTarget",
|
|
414
|
+
{ url },
|
|
415
|
+
{ mutation: true },
|
|
416
|
+
);
|
|
417
|
+
return result.targetId;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/** Close only the named page. The caller must keep another page in the browser. */
|
|
421
|
+
async closeTarget(targetId: string): Promise<void> {
|
|
422
|
+
const result = await this.requireConnection().send<{ success: boolean }>(
|
|
423
|
+
"Target.closeTarget",
|
|
424
|
+
{ targetId },
|
|
425
|
+
{ mutation: true },
|
|
426
|
+
);
|
|
427
|
+
if (!result.success) throw new CdpUnavailableError("Browser tab could not be closed");
|
|
428
|
+
}
|
|
429
|
+
|
|
273
430
|
async reconnect(): Promise<void> {
|
|
274
|
-
|
|
275
|
-
this.
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
431
|
+
this.assertRuntimeOpen();
|
|
432
|
+
await this.withVideoTransition(async () => {
|
|
433
|
+
const targetId = this.targetId;
|
|
434
|
+
this.attachmentGeneration += 1;
|
|
435
|
+
this.invalidateScreencastFrame();
|
|
436
|
+
this.connection?.close();
|
|
437
|
+
this.connection = null;
|
|
438
|
+
this.page = null;
|
|
439
|
+
await this.assertVersion();
|
|
440
|
+
this.assertRuntimeOpen();
|
|
441
|
+
await this.connectCdp(targetId);
|
|
442
|
+
});
|
|
281
443
|
}
|
|
282
444
|
|
|
283
445
|
async health(): Promise<RuntimeHealth> {
|
|
@@ -321,32 +483,73 @@ export class AgentBrowserRuntime {
|
|
|
321
483
|
}
|
|
322
484
|
|
|
323
485
|
async targets(): Promise<RuntimeTarget[]> {
|
|
324
|
-
return listPageTargets(this.requireConnection())
|
|
486
|
+
return (await listPageTargets(this.requireConnection())).filter(
|
|
487
|
+
(target) => !target.url.startsWith(`chrome-extension://${NATIVE_VIDEO_EXTENSION_ID}/`),
|
|
488
|
+
);
|
|
325
489
|
}
|
|
326
490
|
|
|
327
|
-
async selectTarget(targetId: string): Promise<void> {
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
491
|
+
async selectTarget(targetId: string, activate = true): Promise<void> {
|
|
492
|
+
await this.withVideoTransition(async () => {
|
|
493
|
+
const attachmentGeneration = ++this.attachmentGeneration;
|
|
494
|
+
const connection = this.requireConnection();
|
|
495
|
+
const target = (await this.targets()).find((candidate) => candidate.targetId === targetId);
|
|
496
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
497
|
+
if (!target) throw new CdpUnavailableError(`Unknown page target: ${targetId}`);
|
|
498
|
+
this.invalidateScreencastFrame();
|
|
499
|
+
const previous = this.page;
|
|
500
|
+
if (previous) {
|
|
501
|
+
if (this.screencastActive) {
|
|
502
|
+
await previous.send("Page.stopScreencast", {}, { mutation: true });
|
|
503
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
504
|
+
}
|
|
505
|
+
await previous.detach();
|
|
506
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
334
507
|
}
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
508
|
+
this.invalidateScreencastFrame();
|
|
509
|
+
if (activate) {
|
|
510
|
+
await connection.send("Target.activateTarget", { targetId }, { mutation: true });
|
|
511
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
512
|
+
}
|
|
513
|
+
// CDP emulation is session-owned: detaching the former session restores native
|
|
514
|
+
// window metrics. Publish a replacement only after its retained settings succeed.
|
|
515
|
+
this.page = null;
|
|
516
|
+
this.emulationAppliedPage = null;
|
|
517
|
+
const replacement = await attachToTarget(connection, targetId);
|
|
518
|
+
try {
|
|
519
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
520
|
+
await this.bindPageEvents(replacement);
|
|
521
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
522
|
+
await this.restoreConfiguredEmulation(replacement);
|
|
523
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
524
|
+
this.page = replacement;
|
|
525
|
+
this.targetId = targetId;
|
|
526
|
+
} catch (error) {
|
|
527
|
+
await replacement.detach().catch(() => undefined);
|
|
528
|
+
// Detaching a partial emulation can clear target metrics even if a newer
|
|
529
|
+
// attachment won. Its next access must restore intent before admitting pixels.
|
|
530
|
+
this.emulationAppliedPage = null;
|
|
531
|
+
this.invalidateScreencastFrame();
|
|
532
|
+
throw error;
|
|
533
|
+
}
|
|
534
|
+
if (this.screencastActive) await this.startScreencastSession(replacement);
|
|
535
|
+
});
|
|
343
536
|
}
|
|
344
537
|
|
|
345
|
-
private async navigationHistory(
|
|
346
|
-
|
|
538
|
+
private async navigationHistory(
|
|
539
|
+
page: CdpSession,
|
|
540
|
+
timeoutMs = this.timeoutMs,
|
|
541
|
+
): Promise<NavigationHistory> {
|
|
542
|
+
const deadline = Date.now() + timeoutMs;
|
|
347
543
|
for (;;) {
|
|
544
|
+
const remaining = deadline - Date.now();
|
|
545
|
+
if (remaining <= 0)
|
|
546
|
+
throw new CdpUnavailableError("Browser navigation history observation timed out");
|
|
348
547
|
try {
|
|
349
|
-
return await page.send<NavigationHistory>(
|
|
548
|
+
return await page.send<NavigationHistory>(
|
|
549
|
+
"Page.getNavigationHistory",
|
|
550
|
+
{},
|
|
551
|
+
{ timeoutMs: remaining },
|
|
552
|
+
);
|
|
350
553
|
} catch (error) {
|
|
351
554
|
if (!String(error).includes("Not attached to an active page") || Date.now() >= deadline) {
|
|
352
555
|
throw error;
|
|
@@ -357,113 +560,524 @@ export class AgentBrowserRuntime {
|
|
|
357
560
|
}
|
|
358
561
|
}
|
|
359
562
|
}
|
|
360
|
-
|
|
563
|
+
/** Read fresh metadata concurrently, discarding both replies if the document changes.
|
|
564
|
+
* One same-attachment reread shares the original monotonic timeout budget in
|
|
565
|
+
* milliseconds. Failed reads, replacement attachments and repeated document
|
|
566
|
+
* changes reject; this read-only recovery never repeats a native mutation. */
|
|
567
|
+
async state(options: { timeoutMs?: number } = {}): Promise<RuntimePageState> {
|
|
568
|
+
if (
|
|
569
|
+
options.timeoutMs !== undefined &&
|
|
570
|
+
(!Number.isInteger(options.timeoutMs) ||
|
|
571
|
+
options.timeoutMs < 1 ||
|
|
572
|
+
options.timeoutMs > DEFAULT_TIMEOUT_MS)
|
|
573
|
+
) {
|
|
574
|
+
throw new RangeError("Browser metadata timeout must be an integer from 1 to 15000ms");
|
|
575
|
+
}
|
|
576
|
+
const timeoutMs = Math.min(options.timeoutMs ?? this.timeoutMs, this.timeoutMs);
|
|
361
577
|
const page = await this.requirePage();
|
|
362
|
-
const
|
|
363
|
-
const
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
578
|
+
const attachmentGeneration = this.attachmentGeneration;
|
|
579
|
+
const deadline = performance.now() + timeoutMs;
|
|
580
|
+
|
|
581
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
582
|
+
this.assertRuntimeOpen();
|
|
583
|
+
const remaining = attempt === 0 ? timeoutMs : deadline - performance.now();
|
|
584
|
+
if (remaining <= 0) {
|
|
585
|
+
throw new CdpUnavailableError("Browser metadata observation timed out");
|
|
586
|
+
}
|
|
587
|
+
const documentGeneration = this.documentGeneration;
|
|
588
|
+
// Both observations must describe the same document. A navigation discards
|
|
589
|
+
// the entire pair; neither native history nor DOM metadata is cached.
|
|
590
|
+
const [history, evaluated] = await Promise.all([
|
|
591
|
+
this.navigationHistory(page, remaining),
|
|
592
|
+
page.send<{
|
|
593
|
+
result: { value?: { url?: string; title?: string } };
|
|
594
|
+
}>(
|
|
595
|
+
"Runtime.evaluate",
|
|
596
|
+
{
|
|
597
|
+
expression: "({url: location.href, title: document.title})",
|
|
598
|
+
returnByValue: true,
|
|
599
|
+
},
|
|
600
|
+
{ timeoutMs: remaining },
|
|
601
|
+
),
|
|
602
|
+
]);
|
|
603
|
+
this.assertRuntimeOpen();
|
|
604
|
+
if (page !== this.page || attachmentGeneration !== this.attachmentGeneration) {
|
|
605
|
+
throw new CdpUnavailableError("Browser metadata changed while being read");
|
|
606
|
+
}
|
|
607
|
+
// Successful replies can beat an overdue CDP timer in the event queue.
|
|
608
|
+
// Their metadata still cannot outlive the original shared phase budget.
|
|
609
|
+
if (performance.now() >= deadline) {
|
|
610
|
+
throw new CdpUnavailableError("Browser metadata observation timed out");
|
|
611
|
+
}
|
|
612
|
+
if (documentGeneration !== this.documentGeneration) {
|
|
613
|
+
if (attempt === 0) continue;
|
|
614
|
+
throw new CdpUnavailableError("Browser metadata changed while being read");
|
|
615
|
+
}
|
|
616
|
+
return {
|
|
617
|
+
url: evaluated.result.value?.url ?? history.entries[history.currentIndex]?.url ?? "",
|
|
618
|
+
title: evaluated.result.value?.title ?? history.entries[history.currentIndex]?.title ?? "",
|
|
619
|
+
canGoBack: history.currentIndex > 0,
|
|
620
|
+
canGoForward: history.currentIndex < history.entries.length - 1,
|
|
621
|
+
inputGeneration: formatRuntimeInputGeneration(attachmentGeneration, documentGeneration),
|
|
622
|
+
};
|
|
623
|
+
}
|
|
624
|
+
throw new CdpUnavailableError("Browser metadata changed while being read");
|
|
375
625
|
}
|
|
376
626
|
|
|
627
|
+
/** Await native ACK and its main-frame commit, without waiting for scripts or
|
|
628
|
+
* DOMContentLoaded. A missing commit remains uncertain and is never replayed. */
|
|
377
629
|
async navigate(url: string): Promise<void> {
|
|
378
|
-
await this.withInvalidatedScreencast(
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
),
|
|
409
|
-
),
|
|
410
|
-
30_000,
|
|
630
|
+
await this.withInvalidatedScreencast(
|
|
631
|
+
async (page) => {
|
|
632
|
+
const assertCurrent = this.navigationAttachmentGuard(page);
|
|
633
|
+
await waitForNavigationCommit(
|
|
634
|
+
page,
|
|
635
|
+
async () => {
|
|
636
|
+
const result = await page.send<{
|
|
637
|
+
frameId: string;
|
|
638
|
+
errorText?: string;
|
|
639
|
+
loaderId?: string;
|
|
640
|
+
isDownload?: boolean;
|
|
641
|
+
}>(
|
|
642
|
+
"Page.navigate",
|
|
643
|
+
{ url },
|
|
644
|
+
{ mutation: true, timeoutMs: Math.min(this.timeoutMs, NAVIGATION_ACK_TIMEOUT_MS) },
|
|
645
|
+
);
|
|
646
|
+
// Chromium can abort document navigation because the accepted
|
|
647
|
+
// response is a download. This does not replace the current page.
|
|
648
|
+
if (result.errorText && !result.isDownload) {
|
|
649
|
+
throw new Error(`Navigation failed: ${result.errorText}`);
|
|
650
|
+
}
|
|
651
|
+
return {
|
|
652
|
+
frameId: result.frameId,
|
|
653
|
+
...(result.loaderId ? { loaderId: result.loaderId } : {}),
|
|
654
|
+
url,
|
|
655
|
+
...(result.isDownload ? { download: true } : {}),
|
|
656
|
+
};
|
|
657
|
+
},
|
|
658
|
+
assertCurrent,
|
|
659
|
+
Math.min(this.timeoutMs, NAVIGATION_COMMIT_TIMEOUT_MS),
|
|
411
660
|
);
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
clearTimeout(timer);
|
|
416
|
-
page.off("Page.lifecycleEvent", onLifecycle);
|
|
417
|
-
}
|
|
418
|
-
});
|
|
661
|
+
},
|
|
662
|
+
{ preserveStream: true },
|
|
663
|
+
);
|
|
419
664
|
}
|
|
420
665
|
|
|
421
666
|
async back(): Promise<void> {
|
|
422
|
-
await this.
|
|
423
|
-
const history = await this.navigationHistory(page);
|
|
424
|
-
const entry = history.entries[history.currentIndex - 1];
|
|
425
|
-
if (!entry) throw new Error("Browser cannot go back");
|
|
426
|
-
await page.send("Page.navigateToHistoryEntry", { entryId: entry.id }, { mutation: true });
|
|
427
|
-
});
|
|
667
|
+
await this.navigateHistory(-1);
|
|
428
668
|
}
|
|
429
669
|
|
|
430
670
|
async forward(): Promise<void> {
|
|
431
|
-
await this.
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
671
|
+
await this.navigateHistory(1);
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/** History's ACK has no loader receipt. Capture its root loader and target URL
|
|
675
|
+
* in one parallel bounded preflight, then admit only the matching new commit. */
|
|
676
|
+
private async navigateHistory(direction: -1 | 1): Promise<void> {
|
|
677
|
+
await this.withInvalidatedScreencast(
|
|
678
|
+
async (page) => {
|
|
679
|
+
const assertCurrent = this.navigationAttachmentGuard(page);
|
|
680
|
+
const [history, root] = await Promise.all([
|
|
681
|
+
this.navigationHistory(page, Math.min(this.timeoutMs, NAVIGATION_METADATA_TIMEOUT_MS)),
|
|
682
|
+
this.navigationRoot(page),
|
|
683
|
+
]);
|
|
684
|
+
assertCurrent();
|
|
685
|
+
const entry = history.entries[history.currentIndex + direction];
|
|
686
|
+
if (!entry)
|
|
687
|
+
throw new Error(
|
|
688
|
+
direction === -1 ? "Browser cannot go back" : "Browser cannot go forward",
|
|
689
|
+
);
|
|
690
|
+
await waitForNavigationCommit(
|
|
691
|
+
page,
|
|
692
|
+
async () => {
|
|
693
|
+
await page.send(
|
|
694
|
+
"Page.navigateToHistoryEntry",
|
|
695
|
+
{ entryId: entry.id },
|
|
696
|
+
{
|
|
697
|
+
mutation: true,
|
|
698
|
+
timeoutMs: Math.min(this.timeoutMs, NAVIGATION_ACK_TIMEOUT_MS),
|
|
699
|
+
},
|
|
700
|
+
);
|
|
701
|
+
return { frameId: root.id, previousLoaderId: root.loaderId, url: entry.url };
|
|
702
|
+
},
|
|
703
|
+
assertCurrent,
|
|
704
|
+
Math.min(this.timeoutMs, NAVIGATION_COMMIT_TIMEOUT_MS),
|
|
705
|
+
);
|
|
706
|
+
},
|
|
707
|
+
{ preserveStream: true },
|
|
708
|
+
);
|
|
437
709
|
}
|
|
438
710
|
|
|
439
711
|
async reload(ignoreCache = false): Promise<void> {
|
|
440
|
-
await this.withInvalidatedScreencast(
|
|
441
|
-
page
|
|
712
|
+
await this.withInvalidatedScreencast(
|
|
713
|
+
async (page) => {
|
|
714
|
+
const assertCurrent = this.navigationAttachmentGuard(page);
|
|
715
|
+
const root = await this.navigationRoot(page);
|
|
716
|
+
assertCurrent();
|
|
717
|
+
await waitForNavigationCommit(
|
|
718
|
+
page,
|
|
719
|
+
async () => {
|
|
720
|
+
await page.send(
|
|
721
|
+
"Page.reload",
|
|
722
|
+
{ ignoreCache },
|
|
723
|
+
{
|
|
724
|
+
mutation: true,
|
|
725
|
+
timeoutMs: Math.min(this.timeoutMs, NAVIGATION_ACK_TIMEOUT_MS),
|
|
726
|
+
},
|
|
727
|
+
);
|
|
728
|
+
return { frameId: root.id, previousLoaderId: root.loaderId };
|
|
729
|
+
},
|
|
730
|
+
assertCurrent,
|
|
731
|
+
Math.min(this.timeoutMs, NAVIGATION_COMMIT_TIMEOUT_MS),
|
|
732
|
+
);
|
|
733
|
+
},
|
|
734
|
+
{ preserveStream: true },
|
|
735
|
+
);
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
/** A root snapshot is observation only, bounded before mutation publication. */
|
|
739
|
+
private async navigationRoot(page: CdpSession): Promise<{ id: string; loaderId: string }> {
|
|
740
|
+
const result = await page.send<{ frameTree: { frame: { id: string; loaderId: string } } }>(
|
|
741
|
+
"Page.getFrameTree",
|
|
742
|
+
{},
|
|
743
|
+
{ timeoutMs: Math.min(this.timeoutMs, NAVIGATION_METADATA_TIMEOUT_MS) },
|
|
442
744
|
);
|
|
745
|
+
const root = result.frameTree.frame;
|
|
746
|
+
if (!root.id || !root.loaderId)
|
|
747
|
+
throw new CdpUnavailableError("Browser main frame is unavailable");
|
|
748
|
+
return root;
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
/** Keep commit observation on the original attachment, including shutdown. */
|
|
752
|
+
private navigationAttachmentGuard(page: CdpSession): () => void {
|
|
753
|
+
const attachmentGeneration = this.attachmentGeneration;
|
|
754
|
+
return () => {
|
|
755
|
+
this.assertAttachmentCurrent(page.connection, attachmentGeneration);
|
|
756
|
+
if (this.page !== page)
|
|
757
|
+
throw new CdpUnavailableError("Browser page changed during navigation");
|
|
758
|
+
};
|
|
443
759
|
}
|
|
444
760
|
|
|
445
761
|
async emulate(device: DeviceEmulation): Promise<void> {
|
|
446
762
|
this.assertViewport(device);
|
|
447
|
-
await this.
|
|
763
|
+
await this.withVideoTransition(async () => {
|
|
764
|
+
await this.withInvalidatedScreencast(async (page) => {
|
|
765
|
+
const attachmentGeneration = this.attachmentGeneration;
|
|
766
|
+
this.emulationAppliedPage = null;
|
|
767
|
+
await this.applyEmulation(page, device);
|
|
768
|
+
this.assertAttachmentCurrent(page.connection, attachmentGeneration);
|
|
769
|
+
if (this.page !== page)
|
|
770
|
+
throw new CdpUnavailableError("Browser page changed during emulation");
|
|
771
|
+
this.viewport = { ...device };
|
|
772
|
+
this.emulationAppliedPage = page;
|
|
773
|
+
});
|
|
774
|
+
});
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
/** Retain the full mode/UA/screen intent while rendering enough real source pixels for the chosen density. */
|
|
778
|
+
async setCaptureDensity(density: number, baseDeviceScaleFactor: number): Promise<void> {
|
|
779
|
+
const scale = captureDensitySchema.parse(density);
|
|
780
|
+
const viewport = this.requireViewport();
|
|
781
|
+
if (!canUseCaptureDensity(viewport, scale))
|
|
782
|
+
throw new RangeError("Capture density exceeds the supported image bounds");
|
|
783
|
+
if (
|
|
784
|
+
!Number.isFinite(baseDeviceScaleFactor) ||
|
|
785
|
+
baseDeviceScaleFactor < 1 ||
|
|
786
|
+
baseDeviceScaleFactor > 4
|
|
787
|
+
)
|
|
788
|
+
throw new RangeError("Invalid device pixel ratio");
|
|
789
|
+
await this.emulate({
|
|
790
|
+
...viewport,
|
|
791
|
+
captureScale: scale,
|
|
792
|
+
deviceScaleFactor: Math.max(baseDeviceScaleFactor, scale),
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
/** Apply the entire device contract to one CDP session without changing saved intent. */
|
|
797
|
+
private async applyEmulation(page: CdpSession, device: DeviceEmulation): Promise<void> {
|
|
798
|
+
await page.send(
|
|
799
|
+
"Emulation.setDeviceMetricsOverride",
|
|
800
|
+
{
|
|
801
|
+
width: device.width,
|
|
802
|
+
height: device.height,
|
|
803
|
+
deviceScaleFactor: device.deviceScaleFactor,
|
|
804
|
+
mobile: device.mobile,
|
|
805
|
+
screenWidth: device.screenWidth ?? device.width,
|
|
806
|
+
screenHeight: device.screenHeight ?? device.height,
|
|
807
|
+
screenOrientation: {
|
|
808
|
+
type: device.width > device.height ? "landscapePrimary" : "portraitPrimary",
|
|
809
|
+
angle: device.width > device.height ? 90 : 0,
|
|
810
|
+
},
|
|
811
|
+
},
|
|
812
|
+
{ mutation: true },
|
|
813
|
+
);
|
|
814
|
+
await page.send(
|
|
815
|
+
"Emulation.setTouchEmulationEnabled",
|
|
816
|
+
{ enabled: device.touch, maxTouchPoints: device.touch ? 5 : 1 },
|
|
817
|
+
{ mutation: true },
|
|
818
|
+
);
|
|
819
|
+
// A page controlled through its own session may sit behind another tab in the shared
|
|
820
|
+
// browser. Chromium holds a hidden page's mouseMoved acknowledgement for ~5 s (equal to the
|
|
821
|
+
// input idle bound), so every press there failed. Target-scoped focus emulation makes only
|
|
822
|
+
// this page report visible/focused: no tab is raised and no viewer's selection changes.
|
|
823
|
+
await page.send("Emulation.setFocusEmulationEnabled", { enabled: true }, { mutation: true });
|
|
824
|
+
if (device.userAgent) {
|
|
825
|
+
await page.send(
|
|
826
|
+
"Emulation.setUserAgentOverride",
|
|
827
|
+
{ userAgent: device.userAgent, platform: device.platform ?? "" },
|
|
828
|
+
{ mutation: true },
|
|
829
|
+
);
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
/** No frame/input may use a session whose emulated viewport has not been restored. */
|
|
834
|
+
private async restoreConfiguredEmulation(page: CdpSession): Promise<void> {
|
|
835
|
+
const device = this.viewport;
|
|
836
|
+
if (!device || this.emulationAppliedPage === page) return;
|
|
837
|
+
await this.withVideoTransition(async () => {
|
|
838
|
+
const attachmentGeneration = this.attachmentGeneration;
|
|
839
|
+
this.invalidateScreencastFrame();
|
|
840
|
+
this.emulationAppliedPage = null;
|
|
448
841
|
await this.applyEmulation(page, device);
|
|
449
|
-
this.
|
|
842
|
+
this.assertAttachmentCurrent(page.connection, attachmentGeneration);
|
|
843
|
+
if (device !== this.viewport) {
|
|
844
|
+
throw new CdpUnavailableError("Browser device settings changed during restoration");
|
|
845
|
+
}
|
|
846
|
+
this.emulationAppliedPage = page;
|
|
847
|
+
this.invalidateScreencastFrame();
|
|
848
|
+
});
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
/** Reject late configuration completions after a disconnect or newer target selection. */
|
|
852
|
+
private assertAttachmentCurrent(connection: CdpConnection, generation: number): void {
|
|
853
|
+
this.assertRuntimeOpen();
|
|
854
|
+
if (
|
|
855
|
+
connection !== this.connection ||
|
|
856
|
+
!connection.isOpen ||
|
|
857
|
+
generation !== this.attachmentGeneration
|
|
858
|
+
) {
|
|
859
|
+
throw new CdpUnavailableError("Browser attachment changed during emulation restoration");
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
/** Starts only exact restored native tab capture, lazily adding this viewer's quality encoder. */
|
|
864
|
+
async startVideo(quality: NativeVideoQuality = "high"): Promise<NativeVideoRead> {
|
|
865
|
+
return this.readVideo({ quality, waitMs: 0 });
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
/** Long-poll outside the input queue. The caller must fence returned inputGeneration before publication. */
|
|
869
|
+
async readVideo(input: {
|
|
870
|
+
quality: NativeVideoQuality;
|
|
871
|
+
bitrate?: number;
|
|
872
|
+
fps?: number;
|
|
873
|
+
streamId?: string | null;
|
|
874
|
+
afterSequence?: number;
|
|
875
|
+
waitMs?: number;
|
|
876
|
+
requestKeyFrame?: boolean;
|
|
877
|
+
}): Promise<NativeVideoRead & { inputGeneration: string }> {
|
|
878
|
+
if (!this.nativeVideo) {
|
|
879
|
+
return {
|
|
880
|
+
...this.videoReset(),
|
|
881
|
+
status: "unsupported",
|
|
882
|
+
reasonCode: "video-disabled",
|
|
883
|
+
reason: "Encoded video is not enabled on this host",
|
|
884
|
+
};
|
|
885
|
+
}
|
|
886
|
+
const settings = resolveVideoEncoderSettings(input);
|
|
887
|
+
const transitionEpoch = this.videoTransitionEpoch;
|
|
888
|
+
if (!this.videoMayStart(transitionEpoch)) return this.videoReset();
|
|
889
|
+
|
|
890
|
+
// Retirement uses the same stop barrier as viewport changes. The next read
|
|
891
|
+
// resumes from current configuration instead of treating idle expiry as a
|
|
892
|
+
// permanent codec failure or creating a replacement while the old track closes.
|
|
893
|
+
if (this.video?.idleExpired) {
|
|
894
|
+
this.retireVideoForRead();
|
|
895
|
+
return this.videoReset();
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
const page = await this.requirePage();
|
|
899
|
+
// requirePage can itself restore or reconnect. Retry on the next read rather
|
|
900
|
+
// than creating a helper from a configuration that changed across its await.
|
|
901
|
+
if (!this.videoMayStart(transitionEpoch)) return this.videoReset();
|
|
902
|
+
const connection = this.requireConnection();
|
|
903
|
+
const generation = this.attachmentGeneration;
|
|
904
|
+
const viewport = this.requireViewport();
|
|
905
|
+
const recoveryContext = { connection, page, viewport, attachmentGeneration: generation };
|
|
906
|
+
const blocked = this.videoRecovery.blocked(recoveryContext);
|
|
907
|
+
if (blocked === "source-dimensions") {
|
|
908
|
+
return {
|
|
909
|
+
...this.videoReset(),
|
|
910
|
+
status: "unsupported",
|
|
911
|
+
reason: "Native video dimensions do not match the configured viewport",
|
|
912
|
+
};
|
|
913
|
+
}
|
|
914
|
+
if (blocked) {
|
|
915
|
+
return this.videoReset();
|
|
916
|
+
}
|
|
917
|
+
if (!this.video) {
|
|
918
|
+
const pixels = captureDimensions(viewport, viewport.captureScale ?? 1);
|
|
919
|
+
this.video = new NativeVideoCapture({
|
|
920
|
+
connection,
|
|
921
|
+
targetId: page.targetId,
|
|
922
|
+
width: pixels.width,
|
|
923
|
+
height: pixels.height,
|
|
924
|
+
isCurrent: () =>
|
|
925
|
+
this.videoMayStart(transitionEpoch) &&
|
|
926
|
+
connection === this.connection &&
|
|
927
|
+
page === this.page &&
|
|
928
|
+
viewport === this.viewport &&
|
|
929
|
+
generation === this.attachmentGeneration &&
|
|
930
|
+
this.emulationAppliedPage === page,
|
|
931
|
+
});
|
|
932
|
+
}
|
|
933
|
+
const video = this.video;
|
|
934
|
+
const documentGeneration = this.documentGeneration;
|
|
935
|
+
const result = await video.read({ ...input, ...settings });
|
|
936
|
+
if (
|
|
937
|
+
!this.videoMayStart(transitionEpoch) ||
|
|
938
|
+
video.idleExpired ||
|
|
939
|
+
video !== this.video ||
|
|
940
|
+
connection !== this.connection ||
|
|
941
|
+
page !== this.page ||
|
|
942
|
+
viewport !== this.viewport ||
|
|
943
|
+
generation !== this.attachmentGeneration ||
|
|
944
|
+
documentGeneration !== this.documentGeneration
|
|
945
|
+
) {
|
|
946
|
+
return this.videoReset();
|
|
947
|
+
}
|
|
948
|
+
if (video.sourceFailure) {
|
|
949
|
+
// Store failure admission before opening the existing stop barrier. A
|
|
950
|
+
// concurrent read sees either that barrier or the cooldown, never a second
|
|
951
|
+
// track while the failed source/helper is still being retired.
|
|
952
|
+
this.videoRecovery.failed(recoveryContext, video.sourceFailure);
|
|
953
|
+
const permanent = video.sourceFailure === "source-dimensions";
|
|
954
|
+
this.retireVideoForRead();
|
|
955
|
+
if (
|
|
956
|
+
this.stopping ||
|
|
957
|
+
connection !== this.connection ||
|
|
958
|
+
page !== this.page ||
|
|
959
|
+
viewport !== this.viewport ||
|
|
960
|
+
generation !== this.attachmentGeneration
|
|
961
|
+
) {
|
|
962
|
+
return this.videoReset();
|
|
963
|
+
}
|
|
964
|
+
if (permanent) {
|
|
965
|
+
return {
|
|
966
|
+
...this.videoReset(),
|
|
967
|
+
status: "unsupported",
|
|
968
|
+
reason: "Native video dimensions do not match the configured viewport",
|
|
969
|
+
};
|
|
970
|
+
}
|
|
971
|
+
return this.videoReset();
|
|
972
|
+
}
|
|
973
|
+
return {
|
|
974
|
+
...result,
|
|
975
|
+
inputGeneration: formatRuntimeInputGeneration(
|
|
976
|
+
this.attachmentGeneration,
|
|
977
|
+
this.documentGeneration,
|
|
978
|
+
),
|
|
979
|
+
};
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
/** Explicit action barrier; pointer samples preserve the native track and encoder continuity. */
|
|
983
|
+
invalidateQueuedVideoFrames(): void {
|
|
984
|
+
this.video?.invalidateQueuedFrames();
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
/** Establish the stop barrier now, but do not hold a viewing RPC across slow
|
|
988
|
+
* helper teardown. stopVideo owns barrier release; no replacement can start
|
|
989
|
+
* until it settles. Explicit stop callers still await their cleanup. */
|
|
990
|
+
private retireVideoForRead(): void {
|
|
991
|
+
const retirement = this.stopVideo();
|
|
992
|
+
this.videoRetirement = retirement;
|
|
993
|
+
const complete = () => {
|
|
994
|
+
if (this.videoRetirement === retirement) this.videoRetirement = null;
|
|
995
|
+
};
|
|
996
|
+
void retirement.then(complete, () => {
|
|
997
|
+
complete();
|
|
998
|
+
console.warn("[shared-browser] Video retirement failed");
|
|
450
999
|
});
|
|
451
1000
|
}
|
|
452
1001
|
|
|
453
|
-
|
|
1002
|
+
/** Stop admission too: a concurrent reader cannot replace a source still closing. */
|
|
1003
|
+
async stopVideo(): Promise<void> {
|
|
1004
|
+
if (this.videoRetirement) {
|
|
1005
|
+
await this.videoRetirement;
|
|
1006
|
+
return;
|
|
1007
|
+
}
|
|
1008
|
+
await this.withVideoTransition(async () => {});
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
/** Nested device/attachment phases keep one continuous barrier until all phases finish. */
|
|
1012
|
+
private async withVideoTransition<T>(transition: () => Promise<T>): Promise<T> {
|
|
1013
|
+
const releaseJpeg = this.stopping ? () => {} : this.jpegDemand.pin(false);
|
|
1014
|
+
this.videoTransitionDepth += 1;
|
|
1015
|
+
this.videoTransitionEpoch += 1;
|
|
1016
|
+
const video = this.video;
|
|
1017
|
+
this.video = null;
|
|
1018
|
+
try {
|
|
1019
|
+
if (video) await video.stop();
|
|
1020
|
+
return await transition();
|
|
1021
|
+
} finally {
|
|
1022
|
+
this.videoTransitionDepth -= 1;
|
|
1023
|
+
releaseJpeg();
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
private videoMayStart(epoch: number): boolean {
|
|
1028
|
+
return !this.stopping && this.videoTransitionDepth === 0 && epoch === this.videoTransitionEpoch;
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
private videoReset(): NativeVideoRead & { inputGeneration: string } {
|
|
1032
|
+
return {
|
|
1033
|
+
status: "reset",
|
|
1034
|
+
streamId: null,
|
|
1035
|
+
packets: [],
|
|
1036
|
+
inputGeneration: formatRuntimeInputGeneration(
|
|
1037
|
+
this.attachmentGeneration,
|
|
1038
|
+
this.documentGeneration,
|
|
1039
|
+
),
|
|
1040
|
+
};
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
/** Explicit viewing start remains supported, but does not keep unused JPEG work alive. */
|
|
1044
|
+
async startScreencast(quality: number = DEFAULT_JPEG_QUALITY): Promise<void> {
|
|
454
1045
|
if (!Number.isInteger(quality) || quality < 1 || quality > 100) {
|
|
455
1046
|
throw new RangeError("Screencast quality must be an integer from 1 to 100");
|
|
456
1047
|
}
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
1048
|
+
const release = this.jpegDemand.pin();
|
|
1049
|
+
try {
|
|
1050
|
+
await this.jpegDemand.serialize(async () => {
|
|
1051
|
+
const page = await this.requirePage();
|
|
1052
|
+
if (this.screencastActive && this.screencastQuality === quality) return;
|
|
1053
|
+
this.screencastQuality = quality;
|
|
1054
|
+
this.screencastStartFailed = false;
|
|
1055
|
+
this.screencastActive = true;
|
|
1056
|
+
try {
|
|
1057
|
+
await this.startScreencastSession(page);
|
|
1058
|
+
} catch (error) {
|
|
1059
|
+
this.screencastActive = false;
|
|
1060
|
+
this.screencastStartFailed = true;
|
|
1061
|
+
throw error;
|
|
1062
|
+
}
|
|
1063
|
+
});
|
|
1064
|
+
} finally {
|
|
1065
|
+
release();
|
|
1066
|
+
}
|
|
461
1067
|
}
|
|
462
1068
|
|
|
1069
|
+
/** Explicit detach cleanup and idle retirement use the same native transition FIFO. */
|
|
463
1070
|
async stopScreencast(): Promise<void> {
|
|
1071
|
+
this.jpegDemand.cancel();
|
|
1072
|
+
await this.jpegDemand.serialize(() => this.stopScreencastSession());
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
private async stopScreencastSession(): Promise<void> {
|
|
1076
|
+
const wasActive = this.screencastActive;
|
|
464
1077
|
this.screencastActive = false;
|
|
1078
|
+
this.screencastStartFailed = false;
|
|
465
1079
|
this.invalidateScreencastFrame();
|
|
466
|
-
if (!this.page) return;
|
|
1080
|
+
if (!wasActive || !this.page) return;
|
|
467
1081
|
try {
|
|
468
1082
|
await this.page.send("Page.stopScreencast", {}, { mutation: true });
|
|
469
1083
|
} finally {
|
|
@@ -471,71 +1085,288 @@ export class AgentBrowserRuntime {
|
|
|
471
1085
|
}
|
|
472
1086
|
}
|
|
473
1087
|
|
|
474
|
-
|
|
475
|
-
|
|
1088
|
+
/** Return truthful fresh stream/fallback pixels; mutation/session changes revoke pending captures. */
|
|
1089
|
+
async frame(
|
|
1090
|
+
maxBytes: number,
|
|
1091
|
+
quality: number = DEFAULT_JPEG_QUALITY,
|
|
1092
|
+
waitMs = 500,
|
|
1093
|
+
): Promise<RuntimeFrame> {
|
|
1094
|
+
if (!Number.isInteger(maxBytes) || maxBytes < 1) {
|
|
476
1095
|
throw new RangeError("maxBytes must be positive");
|
|
477
|
-
|
|
478
|
-
if (
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
//
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
1096
|
+
}
|
|
1097
|
+
if (!Number.isFinite(quality)) throw new RangeError("JPEG quality must be finite");
|
|
1098
|
+
const requestedQuality = Math.max(1, Math.min(100, Math.round(quality)));
|
|
1099
|
+
// All reads protect in-flight pixels, but screenshot-only qualities do not
|
|
1100
|
+
// renew the shared JPEG stream when a viewer changes their preference.
|
|
1101
|
+
const release = this.jpegDemand.pin(requestedQuality === this.screencastQuality);
|
|
1102
|
+
try {
|
|
1103
|
+
await this.jpegDemand.serialize(async () => {
|
|
1104
|
+
const page = await this.requirePage();
|
|
1105
|
+
// Nonmatching quality keeps the existing screenshot-only path. A video
|
|
1106
|
+
// viewer choosing JPEG100 must not start an unused JPEG95 stream.
|
|
1107
|
+
if (
|
|
1108
|
+
requestedQuality !== this.screencastQuality ||
|
|
1109
|
+
this.screencastActive ||
|
|
1110
|
+
this.screencastStartFailed
|
|
1111
|
+
) {
|
|
1112
|
+
return;
|
|
1113
|
+
}
|
|
1114
|
+
this.screencastActive = true;
|
|
1115
|
+
try {
|
|
1116
|
+
await this.startScreencastSession(page);
|
|
1117
|
+
} catch {
|
|
1118
|
+
// Shutdown is terminal, not an optional stream failure to fall back from.
|
|
1119
|
+
this.assertRuntimeOpen();
|
|
1120
|
+
// Stream support is optional. Preserve fresh screenshots, and avoid
|
|
1121
|
+
// retrying a failed setup at every 250ms poll in the same demand period.
|
|
1122
|
+
console.warn("Shared Browser JPEG screencast is unavailable; using screenshot fallback");
|
|
1123
|
+
this.screencastActive = false;
|
|
1124
|
+
this.screencastStartFailed = true;
|
|
1125
|
+
}
|
|
497
1126
|
});
|
|
498
|
-
|
|
499
|
-
if (
|
|
500
|
-
|
|
1127
|
+
this.assertRuntimeOpen();
|
|
1128
|
+
if (requestedQuality !== this.screencastQuality) {
|
|
1129
|
+
// Viewers choose their own quality; never restart the shared stream or
|
|
1130
|
+
// classify its health from another viewer's screenshot preference.
|
|
1131
|
+
const cached = this.capturePolicy.readScreenshot(
|
|
1132
|
+
performance.now(),
|
|
1133
|
+
requestedQuality,
|
|
1134
|
+
maxBytes,
|
|
1135
|
+
);
|
|
1136
|
+
return cached ?? (await this.captureScreenshot(maxBytes, requestedQuality, true));
|
|
1137
|
+
}
|
|
1138
|
+
// Expiry discards pixels without resetting fallback dwell/recovery evidence.
|
|
1139
|
+
// Mutations and connection changes use the stronger invalidation below.
|
|
1140
|
+
const generation = this.capturePolicy.currentGeneration();
|
|
1141
|
+
const receivedAt = this.screencastFrameReceivedAt;
|
|
1142
|
+
if (receivedAt === null || performance.now() - receivedAt > CAPTURE_MAX_AGE_MS) {
|
|
1143
|
+
this.screencastFrame = null;
|
|
1144
|
+
this.screencastFrameReceivedAt = null;
|
|
1145
|
+
}
|
|
1146
|
+
if (this.screencastActive && this.capturePolicy.canUseStream(performance.now())) {
|
|
1147
|
+
const streamed = this.screencastFrame ?? (await this.waitForFrame(waitMs));
|
|
1148
|
+
this.assertRuntimeOpen();
|
|
1149
|
+
if (!this.capturePolicy.isCurrent(generation)) {
|
|
1150
|
+
throw new CdpUnavailableError(
|
|
1151
|
+
"Browser capture was invalidated while waiting for a frame",
|
|
1152
|
+
);
|
|
1153
|
+
}
|
|
1154
|
+
if (streamed && streamed.byteLength <= maxBytes) {
|
|
1155
|
+
return streamed;
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
this.capturePolicy.enterFallback(performance.now());
|
|
1160
|
+
const cached = this.capturePolicy.readScreenshot(
|
|
1161
|
+
performance.now(),
|
|
1162
|
+
requestedQuality,
|
|
1163
|
+
maxBytes,
|
|
1164
|
+
);
|
|
1165
|
+
if (cached) {
|
|
1166
|
+
return cached;
|
|
1167
|
+
}
|
|
1168
|
+
return await this.captureScreenshot(maxBytes, requestedQuality);
|
|
1169
|
+
} finally {
|
|
1170
|
+
release();
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
/** Capture the configured viewport and cache only an unchanged-session/mutation completion. */
|
|
1175
|
+
private async captureScreenshot(
|
|
1176
|
+
maxBytes: number,
|
|
1177
|
+
requestedQuality: number,
|
|
1178
|
+
preserveStreamRecovery = false,
|
|
1179
|
+
): Promise<RuntimeFrame> {
|
|
1180
|
+
const page = await this.requirePage();
|
|
1181
|
+
const screenshotGeneration = this.capturePolicy.beginScreenshot({
|
|
1182
|
+
preserveRecovery: preserveStreamRecovery,
|
|
1183
|
+
});
|
|
1184
|
+
try {
|
|
1185
|
+
const viewport = this.requireViewport();
|
|
1186
|
+
const expectedPixels = captureDimensions(viewport, viewport.captureScale);
|
|
1187
|
+
const readClip = async () => {
|
|
1188
|
+
const metrics = await page.send<{ cssVisualViewport: { pageX: number; pageY: number } }>(
|
|
1189
|
+
"Page.getLayoutMetrics",
|
|
1190
|
+
);
|
|
1191
|
+
this.assertCaptureCurrent(page, screenshotGeneration);
|
|
501
1192
|
return {
|
|
502
|
-
|
|
503
|
-
|
|
1193
|
+
x: metrics.cssVisualViewport.pageX,
|
|
1194
|
+
y: metrics.cssVisualViewport.pageY,
|
|
504
1195
|
width: viewport.width,
|
|
505
1196
|
height: viewport.height,
|
|
1197
|
+
scale: (viewport.captureScale ?? 1) / viewport.deviceScaleFactor,
|
|
1198
|
+
};
|
|
1199
|
+
};
|
|
1200
|
+
// Explicitly capture the visible CSS viewport, independent of the native window size.
|
|
1201
|
+
// Preserve scroll position and normalize device pixel ratio to the frame/input contract.
|
|
1202
|
+
let clip = await readClip();
|
|
1203
|
+
let resynced = false;
|
|
1204
|
+
// Reduce quality gradually only when a detailed frame exceeds the byte budget.
|
|
1205
|
+
const initialQuality = requestedQuality;
|
|
1206
|
+
const candidates = [...new Set([initialQuality, 90, 85, 75, 65, 50, 35, 20, 10, 1])].filter(
|
|
1207
|
+
(candidate) => candidate <= initialQuality,
|
|
1208
|
+
);
|
|
1209
|
+
const capture = async (quality: number) => {
|
|
1210
|
+
this.assertCaptureCurrent(page, screenshotGeneration);
|
|
1211
|
+
const result = await page.send<{ data: string }>("Page.captureScreenshot", {
|
|
1212
|
+
format: "jpeg",
|
|
1213
|
+
quality,
|
|
1214
|
+
fromSurface: true,
|
|
1215
|
+
// Beyond-viewport capture replaces emulation and resets enabled touch.
|
|
1216
|
+
captureBeyondViewport: false,
|
|
1217
|
+
clip,
|
|
1218
|
+
});
|
|
1219
|
+
this.assertCaptureCurrent(page, screenshotGeneration);
|
|
1220
|
+
const dimensions = readJpegFrameDimensions(result.data);
|
|
1221
|
+
if (!dimensions) throw new Error("Chromium returned a malformed JPEG screenshot");
|
|
1222
|
+
return {
|
|
1223
|
+
data: result.data,
|
|
1224
|
+
dimensions,
|
|
1225
|
+
byteLength: Buffer.byteLength(result.data, "base64"),
|
|
1226
|
+
};
|
|
1227
|
+
};
|
|
1228
|
+
const matchesViewport = (dimensions: { width: number; height: number }) =>
|
|
1229
|
+
dimensions.width === expectedPixels.width && dimensions.height === expectedPixels.height;
|
|
1230
|
+
|
|
1231
|
+
for (const candidate of candidates) {
|
|
1232
|
+
const boundedQuality = Math.max(1, Math.min(100, Math.round(candidate)));
|
|
1233
|
+
let pixels = await capture(boundedQuality);
|
|
1234
|
+
if (!matchesViewport(pixels.dimensions) && !resynced) {
|
|
1235
|
+
// Retain upstream viewport recovery, with captureScale rather than DPR:
|
|
1236
|
+
// mobile CSS emulation and image density are independently configured.
|
|
1237
|
+
// Recapture pixels only; uncertain input is never replayed.
|
|
1238
|
+
resynced = true;
|
|
1239
|
+
const attachmentGeneration = this.attachmentGeneration;
|
|
1240
|
+
await this.applyEmulation(page, viewport);
|
|
1241
|
+
this.assertAttachmentCurrent(page.connection, attachmentGeneration);
|
|
1242
|
+
this.assertCaptureCurrent(page, screenshotGeneration);
|
|
1243
|
+
if (viewport !== this.viewport)
|
|
1244
|
+
throw new CdpUnavailableError("Browser viewport changed during resync");
|
|
1245
|
+
this.emulationAppliedPage = page;
|
|
1246
|
+
clip = await readClip();
|
|
1247
|
+
pixels = await capture(boundedQuality);
|
|
1248
|
+
}
|
|
1249
|
+
if (!matchesViewport(pixels.dimensions)) {
|
|
1250
|
+
throw new Error(
|
|
1251
|
+
`Chromium returned a ${pixels.dimensions.width}x${pixels.dimensions.height} JPEG instead of ${expectedPixels.width}x${expectedPixels.height}`,
|
|
1252
|
+
);
|
|
1253
|
+
}
|
|
1254
|
+
if (pixels.byteLength > maxBytes) continue;
|
|
1255
|
+
|
|
1256
|
+
const frame: RuntimeFrame = {
|
|
1257
|
+
dataBase64: pixels.data,
|
|
1258
|
+
byteLength: pixels.byteLength,
|
|
1259
|
+
width: pixels.dimensions.width,
|
|
1260
|
+
height: pixels.dimensions.height,
|
|
506
1261
|
transport: "screenshot",
|
|
507
1262
|
capturedAt: new Date().toISOString(),
|
|
508
|
-
}
|
|
1263
|
+
};
|
|
1264
|
+
this.capturePolicy.rememberScreenshot(
|
|
1265
|
+
screenshotGeneration,
|
|
1266
|
+
frame,
|
|
1267
|
+
performance.now(),
|
|
1268
|
+
requestedQuality,
|
|
1269
|
+
maxBytes,
|
|
1270
|
+
);
|
|
1271
|
+
return frame;
|
|
509
1272
|
}
|
|
1273
|
+
throw new Error(`JPEG screenshot exceeds ${maxBytes} bytes at minimum quality`);
|
|
1274
|
+
} finally {
|
|
1275
|
+
this.capturePolicy.endScreenshot(screenshotGeneration, performance.now());
|
|
510
1276
|
}
|
|
511
|
-
throw new Error(`JPEG screenshot exceeds ${maxBytes} bytes at minimum quality`);
|
|
512
1277
|
}
|
|
513
1278
|
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
1279
|
+
/** Refuse async pixels from a superseded session or mutation, without silently replaying capture. */
|
|
1280
|
+
private assertCaptureCurrent(page: CdpSession, generation: number): void {
|
|
1281
|
+
this.assertRuntimeOpen();
|
|
1282
|
+
if (page !== this.page || !this.capturePolicy.isCurrent(generation)) {
|
|
1283
|
+
throw new CdpUnavailableError("Browser capture was invalidated while taking a screenshot");
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
/** Invalidate pre-input pixels even if Chromium cannot confirm the mutation. */
|
|
1288
|
+
private async dispatchInput(
|
|
1289
|
+
method: string,
|
|
1290
|
+
params: Record<string, unknown>,
|
|
1291
|
+
gestureId?: string,
|
|
1292
|
+
mouseModifiers?: number,
|
|
1293
|
+
track?: () => void,
|
|
1294
|
+
): Promise<void> {
|
|
1295
|
+
try {
|
|
1296
|
+
const page = await this.requirePage();
|
|
1297
|
+
if (gestureId) await this.assertLiveInput(gestureId);
|
|
1298
|
+
// Authority is validated. Record cleanup intent only now, immediately before
|
|
1299
|
+
// native publication, so rejected unsent packets never alter it while an
|
|
1300
|
+
// uncertain acknowledgement still leaves it in place.
|
|
1301
|
+
if (track) {
|
|
1302
|
+
if (this.heldOwner && this.heldOwner !== page) this.discardHeldState();
|
|
1303
|
+
track();
|
|
1304
|
+
if (this.holdsInput()) this.heldOwner ??= page;
|
|
1305
|
+
}
|
|
1306
|
+
const originalInput = gestureId ? this.liveInput : null;
|
|
1307
|
+
// A physical DOM mouse event may arrive after Control/Shift was pressed
|
|
1308
|
+
// outside this canvas. Explicit zero also overrides a prior held-key mask.
|
|
1309
|
+
const modifiers = mouseModifiers ?? heldKeyModifiers(this.heldKeys.values());
|
|
1310
|
+
const nativeParams =
|
|
1311
|
+
method === "Input.dispatchMouseEvent" && (mouseModifiers !== undefined || modifiers !== 0)
|
|
1312
|
+
? { ...params, modifiers }
|
|
1313
|
+
: params;
|
|
1314
|
+
await page.send(method, nativeParams, { mutation: true });
|
|
1315
|
+
if (gestureId) {
|
|
1316
|
+
// An input can synchronously activate a link or submit a form. Its CDP
|
|
1317
|
+
// acknowledgment succeeds even when navigation ends the old channel.
|
|
1318
|
+
// Attachment replacement and uncertain sends retain their failure path.
|
|
1319
|
+
if (
|
|
1320
|
+
originalInput &&
|
|
1321
|
+
page === this.page &&
|
|
1322
|
+
originalInput.generation === this.attachmentGeneration &&
|
|
1323
|
+
originalInput.documentGeneration !== this.documentGeneration
|
|
1324
|
+
) {
|
|
1325
|
+
await this.endLiveInput(gestureId);
|
|
1326
|
+
return;
|
|
1327
|
+
}
|
|
1328
|
+
await this.assertLiveInput(gestureId);
|
|
1329
|
+
this.renewLiveInput(gestureId);
|
|
1330
|
+
}
|
|
1331
|
+
} finally {
|
|
1332
|
+
this.invalidateScreencastFrame();
|
|
1333
|
+
}
|
|
525
1334
|
}
|
|
526
1335
|
|
|
527
|
-
async mouseMove(x: number, y: number): Promise<void> {
|
|
1336
|
+
async mouseMove(x: number, y: number, gestureId?: string, modifiers?: number): Promise<void> {
|
|
528
1337
|
this.assertPoint(x, y);
|
|
529
|
-
|
|
1338
|
+
this.assertMouseModifiers(modifiers);
|
|
1339
|
+
await this.dispatchInput(
|
|
530
1340
|
"Input.dispatchMouseEvent",
|
|
531
1341
|
{
|
|
532
1342
|
type: "mouseMoved",
|
|
533
1343
|
x,
|
|
534
1344
|
y,
|
|
535
|
-
button
|
|
1345
|
+
// Native scrollbar dragging needs the held button as well as its mask.
|
|
1346
|
+
button: this.heldMoveButton(),
|
|
536
1347
|
buttons: this.buttonMask(),
|
|
537
1348
|
},
|
|
538
|
-
|
|
1349
|
+
gestureId,
|
|
1350
|
+
modifiers,
|
|
1351
|
+
() => {
|
|
1352
|
+
this.lastMousePoint = { x, y };
|
|
1353
|
+
},
|
|
1354
|
+
);
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
/** Fixed native mouse departure; never maps an outside point to a page control. */
|
|
1358
|
+
async mouseLeave(gestureId: string): Promise<void> {
|
|
1359
|
+
if (this.heldButtons.size > 0) throw new Error("Release held mouse buttons before leaving");
|
|
1360
|
+
await this.dispatchInput(
|
|
1361
|
+
"Input.dispatchMouseEvent",
|
|
1362
|
+
{
|
|
1363
|
+
type: "mouseMoved",
|
|
1364
|
+
x: -1,
|
|
1365
|
+
y: -1,
|
|
1366
|
+
button: "none",
|
|
1367
|
+
buttons: 0,
|
|
1368
|
+
},
|
|
1369
|
+
gestureId,
|
|
539
1370
|
);
|
|
540
1371
|
}
|
|
541
1372
|
|
|
@@ -544,9 +1375,12 @@ export class AgentBrowserRuntime {
|
|
|
544
1375
|
y: number,
|
|
545
1376
|
button: MouseButton = "left",
|
|
546
1377
|
clickCount = 1,
|
|
1378
|
+
gestureId?: string,
|
|
1379
|
+
modifiers?: number,
|
|
547
1380
|
): Promise<void> {
|
|
548
1381
|
this.assertPoint(x, y);
|
|
549
|
-
|
|
1382
|
+
this.assertMouseModifiers(modifiers);
|
|
1383
|
+
await this.dispatchInput(
|
|
550
1384
|
"Input.dispatchMouseEvent",
|
|
551
1385
|
{
|
|
552
1386
|
type: "mousePressed",
|
|
@@ -556,14 +1390,28 @@ export class AgentBrowserRuntime {
|
|
|
556
1390
|
buttons: this.buttonMask(button),
|
|
557
1391
|
clickCount,
|
|
558
1392
|
},
|
|
559
|
-
|
|
1393
|
+
gestureId,
|
|
1394
|
+
modifiers,
|
|
1395
|
+
() => {
|
|
1396
|
+
// Publication may succeed before an acknowledgement is lost, so a
|
|
1397
|
+
// possibly held button is tracked from the send boundary.
|
|
1398
|
+
this.heldButtons.add(button);
|
|
1399
|
+
this.lastMousePoint = { x, y };
|
|
1400
|
+
},
|
|
560
1401
|
);
|
|
561
|
-
this.heldButtons.add(button);
|
|
562
1402
|
}
|
|
563
1403
|
|
|
564
|
-
async mouseUp(
|
|
1404
|
+
async mouseUp(
|
|
1405
|
+
x: number,
|
|
1406
|
+
y: number,
|
|
1407
|
+
button: MouseButton = "left",
|
|
1408
|
+
clickCount = 1,
|
|
1409
|
+
gestureId?: string,
|
|
1410
|
+
modifiers?: number,
|
|
1411
|
+
): Promise<void> {
|
|
565
1412
|
this.assertPoint(x, y);
|
|
566
|
-
|
|
1413
|
+
this.assertMouseModifiers(modifiers);
|
|
1414
|
+
await this.dispatchInput(
|
|
567
1415
|
"Input.dispatchMouseEvent",
|
|
568
1416
|
{
|
|
569
1417
|
type: "mouseReleased",
|
|
@@ -573,14 +1421,26 @@ export class AgentBrowserRuntime {
|
|
|
573
1421
|
buttons: this.buttonMask(undefined, button),
|
|
574
1422
|
clickCount,
|
|
575
1423
|
},
|
|
576
|
-
|
|
1424
|
+
gestureId,
|
|
1425
|
+
modifiers,
|
|
1426
|
+
() => {
|
|
1427
|
+
this.lastMousePoint = { x, y };
|
|
1428
|
+
},
|
|
577
1429
|
);
|
|
578
1430
|
this.heldButtons.delete(button);
|
|
579
1431
|
}
|
|
580
1432
|
|
|
581
|
-
async wheel(
|
|
1433
|
+
async wheel(
|
|
1434
|
+
x: number,
|
|
1435
|
+
y: number,
|
|
1436
|
+
deltaX: number,
|
|
1437
|
+
deltaY: number,
|
|
1438
|
+
gestureId?: string,
|
|
1439
|
+
modifiers?: number,
|
|
1440
|
+
): Promise<void> {
|
|
582
1441
|
this.assertPoint(x, y);
|
|
583
|
-
|
|
1442
|
+
this.assertMouseModifiers(modifiers);
|
|
1443
|
+
await this.dispatchInput(
|
|
584
1444
|
"Input.dispatchMouseEvent",
|
|
585
1445
|
{
|
|
586
1446
|
type: "mouseWheel",
|
|
@@ -591,16 +1451,46 @@ export class AgentBrowserRuntime {
|
|
|
591
1451
|
button: "none",
|
|
592
1452
|
buttons: this.buttonMask(),
|
|
593
1453
|
},
|
|
594
|
-
|
|
1454
|
+
gestureId,
|
|
1455
|
+
modifiers,
|
|
595
1456
|
);
|
|
596
1457
|
}
|
|
597
1458
|
|
|
598
|
-
|
|
599
|
-
|
|
1459
|
+
/** Insert a committed paste/IME value on the original guarded attachment. */
|
|
1460
|
+
async insertText(text: string, gestureId?: string): Promise<void> {
|
|
1461
|
+
await this.dispatchInput("Input.insertText", { text }, gestureId);
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
/** Track cleanup intent before publication, including an uncertain key-down reply. */
|
|
1465
|
+
async dispatchKey(event: BrowserGestureKeyEvent, gestureId: string): Promise<void> {
|
|
1466
|
+
const parsed = browserGestureKeySchema.parse(event);
|
|
1467
|
+
await this.assertLiveInput(gestureId);
|
|
1468
|
+
const held = this.heldKeys.has(parsed.code);
|
|
1469
|
+
if (parsed.type === "down" && held !== parsed.repeat) {
|
|
1470
|
+
throw new Error(
|
|
1471
|
+
parsed.repeat ? "Key repeat has no matching press" : "Key is already pressed",
|
|
1472
|
+
);
|
|
1473
|
+
}
|
|
1474
|
+
if (parsed.type === "up" && !held) throw new Error("Key release has no matching press");
|
|
1475
|
+
if (parsed.type === "down") this.assertHeldKeyCapacity(parsed.code);
|
|
1476
|
+
await this.dispatchInput(
|
|
1477
|
+
"Input.dispatchKeyEvent",
|
|
1478
|
+
nativeKeyEvent(parsed),
|
|
1479
|
+
gestureId,
|
|
1480
|
+
undefined,
|
|
1481
|
+
parsed.type === "down"
|
|
1482
|
+
? () => this.heldKeys.set(parsed.code, { key: parsed.key, code: parsed.code })
|
|
1483
|
+
: undefined,
|
|
1484
|
+
);
|
|
1485
|
+
if (parsed.type === "up") this.heldKeys.delete(parsed.code);
|
|
600
1486
|
}
|
|
601
1487
|
|
|
602
|
-
|
|
603
|
-
|
|
1488
|
+
/** Track intent before publication. Internal discrete presses can use the
|
|
1489
|
+
* same opaque channel as live input so a lost ACK retains original-page cleanup. */
|
|
1490
|
+
async keyDown(key: string, code = key, gestureId?: string): Promise<void> {
|
|
1491
|
+
if (gestureId) await this.assertLiveInput(gestureId);
|
|
1492
|
+
this.assertHeldKeyCapacity(code);
|
|
1493
|
+
await this.dispatchInput(
|
|
604
1494
|
"Input.dispatchKeyEvent",
|
|
605
1495
|
{
|
|
606
1496
|
type: "keyDown",
|
|
@@ -608,58 +1498,215 @@ export class AgentBrowserRuntime {
|
|
|
608
1498
|
code,
|
|
609
1499
|
text: key.length === 1 ? key : undefined,
|
|
610
1500
|
},
|
|
611
|
-
|
|
1501
|
+
gestureId,
|
|
1502
|
+
undefined,
|
|
1503
|
+
() => this.heldKeys.set(code, { key, code }),
|
|
612
1504
|
);
|
|
613
|
-
this.heldKeys.add(key);
|
|
614
1505
|
}
|
|
615
1506
|
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
);
|
|
622
|
-
this.heldKeys.delete(key);
|
|
1507
|
+
/** Release on a current channel; its end cleanup still owns the original
|
|
1508
|
+
* page if navigation or attachment loss prevents this ordinary key-up. */
|
|
1509
|
+
async keyUp(key: string, code = key, gestureId?: string): Promise<void> {
|
|
1510
|
+
await this.dispatchInput("Input.dispatchKeyEvent", { type: "keyUp", key, code }, gestureId);
|
|
1511
|
+
this.heldKeys.delete(code);
|
|
623
1512
|
}
|
|
624
1513
|
|
|
1514
|
+
/**
|
|
1515
|
+
* The caller supplies the complete active contact set. CDP touchMove does not
|
|
1516
|
+
* release omitted contacts, so publish explicit contact ends before moving the
|
|
1517
|
+
* survivors. Any uncertain publication retains cleanup intent until cancel.
|
|
1518
|
+
*/
|
|
625
1519
|
async touch(
|
|
626
1520
|
type: "touchStart" | "touchMove" | "touchEnd" | "touchCancel",
|
|
627
1521
|
points: Array<{ x: number; y: number; id?: number }>,
|
|
1522
|
+
gestureId?: string,
|
|
628
1523
|
): Promise<void> {
|
|
629
1524
|
for (const point of points) this.assertPoint(point.x, point.y);
|
|
630
|
-
|
|
1525
|
+
if (gestureId) await this.assertLiveInput(gestureId);
|
|
1526
|
+
const contacts = points.map((point, index) => ({ ...point, id: point.id ?? index }));
|
|
1527
|
+
const activeIds = new Set(contacts.map((point) => point.id));
|
|
1528
|
+
const removed =
|
|
1529
|
+
type === "touchMove"
|
|
1530
|
+
? [...this.activeTouches.values()].filter((point) => !activeIds.has(point.id))
|
|
1531
|
+
: [];
|
|
1532
|
+
const hold = () => {
|
|
1533
|
+
this.heldTouches = true;
|
|
1534
|
+
};
|
|
1535
|
+
if (removed.length > 0) {
|
|
1536
|
+
await this.dispatchInput(
|
|
1537
|
+
"Input.dispatchTouchEvent",
|
|
1538
|
+
{ type: "touchEnd", touchPoints: removed },
|
|
1539
|
+
gestureId,
|
|
1540
|
+
undefined,
|
|
1541
|
+
hold,
|
|
1542
|
+
);
|
|
1543
|
+
}
|
|
1544
|
+
await this.dispatchInput(
|
|
631
1545
|
"Input.dispatchTouchEvent",
|
|
632
|
-
{
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
1546
|
+
{ type, touchPoints: contacts },
|
|
1547
|
+
gestureId,
|
|
1548
|
+
undefined,
|
|
1549
|
+
hold,
|
|
1550
|
+
);
|
|
1551
|
+
// Navigation cleanup already discarded this channel's contacts. Do not
|
|
1552
|
+
// recreate held-touch state after its acknowledged start caused navigation.
|
|
1553
|
+
if (gestureId && this.liveInput?.id !== gestureId) return;
|
|
1554
|
+
this.activeTouches.clear();
|
|
1555
|
+
if (type === "touchStart" || type === "touchMove") {
|
|
1556
|
+
for (const point of contacts) this.activeTouches.set(point.id, point);
|
|
1557
|
+
}
|
|
1558
|
+
this.heldTouches = this.activeTouches.size > 0;
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
/**
|
|
1562
|
+
* Bind decoded geometry to its exact native attachment and document. Compare
|
|
1563
|
+
* after cleanup/reattach awaits so a same-URL reload cannot silently admit a
|
|
1564
|
+
* press against a replacement page. A mismatch publishes no physical input.
|
|
1565
|
+
*/
|
|
1566
|
+
async beginLiveInput(id: string, expectedInputGeneration: string): Promise<void> {
|
|
1567
|
+
if (this.liveInput) await this.endLiveInput(this.liveInput.id);
|
|
1568
|
+
const page = await this.requirePage();
|
|
1569
|
+
const actualInputGeneration = formatRuntimeInputGeneration(
|
|
1570
|
+
this.attachmentGeneration,
|
|
1571
|
+
this.documentGeneration,
|
|
641
1572
|
);
|
|
1573
|
+
if (expectedInputGeneration !== actualInputGeneration) {
|
|
1574
|
+
throw new CdpUnavailableError("Live browser input attachment changed before admission");
|
|
1575
|
+
}
|
|
1576
|
+
// No asynchronous boundary may separate this comparison from the binding.
|
|
1577
|
+
const now = Date.now();
|
|
1578
|
+
this.liveInput = {
|
|
1579
|
+
id,
|
|
1580
|
+
page,
|
|
1581
|
+
generation: this.attachmentGeneration,
|
|
1582
|
+
documentGeneration: this.documentGeneration,
|
|
1583
|
+
idleUntil: now + GESTURE_IDLE_MS,
|
|
1584
|
+
expiresAt: now + GESTURE_LIFETIME_MS,
|
|
1585
|
+
timer: null,
|
|
1586
|
+
};
|
|
1587
|
+
this.scheduleLiveInputExpiry();
|
|
1588
|
+
}
|
|
1589
|
+
|
|
1590
|
+
/** Compare after every asynchronous transport boundary, including automatic reconnect/reattach. */
|
|
1591
|
+
async assertLiveInput(id: string): Promise<void> {
|
|
1592
|
+
const page = await this.requirePage();
|
|
1593
|
+
if (
|
|
1594
|
+
this.liveInput?.id === id &&
|
|
1595
|
+
(Date.now() >= this.liveInput.idleUntil || Date.now() >= this.liveInput.expiresAt)
|
|
1596
|
+
) {
|
|
1597
|
+
await this.endLiveInput(id);
|
|
1598
|
+
}
|
|
1599
|
+
if (
|
|
1600
|
+
!this.liveInput ||
|
|
1601
|
+
this.liveInput.id !== id ||
|
|
1602
|
+
this.liveInput.page !== page ||
|
|
1603
|
+
this.liveInput.generation !== this.attachmentGeneration ||
|
|
1604
|
+
this.liveInput.documentGeneration !== this.documentGeneration
|
|
1605
|
+
) {
|
|
1606
|
+
throw new CdpUnavailableError("Live browser input attachment changed");
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
/** Cleanup is idempotent and targets the original attachment, never whatever is currently active. */
|
|
1611
|
+
async endLiveInput(id: string): Promise<void> {
|
|
1612
|
+
const current = this.liveInput;
|
|
1613
|
+
if (!current || current.id !== id) return;
|
|
1614
|
+
this.liveInput = null;
|
|
1615
|
+
if (current.timer) clearTimeout(current.timer);
|
|
1616
|
+
await this.releaseHeldInput(current.page);
|
|
642
1617
|
}
|
|
643
1618
|
|
|
644
|
-
|
|
645
|
-
|
|
1619
|
+
/** Only acknowledged input extends idle time; metadata/checks cannot keep a stranded press alive. */
|
|
1620
|
+
private renewLiveInput(id: string): void {
|
|
1621
|
+
if (this.liveInput?.id !== id) return;
|
|
1622
|
+
this.liveInput.idleUntil = Date.now() + GESTURE_IDLE_MS;
|
|
1623
|
+
this.scheduleLiveInputExpiry();
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
/** Runtime-owned expiry survives plugin disconnect or a fenced supervisor IPC client. */
|
|
1627
|
+
private scheduleLiveInputExpiry(): void {
|
|
1628
|
+
const current = this.liveInput;
|
|
1629
|
+
if (!current) return;
|
|
1630
|
+
if (current.timer) clearTimeout(current.timer);
|
|
1631
|
+
const delay = Math.max(0, Math.min(current.idleUntil, current.expiresAt) - Date.now());
|
|
1632
|
+
current.timer = setTimeout(() => {
|
|
1633
|
+
if (this.liveInput !== current) return;
|
|
1634
|
+
void this.endLiveInput(current.id).catch(() => undefined);
|
|
1635
|
+
}, delay);
|
|
1636
|
+
current.timer.unref?.();
|
|
1637
|
+
}
|
|
1638
|
+
|
|
1639
|
+
/** Fixed hit-test, safe cursor names only. Cursor failures cannot fail an acknowledged input. */
|
|
1640
|
+
async cursorAt(x: number, y: number, gestureId: string): Promise<BrowserCursor | null> {
|
|
1641
|
+
try {
|
|
1642
|
+
this.assertPoint(x, y);
|
|
1643
|
+
await this.assertLiveInput(gestureId);
|
|
1644
|
+
const page = this.liveInput!.page;
|
|
1645
|
+
const result = await page.send<{ result: { value?: unknown } }>(
|
|
1646
|
+
"Runtime.evaluate",
|
|
1647
|
+
{
|
|
1648
|
+
expression: browserCursorExpression(x, y),
|
|
1649
|
+
returnByValue: true,
|
|
1650
|
+
},
|
|
1651
|
+
{ timeoutMs: Math.min(this.timeoutMs, CURSOR_SAMPLE_TIMEOUT_MS) },
|
|
1652
|
+
);
|
|
1653
|
+
await this.assertLiveInput(gestureId);
|
|
1654
|
+
const parsed = browserCursorSchema.safeParse(result.result.value);
|
|
1655
|
+
return parsed.success ? parsed.data : null;
|
|
1656
|
+
} catch {
|
|
1657
|
+
return null;
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
private holdsInput(): boolean {
|
|
1662
|
+
return this.heldButtons.size > 0 || this.heldKeys.size > 0 || this.heldTouches;
|
|
1663
|
+
}
|
|
1664
|
+
|
|
1665
|
+
/** Forget held state whose attachment is gone; it is never replayed on a replacement. */
|
|
1666
|
+
private discardHeldState(): void {
|
|
1667
|
+
this.heldButtons.clear();
|
|
1668
|
+
this.heldKeys.clear();
|
|
1669
|
+
this.heldTouches = false;
|
|
1670
|
+
this.activeTouches.clear();
|
|
1671
|
+
this.heldOwner = null;
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
/**
|
|
1675
|
+
* Release only what `page` itself holds. State published on another attachment
|
|
1676
|
+
* is left to that attachment's own cleanup, never retargeted here.
|
|
1677
|
+
*/
|
|
1678
|
+
async releaseHeldInput(page = this.page): Promise<void> {
|
|
646
1679
|
if (!page) {
|
|
647
|
-
this.
|
|
648
|
-
this.heldKeys.clear();
|
|
1680
|
+
this.discardHeldState();
|
|
649
1681
|
return;
|
|
650
1682
|
}
|
|
1683
|
+
if (this.heldOwner && this.heldOwner !== page) return;
|
|
1684
|
+
this.heldOwner = null;
|
|
651
1685
|
const buttons = [...this.heldButtons];
|
|
652
|
-
const
|
|
1686
|
+
const { x, y } = this.lastMousePoint;
|
|
1687
|
+
const keys = [...this.heldKeys.values()];
|
|
653
1688
|
this.heldButtons.clear();
|
|
654
1689
|
this.heldKeys.clear();
|
|
1690
|
+
const touchHeld = this.heldTouches;
|
|
1691
|
+
this.heldTouches = false;
|
|
1692
|
+
this.activeTouches.clear();
|
|
655
1693
|
await Promise.allSettled([
|
|
1694
|
+
...(touchHeld
|
|
1695
|
+
? [
|
|
1696
|
+
page.send(
|
|
1697
|
+
"Input.dispatchTouchEvent",
|
|
1698
|
+
{ type: "touchCancel", touchPoints: [] },
|
|
1699
|
+
{ mutation: true },
|
|
1700
|
+
),
|
|
1701
|
+
]
|
|
1702
|
+
: []),
|
|
656
1703
|
...buttons.map((button) =>
|
|
657
1704
|
page.send(
|
|
658
1705
|
"Input.dispatchMouseEvent",
|
|
659
1706
|
{
|
|
660
1707
|
type: "mouseReleased",
|
|
661
|
-
x
|
|
662
|
-
y
|
|
1708
|
+
x,
|
|
1709
|
+
y,
|
|
663
1710
|
button,
|
|
664
1711
|
buttons: 0,
|
|
665
1712
|
clickCount: 1,
|
|
@@ -667,13 +1714,13 @@ export class AgentBrowserRuntime {
|
|
|
667
1714
|
{ mutation: true },
|
|
668
1715
|
),
|
|
669
1716
|
),
|
|
670
|
-
...keys.map((key) =>
|
|
1717
|
+
...keys.map(({ key, code }) =>
|
|
671
1718
|
page.send(
|
|
672
1719
|
"Input.dispatchKeyEvent",
|
|
673
1720
|
{
|
|
674
1721
|
type: "keyUp",
|
|
675
1722
|
key,
|
|
676
|
-
code
|
|
1723
|
+
code,
|
|
677
1724
|
},
|
|
678
1725
|
{ mutation: true },
|
|
679
1726
|
),
|
|
@@ -681,35 +1728,112 @@ export class AgentBrowserRuntime {
|
|
|
681
1728
|
]);
|
|
682
1729
|
}
|
|
683
1730
|
|
|
1731
|
+
/**
|
|
1732
|
+
* Concurrent callers share one in-flight shutdown; a failed one is not remembered as
|
|
1733
|
+
* success, so a later call retries and reports the same unconfirmed state.
|
|
1734
|
+
*/
|
|
684
1735
|
async shutdown(force = false): Promise<void> {
|
|
685
|
-
if (this.
|
|
1736
|
+
if (this.shutDown) return;
|
|
1737
|
+
if (this.shutdownInFlight) return this.shutdownInFlight;
|
|
1738
|
+
const operation = this.performShutdown(force);
|
|
1739
|
+
this.shutdownInFlight = operation;
|
|
1740
|
+
try {
|
|
1741
|
+
await operation;
|
|
1742
|
+
this.shutDown = true;
|
|
1743
|
+
} finally {
|
|
1744
|
+
if (this.shutdownInFlight === operation) this.shutdownInFlight = null;
|
|
1745
|
+
}
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
private async performShutdown(force: boolean): Promise<void> {
|
|
686
1749
|
this.stopping = true;
|
|
687
|
-
|
|
1750
|
+
this.jpegDemand.close();
|
|
1751
|
+
this.screencastActive = false;
|
|
688
1752
|
this.invalidateScreencastFrame();
|
|
689
|
-
|
|
1753
|
+
await this.withVideoTransition(async () => {
|
|
1754
|
+
if (this.liveInput) await this.endLiveInput(this.liveInput.id);
|
|
1755
|
+
await this.releaseHeldInput();
|
|
1756
|
+
this.invalidateScreencastFrame();
|
|
1757
|
+
// Only the controller that launched the browser owns its session daemon.
|
|
1758
|
+
if (this.ownsBrowser) await this.stopDaemon(force);
|
|
1759
|
+
this.connection?.close();
|
|
1760
|
+
this.connection = null;
|
|
1761
|
+
this.page = null;
|
|
1762
|
+
this.targetId = null;
|
|
1763
|
+
this.invalidateScreencastFrame();
|
|
1764
|
+
});
|
|
1765
|
+
}
|
|
1766
|
+
|
|
1767
|
+
/** Close, then require observed exit; escalate to a verified-owned SIGKILL and observe that too. */
|
|
1768
|
+
private async stopDaemon(force: boolean): Promise<void> {
|
|
1769
|
+
let forced = force;
|
|
1770
|
+
if (!forced) {
|
|
690
1771
|
try {
|
|
691
1772
|
await this.invoke(["--session", this.session, "--json", "close"]);
|
|
692
1773
|
} catch {
|
|
693
|
-
|
|
694
|
-
await this.shutdown(true);
|
|
695
|
-
return;
|
|
1774
|
+
forced = true;
|
|
696
1775
|
}
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
if (
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
1776
|
+
// close returns before the session daemon exits. A same-session relaunch would reach
|
|
1777
|
+
// that daemon with its original launch environment (e.g. a stopped private display).
|
|
1778
|
+
if (!forced && !(await this.daemonExited())) forced = true;
|
|
1779
|
+
}
|
|
1780
|
+
if (forced) await this.terminateDaemon();
|
|
1781
|
+
if (forced || this.daemonLaunched) await this.removeIpcMetadata();
|
|
1782
|
+
this.daemonLaunched = false;
|
|
1783
|
+
this.daemon = null;
|
|
1784
|
+
}
|
|
1785
|
+
|
|
1786
|
+
/**
|
|
1787
|
+
* Records the PID-file daemon. Only a PID whose environment carries this runtime's own
|
|
1788
|
+
* nonce gets a verified identity; anything else is observable but never signalled.
|
|
1789
|
+
*/
|
|
1790
|
+
private async captureDaemon(): Promise<void> {
|
|
1791
|
+
const pid = await this.daemonPid();
|
|
1792
|
+
if (pid === null) return;
|
|
1793
|
+
const probe = await probeProcess(pid, this.ownerNonce);
|
|
1794
|
+
this.daemon = { pid, identity: probe.status === "owned" ? probe.identity : null };
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
/**
|
|
1798
|
+
* Observes the daemon this runtime launched. Only `gone` is a positive exit; `unknown`
|
|
1799
|
+
* (no PID record, unreadable identity) is never success and never authorises a signal.
|
|
1800
|
+
*/
|
|
1801
|
+
private async daemonState(): Promise<"gone" | "owned" | "alive" | "unknown"> {
|
|
1802
|
+
if (!this.daemonLaunched) return "gone";
|
|
1803
|
+
if (!this.daemon) {
|
|
1804
|
+
await this.captureDaemon();
|
|
1805
|
+
if (!this.daemon) return "unknown";
|
|
1806
|
+
}
|
|
1807
|
+
const { pid, identity } = this.daemon;
|
|
1808
|
+
if (!identity) return processExists(pid) ? "alive" : "gone";
|
|
1809
|
+
const probe = await probeProcess(pid, this.ownerNonce);
|
|
1810
|
+
if (probe.status === "absent" || probe.status === "foreign") return "gone";
|
|
1811
|
+
if (probe.status === "owned")
|
|
1812
|
+
return probe.identity.startTicks === identity.startTicks ? "owned" : "gone";
|
|
1813
|
+
return processExists(pid) ? "unknown" : "gone";
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
/** True only after positive observation of exit, within the bound. */
|
|
1817
|
+
private async daemonExited(): Promise<boolean> {
|
|
1818
|
+
for (const deadline = Date.now() + this.daemonExitWaitMs; Date.now() < deadline; ) {
|
|
1819
|
+
if ((await this.daemonState()) === "gone") return true;
|
|
1820
|
+
await new Promise((resolve) => setTimeout(resolve, SCREENCAST_SETUP_RETRY_MS));
|
|
1821
|
+
}
|
|
1822
|
+
return (await this.daemonState()) === "gone";
|
|
1823
|
+
}
|
|
1824
|
+
|
|
1825
|
+
/** Signals only a daemon freshly re-verified as owned, then confirms its exit. */
|
|
1826
|
+
private async terminateDaemon(): Promise<void> {
|
|
1827
|
+
if ((await this.daemonState()) === "owned" && this.daemon) {
|
|
1828
|
+
try {
|
|
1829
|
+
process.kill(this.daemon.pid, "SIGKILL");
|
|
1830
|
+
} catch (error) {
|
|
1831
|
+
if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
|
|
705
1832
|
}
|
|
706
|
-
await this.removeIpcMetadata();
|
|
707
1833
|
}
|
|
708
|
-
|
|
709
|
-
this.
|
|
710
|
-
|
|
711
|
-
this.targetId = null;
|
|
712
|
-
this.invalidateScreencastFrame();
|
|
1834
|
+
// Unverified or unknown daemons are never signalled, but exit must still be observed.
|
|
1835
|
+
if (!(await this.daemonExited()))
|
|
1836
|
+
throw new AgentBrowserUnavailableError("agent-browser daemon exit was not confirmed");
|
|
713
1837
|
}
|
|
714
1838
|
|
|
715
1839
|
private async assertVersion(): Promise<void> {
|
|
@@ -759,8 +1883,18 @@ export class AgentBrowserRuntime {
|
|
|
759
1883
|
}
|
|
760
1884
|
}
|
|
761
1885
|
|
|
762
|
-
private async connectCdp(
|
|
1886
|
+
private async connectCdp(
|
|
1887
|
+
preferredTargetId: string | null = null,
|
|
1888
|
+
activate = this.ownsBrowser,
|
|
1889
|
+
): Promise<void> {
|
|
1890
|
+
this.assertRuntimeOpen();
|
|
1891
|
+
// A launch-selected page remains the intended target even if its first
|
|
1892
|
+
// attachment fails. Otherwise a retry could silently adopt another tab.
|
|
1893
|
+
if (this.targetId === null && preferredTargetId !== null) {
|
|
1894
|
+
this.targetId = preferredTargetId;
|
|
1895
|
+
}
|
|
763
1896
|
const response = await this.invoke(["--session", this.session, "--json", "get", "cdp-url"]);
|
|
1897
|
+
this.assertRuntimeOpen();
|
|
764
1898
|
const url = findString(response, ["cdpUrl", "cdp_url", "url"]);
|
|
765
1899
|
if (!url) throw new AgentBrowserIncompatibleError("agent-browser did not return a CDP URL");
|
|
766
1900
|
const endpoint = new URL(url);
|
|
@@ -771,77 +1905,102 @@ export class AgentBrowserRuntime {
|
|
|
771
1905
|
commandTimeoutMs: this.timeoutMs,
|
|
772
1906
|
connectTimeoutMs: this.timeoutMs,
|
|
773
1907
|
});
|
|
1908
|
+
if (this.stopping) {
|
|
1909
|
+
connection.close();
|
|
1910
|
+
this.assertRuntimeOpen();
|
|
1911
|
+
}
|
|
774
1912
|
this.connection = connection;
|
|
775
1913
|
connection.once("disconnect", () => {
|
|
776
1914
|
if (this.connection !== connection) return;
|
|
777
1915
|
this.page = null;
|
|
778
|
-
|
|
1916
|
+
// Keep the selected page identity. A transport loss does not authorize
|
|
1917
|
+
// adopting another tab, even when the original page still exists.
|
|
1918
|
+
this.emulationAppliedPage = null;
|
|
1919
|
+
this.attachmentGeneration += 1;
|
|
779
1920
|
this.invalidateScreencastFrame();
|
|
780
1921
|
});
|
|
781
1922
|
const targets = await listPageTargets(this.connection);
|
|
1923
|
+
this.assertRuntimeOpen();
|
|
1924
|
+
const expectedTargetId = preferredTargetId ?? this.targetId;
|
|
782
1925
|
const target =
|
|
783
|
-
|
|
1926
|
+
expectedTargetId === null
|
|
1927
|
+
? targets[0]
|
|
1928
|
+
: targets.find((candidate) => candidate.targetId === expectedTargetId);
|
|
1929
|
+
if (expectedTargetId !== null && !target) {
|
|
1930
|
+
throw new CdpUnavailableError("The selected browser page is no longer available");
|
|
1931
|
+
}
|
|
784
1932
|
if (!target) throw new CdpUnavailableError("Chromium has no page target");
|
|
785
|
-
await this.selectTarget(target.targetId);
|
|
1933
|
+
await this.selectTarget(target.targetId, activate);
|
|
786
1934
|
}
|
|
787
1935
|
|
|
788
1936
|
private async requirePage(): Promise<CdpSession> {
|
|
789
|
-
|
|
1937
|
+
this.assertRuntimeOpen();
|
|
1938
|
+
if (this.page && this.connection?.isOpen) {
|
|
1939
|
+
const page = this.page;
|
|
1940
|
+
await this.restoreConfiguredEmulation(page);
|
|
1941
|
+
this.assertRuntimeOpen();
|
|
1942
|
+
if (this.page !== page || !this.connection?.isOpen) {
|
|
1943
|
+
throw new CdpUnavailableError("Browser page changed during emulation restoration");
|
|
1944
|
+
}
|
|
1945
|
+
return page;
|
|
1946
|
+
}
|
|
790
1947
|
await this.reconnect();
|
|
1948
|
+
this.assertRuntimeOpen();
|
|
791
1949
|
if (!this.page) throw new CdpUnavailableError("No page target is attached");
|
|
792
1950
|
return this.page;
|
|
793
1951
|
}
|
|
794
1952
|
|
|
795
1953
|
private requireConnection(): CdpConnection {
|
|
1954
|
+
this.assertRuntimeOpen();
|
|
796
1955
|
if (!this.connection?.isOpen) throw new CdpUnavailableError("Browser runtime is disconnected");
|
|
797
1956
|
return this.connection;
|
|
798
1957
|
}
|
|
799
1958
|
|
|
1959
|
+
/** Shutdown cleanup uses captured attachments directly; acquisition cannot revive them. */
|
|
1960
|
+
private assertRuntimeOpen(): void {
|
|
1961
|
+
if (this.stopping) throw new CdpUnavailableError("Browser runtime is shutting down");
|
|
1962
|
+
}
|
|
1963
|
+
|
|
800
1964
|
private requireViewport(): BrowserViewport {
|
|
801
1965
|
if (!this.viewport) throw new AgentBrowserIncompatibleError("Viewport has not been configured");
|
|
802
1966
|
return this.viewport;
|
|
803
1967
|
}
|
|
804
1968
|
|
|
805
|
-
private async
|
|
806
|
-
|
|
1969
|
+
private async bindPageEvents(page: CdpSession): Promise<void> {
|
|
1970
|
+
// URL equality is insufficient for reload. Main-document and same-document
|
|
1971
|
+
// navigation revoke the channel without treating child-frame loads as a reset.
|
|
1972
|
+
let rootFrameId: string | undefined;
|
|
1973
|
+
page.on("Page.frameNavigated", (event: { frame: { id: string; parentId?: string } }) => {
|
|
1974
|
+
if (page !== this.page || event.frame.parentId) return;
|
|
1975
|
+
rootFrameId = event.frame.id;
|
|
1976
|
+
this.documentGeneration += 1;
|
|
1977
|
+
this.invalidateScreencastFrame();
|
|
1978
|
+
this.video?.invalidateQueuedFrames();
|
|
1979
|
+
void this.endLiveInput(this.liveInput?.id ?? "").catch(() => undefined);
|
|
1980
|
+
});
|
|
1981
|
+
page.on("Page.navigatedWithinDocument", (event: { frameId: string }) => {
|
|
1982
|
+
if (page !== this.page || event.frameId !== rootFrameId) return;
|
|
1983
|
+
this.documentGeneration += 1;
|
|
1984
|
+
this.invalidateScreencastFrame();
|
|
1985
|
+
this.video?.invalidateQueuedFrames();
|
|
1986
|
+
void this.endLiveInput(this.liveInput?.id ?? "").catch(() => undefined);
|
|
1987
|
+
});
|
|
1988
|
+
page.on("Page.screencastFrame", (event: ScreencastFrame) =>
|
|
1989
|
+
this.onScreencastFrame(page, event),
|
|
1990
|
+
);
|
|
807
1991
|
page.on("event", (event: CdpEvent) => {
|
|
808
|
-
if (event.method === "Inspector.targetCrashed"
|
|
1992
|
+
if (event.method === "Inspector.targetCrashed" && this.page === page) {
|
|
1993
|
+
this.documentGeneration += 1;
|
|
1994
|
+
void this.stopVideo();
|
|
1995
|
+
void this.endLiveInput(this.liveInput?.id ?? "").catch(() => undefined);
|
|
1996
|
+
this.emulationAppliedPage = null;
|
|
1997
|
+
this.invalidateScreencastFrame();
|
|
1998
|
+
}
|
|
809
1999
|
});
|
|
810
2000
|
await Promise.all([page.send("Page.enable"), page.send("Runtime.enable")]);
|
|
2001
|
+
const tree = await page.send<{ frameTree?: { frame: { id: string } } }>("Page.getFrameTree");
|
|
2002
|
+
rootFrameId = tree.frameTree?.frame.id ?? rootFrameId;
|
|
811
2003
|
await this.navigationHistory(page);
|
|
812
|
-
// Emulation is scoped to the CDP session, so every fresh attach starts without it.
|
|
813
|
-
if (this.viewport) await this.applyEmulation(page, this.viewport);
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
private async applyEmulation(page: CdpSession, device: DeviceEmulation): Promise<void> {
|
|
817
|
-
await page.send(
|
|
818
|
-
"Emulation.setDeviceMetricsOverride",
|
|
819
|
-
{
|
|
820
|
-
width: device.width,
|
|
821
|
-
height: device.height,
|
|
822
|
-
deviceScaleFactor: device.deviceScaleFactor,
|
|
823
|
-
mobile: device.mobile,
|
|
824
|
-
screenWidth: device.screenWidth ?? device.width,
|
|
825
|
-
screenHeight: device.screenHeight ?? device.height,
|
|
826
|
-
screenOrientation: {
|
|
827
|
-
type: device.width > device.height ? "landscapePrimary" : "portraitPrimary",
|
|
828
|
-
angle: device.width > device.height ? 90 : 0,
|
|
829
|
-
},
|
|
830
|
-
},
|
|
831
|
-
{ mutation: true },
|
|
832
|
-
);
|
|
833
|
-
await page.send(
|
|
834
|
-
"Emulation.setTouchEmulationEnabled",
|
|
835
|
-
{ enabled: device.touch, maxTouchPoints: device.touch ? 5 : 1 },
|
|
836
|
-
{ mutation: true },
|
|
837
|
-
);
|
|
838
|
-
if (device.userAgent) {
|
|
839
|
-
await page.send(
|
|
840
|
-
"Emulation.setUserAgentOverride",
|
|
841
|
-
{ userAgent: device.userAgent, platform: device.platform ?? "" },
|
|
842
|
-
{ mutation: true },
|
|
843
|
-
);
|
|
844
|
-
}
|
|
845
2004
|
}
|
|
846
2005
|
|
|
847
2006
|
private async startScreencastSession(page: CdpSession): Promise<void> {
|
|
@@ -849,23 +2008,34 @@ export class AgentBrowserRuntime {
|
|
|
849
2008
|
let candidate = page;
|
|
850
2009
|
for (;;) {
|
|
851
2010
|
try {
|
|
2011
|
+
this.assertRuntimeOpen();
|
|
852
2012
|
await Promise.all([candidate.send("Page.enable"), candidate.send("Runtime.enable")]);
|
|
2013
|
+
this.assertRuntimeOpen();
|
|
853
2014
|
await this.navigationHistory(candidate);
|
|
2015
|
+
this.assertRuntimeOpen();
|
|
854
2016
|
if (this.page !== candidate) {
|
|
855
2017
|
candidate = await this.requirePage();
|
|
856
2018
|
continue;
|
|
857
2019
|
}
|
|
2020
|
+
await this.restoreConfiguredEmulation(candidate);
|
|
2021
|
+
this.assertRuntimeOpen();
|
|
2022
|
+
if (this.page !== candidate) {
|
|
2023
|
+
throw new CdpUnavailableError("Browser page changed before screencast start");
|
|
2024
|
+
}
|
|
858
2025
|
await candidate.send(
|
|
859
2026
|
"Page.startScreencast",
|
|
860
2027
|
{ format: "jpeg", quality: this.screencastQuality, everyNthFrame: 1 },
|
|
861
2028
|
{ mutation: true },
|
|
862
2029
|
);
|
|
2030
|
+
this.assertRuntimeOpen();
|
|
863
2031
|
return;
|
|
864
2032
|
} catch (error) {
|
|
2033
|
+
this.assertRuntimeOpen();
|
|
865
2034
|
if (Date.now() >= deadline) throw error;
|
|
866
2035
|
try {
|
|
867
2036
|
candidate = await this.reattachPageForScreencast(candidate);
|
|
868
2037
|
} catch (reattachError) {
|
|
2038
|
+
this.assertRuntimeOpen();
|
|
869
2039
|
if (Date.now() >= deadline) throw reattachError;
|
|
870
2040
|
}
|
|
871
2041
|
const { promise, resolve } = Promise.withResolvers<void>();
|
|
@@ -876,66 +2046,140 @@ export class AgentBrowserRuntime {
|
|
|
876
2046
|
}
|
|
877
2047
|
|
|
878
2048
|
private async reattachPageForScreencast(previous: CdpSession): Promise<CdpSession> {
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
await connection.send(
|
|
884
|
-
"Target.activateTarget",
|
|
885
|
-
{ targetId: target.targetId },
|
|
886
|
-
{ mutation: true },
|
|
887
|
-
);
|
|
888
|
-
const replacement = await attachToTarget(connection, target.targetId);
|
|
889
|
-
await this.preparePage(replacement);
|
|
890
|
-
if (this.page === previous) {
|
|
891
|
-
this.page = replacement;
|
|
892
|
-
this.targetId = target.targetId;
|
|
893
|
-
} else {
|
|
894
|
-
await replacement.detach().catch(() => undefined);
|
|
2049
|
+
this.assertRuntimeOpen();
|
|
2050
|
+
if (this.page !== previous) {
|
|
2051
|
+
if (!this.page) throw new CdpUnavailableError("Browser page changed during reattachment");
|
|
2052
|
+
return this.page;
|
|
895
2053
|
}
|
|
896
|
-
|
|
897
|
-
|
|
2054
|
+
return this.withVideoTransition(async () => {
|
|
2055
|
+
const attachmentGeneration = ++this.attachmentGeneration;
|
|
2056
|
+
const connection = this.requireConnection();
|
|
2057
|
+
const targets = await listPageTargets(connection);
|
|
2058
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
2059
|
+
const target = targets.find((candidate) => candidate.targetId === previous.targetId);
|
|
2060
|
+
if (!target)
|
|
2061
|
+
throw new CdpUnavailableError("The selected browser page is no longer available");
|
|
2062
|
+
await connection.send(
|
|
2063
|
+
"Target.activateTarget",
|
|
2064
|
+
{ targetId: target.targetId },
|
|
2065
|
+
{ mutation: true },
|
|
2066
|
+
);
|
|
2067
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
2068
|
+
const replacement = await attachToTarget(connection, target.targetId);
|
|
2069
|
+
if (this.page !== previous || attachmentGeneration !== this.attachmentGeneration) {
|
|
2070
|
+
await replacement.detach().catch(() => undefined);
|
|
2071
|
+
// The losing controller's detach may clear target overrides. Repair the
|
|
2072
|
+
// current controller on its next access rather than trust earlier pixels.
|
|
2073
|
+
this.emulationAppliedPage = null;
|
|
2074
|
+
this.invalidateScreencastFrame();
|
|
2075
|
+
if (!this.page) throw new CdpUnavailableError("Browser page changed during reattachment");
|
|
2076
|
+
return this.page;
|
|
2077
|
+
}
|
|
2078
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
2079
|
+
|
|
2080
|
+
this.invalidateScreencastFrame();
|
|
2081
|
+
// Detaching drops the session that holds any pressed button; release there first.
|
|
2082
|
+
if (this.liveInput) await this.endLiveInput(this.liveInput.id);
|
|
2083
|
+
await this.releaseHeldInput(previous);
|
|
2084
|
+
this.page = null;
|
|
2085
|
+
this.emulationAppliedPage = null;
|
|
2086
|
+
// Detach first: the former session can otherwise clear the newly applied override.
|
|
2087
|
+
try {
|
|
2088
|
+
await previous.detach();
|
|
2089
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
2090
|
+
await this.bindPageEvents(replacement);
|
|
2091
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
2092
|
+
await this.restoreConfiguredEmulation(replacement);
|
|
2093
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
2094
|
+
this.page = replacement;
|
|
2095
|
+
this.targetId = target.targetId;
|
|
2096
|
+
return replacement;
|
|
2097
|
+
} catch (error) {
|
|
2098
|
+
await replacement.detach().catch(() => undefined);
|
|
2099
|
+
// Detaching a partial emulation can clear target metrics even if a newer
|
|
2100
|
+
// attachment won. Its next access must restore intent before admitting pixels.
|
|
2101
|
+
this.emulationAppliedPage = null;
|
|
2102
|
+
this.invalidateScreencastFrame();
|
|
2103
|
+
throw error;
|
|
2104
|
+
}
|
|
2105
|
+
});
|
|
898
2106
|
}
|
|
899
2107
|
|
|
900
2108
|
private async withInvalidatedScreencast(
|
|
901
2109
|
mutation: (page: CdpSession) => Promise<unknown>,
|
|
2110
|
+
options: { preserveStream?: boolean } = {},
|
|
902
2111
|
): Promise<void> {
|
|
903
|
-
const
|
|
904
|
-
const restart = this.screencastActive;
|
|
905
|
-
this.invalidateScreencastFrame();
|
|
906
|
-
if (restart) await page.send("Page.stopScreencast", {}, { mutation: true });
|
|
2112
|
+
const releaseJpeg = this.jpegDemand.pin(false);
|
|
907
2113
|
try {
|
|
908
|
-
await
|
|
909
|
-
|
|
2114
|
+
const page = await this.requirePage();
|
|
2115
|
+
// Ordinary navigation is observed by the same native stream. Optional
|
|
2116
|
+
// capture teardown/setup must not consume the navigation ACK budget.
|
|
2117
|
+
const restart = this.screencastActive && !options.preserveStream;
|
|
910
2118
|
this.invalidateScreencastFrame();
|
|
911
|
-
if (restart
|
|
2119
|
+
if (restart) await page.send("Page.stopScreencast", {}, { mutation: true });
|
|
2120
|
+
this.assertRuntimeOpen();
|
|
2121
|
+
try {
|
|
2122
|
+
await mutation(page);
|
|
2123
|
+
this.invalidateQueuedVideoFrames();
|
|
2124
|
+
} finally {
|
|
2125
|
+
this.invalidateScreencastFrame();
|
|
2126
|
+
if (restart && this.page === page) {
|
|
2127
|
+
this.assertRuntimeOpen();
|
|
2128
|
+
await this.restoreConfiguredEmulation(page);
|
|
2129
|
+
await this.startScreencastSession(page);
|
|
2130
|
+
}
|
|
2131
|
+
}
|
|
2132
|
+
} finally {
|
|
2133
|
+
releaseJpeg();
|
|
912
2134
|
}
|
|
913
2135
|
}
|
|
914
2136
|
|
|
2137
|
+
/** Clear JPEG stream/screenshot receipts after input or transport changes. */
|
|
915
2138
|
private invalidateScreencastFrame(): void {
|
|
2139
|
+
this.capturePolicy.invalidate();
|
|
2140
|
+
this.screencastSourceTime.invalidate(Date.now());
|
|
916
2141
|
this.screencastFrame = null;
|
|
2142
|
+
this.screencastFrameReceivedAt = null;
|
|
917
2143
|
this.resolveFrameWaiters(null);
|
|
918
2144
|
}
|
|
919
2145
|
|
|
920
|
-
private
|
|
921
|
-
|
|
922
|
-
|
|
2146
|
+
private onScreencastFrame(page: CdpSession, event: ScreencastFrame): void {
|
|
2147
|
+
// A late frame belongs to its emitting session, even after page reattachment.
|
|
2148
|
+
void page
|
|
2149
|
+
.send("Page.screencastFrameAck", { sessionId: event.sessionId })
|
|
923
2150
|
.catch(() => undefined);
|
|
924
|
-
|
|
925
|
-
|
|
2151
|
+
if (
|
|
2152
|
+
this.stopping ||
|
|
2153
|
+
page !== this.page ||
|
|
2154
|
+
!this.screencastActive ||
|
|
2155
|
+
this.emulationAppliedPage !== page
|
|
2156
|
+
)
|
|
2157
|
+
return;
|
|
2158
|
+
const dimensions = readJpegFrameDimensions(event.data);
|
|
926
2159
|
const viewport = this.viewport;
|
|
927
|
-
|
|
2160
|
+
// Chromium can report the new device dimensions while emitting a clipped transition image.
|
|
2161
|
+
// Reject those pixels and use the bounded screenshot fallback rather than mislabeling them.
|
|
2162
|
+
if (!viewport || !dimensions) return;
|
|
2163
|
+
const expectedPixels = captureDimensions(viewport, viewport.captureScale);
|
|
2164
|
+
if (dimensions.width !== expectedPixels.width || dimensions.height !== expectedPixels.height)
|
|
2165
|
+
return;
|
|
2166
|
+
const now = performance.now();
|
|
2167
|
+
const source = this.screencastSourceTime.accept(event.metadata.timestamp, Date.now(), now);
|
|
2168
|
+
if (!source) return;
|
|
2169
|
+
const { width, height } = dimensions;
|
|
928
2170
|
const frame: RuntimeFrame = {
|
|
929
2171
|
dataBase64: event.data,
|
|
930
2172
|
byteLength: Buffer.byteLength(event.data, "base64"),
|
|
931
2173
|
width,
|
|
932
2174
|
height,
|
|
933
2175
|
transport: "cdp-screencast",
|
|
934
|
-
capturedAt:
|
|
2176
|
+
capturedAt: source.capturedAt,
|
|
935
2177
|
};
|
|
2178
|
+
this.capturePolicy.observeStream(`provider:${event.metadata.timestamp}`, now);
|
|
936
2179
|
this.screencastFrame = frame;
|
|
2180
|
+
this.screencastFrameReceivedAt = source.receivedAt;
|
|
937
2181
|
this.resolveFrameWaiters(frame);
|
|
938
|
-
}
|
|
2182
|
+
}
|
|
939
2183
|
|
|
940
2184
|
private waitForFrame(waitMs: number): Promise<RuntimeFrame | null> {
|
|
941
2185
|
const { promise, resolve } = Promise.withResolvers<RuntimeFrame | null>();
|
|
@@ -958,6 +2202,17 @@ export class AgentBrowserRuntime {
|
|
|
958
2202
|
}
|
|
959
2203
|
|
|
960
2204
|
private assertViewport(viewport: BrowserViewport): void {
|
|
2205
|
+
const captureScale = viewport.captureScale ?? 1;
|
|
2206
|
+
const pixels = captureDimensions(viewport, captureScale);
|
|
2207
|
+
if (
|
|
2208
|
+
!Number.isFinite(captureScale) ||
|
|
2209
|
+
captureScale < 1 ||
|
|
2210
|
+
captureScale > 2 ||
|
|
2211
|
+
pixels.width > MAX_VIEWPORT.width ||
|
|
2212
|
+
pixels.height > MAX_VIEWPORT.height
|
|
2213
|
+
) {
|
|
2214
|
+
throw new RangeError("Capture density exceeds the supported image bounds");
|
|
2215
|
+
}
|
|
961
2216
|
if (
|
|
962
2217
|
!Number.isInteger(viewport.width) ||
|
|
963
2218
|
viewport.width < 1 ||
|
|
@@ -986,6 +2241,35 @@ export class AgentBrowserRuntime {
|
|
|
986
2241
|
}
|
|
987
2242
|
}
|
|
988
2243
|
|
|
2244
|
+
/** CDP/Puppeteer priority for supported held buttons, independent of press order.
|
|
2245
|
+
* https://github.com/puppeteer/puppeteer/blob/main/packages/puppeteer-core/src/cdp/Input.ts
|
|
2246
|
+
*/
|
|
2247
|
+
private heldMoveButton(): MouseButton | "none" {
|
|
2248
|
+
for (const button of ["left", "right", "middle"] as const) {
|
|
2249
|
+
if (this.heldButtons.has(button)) {
|
|
2250
|
+
return button;
|
|
2251
|
+
}
|
|
2252
|
+
}
|
|
2253
|
+
return "none";
|
|
2254
|
+
}
|
|
2255
|
+
|
|
2256
|
+
/** Validate before recording held-button intent or sending a native packet. */
|
|
2257
|
+
private assertMouseModifiers(modifiers: number | undefined): void {
|
|
2258
|
+
if (
|
|
2259
|
+
modifiers !== undefined &&
|
|
2260
|
+
(!Number.isInteger(modifiers) || modifiers < 0 || modifiers > 15)
|
|
2261
|
+
) {
|
|
2262
|
+
throw new RangeError("Mouse modifiers must be an integer from 0 to 15");
|
|
2263
|
+
}
|
|
2264
|
+
}
|
|
2265
|
+
|
|
2266
|
+
/** Both live and discrete keyboard paths must bound release fanout before sending. */
|
|
2267
|
+
private assertHeldKeyCapacity(code: string): void {
|
|
2268
|
+
if (!this.heldKeys.has(code) && this.heldKeys.size >= MAX_HELD_BROWSER_KEYS) {
|
|
2269
|
+
throw new Error("Too many simultaneously held browser keys");
|
|
2270
|
+
}
|
|
2271
|
+
}
|
|
2272
|
+
|
|
989
2273
|
private buttonMask(add?: MouseButton, remove?: MouseButton): number {
|
|
990
2274
|
const active = new Set(this.heldButtons);
|
|
991
2275
|
if (add) active.add(add);
|