@omercnet/paseo-shared-browser 1.2.2 → 2.0.0-next.244.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/OVERVIEW.md +2 -2
- package/README.md +316 -27
- package/REMOTE_BROWSER_RESEARCH.md +395 -0
- package/STREAMING_DESIGN.md +196 -0
- package/client/browser-auto-capture-density.test.ts +102 -0
- package/client/browser-auto-capture-density.ts +30 -0
- package/client/browser-canvas-input.test.ts +198 -0
- package/client/browser-canvas-input.ts +202 -0
- package/client/browser-canvas-keyboard-web.test.ts +113 -0
- package/client/browser-canvas-keyboard-web.ts +230 -0
- package/client/browser-canvas-keyboard.test.ts +123 -0
- package/client/browser-canvas-keyboard.ts +126 -0
- package/client/browser-canvas-layout.test.ts +109 -0
- package/client/browser-canvas-layout.ts +51 -0
- package/client/browser-canvas-viewport-web.test.ts +77 -0
- package/client/browser-canvas-viewport-web.ts +63 -0
- package/client/browser-canvas-viewport.tsx +152 -0
- package/client/browser-capture-cadence.test.ts +9 -0
- package/client/browser-capture-cadence.ts +6 -0
- package/client/browser-capture-density-controls.tsx +75 -0
- package/client/browser-chrome.tsx +234 -0
- package/client/browser-client-presence.ts +184 -0
- package/client/browser-compose-controls.test.tsx +275 -0
- package/client/browser-compose-controls.tsx +174 -0
- package/client/browser-control-button.tsx +109 -0
- package/client/browser-control-dialog.tsx +71 -0
- package/client/browser-control-request.test.ts +27 -0
- package/client/browser-control-request.ts +57 -0
- package/client/browser-control-tooltip-web.ts +10 -0
- package/client/browser-dialog-body.tsx +22 -0
- package/client/browser-emulation-mode.test.ts +75 -0
- package/client/browser-emulation-mode.ts +98 -0
- package/client/browser-fill-viewport.test.ts +45 -0
- package/client/browser-fill-viewport.ts +28 -0
- package/client/browser-frame-buffer.test.ts +157 -0
- package/client/browser-frame-buffer.ts +148 -0
- package/client/browser-frame-image.tsx +62 -0
- package/client/browser-image-load.test.ts +23 -0
- package/client/browser-image-load.ts +26 -0
- package/client/browser-input-queue.test.ts +1254 -0
- package/client/browser-input-queue.ts +519 -0
- package/client/browser-menu-placement.test.ts +90 -0
- package/client/browser-menu-placement.ts +78 -0
- package/client/browser-panel-styles.ts +384 -0
- package/client/browser-quality-controls.test.ts +26 -0
- package/client/browser-quality-controls.tsx +139 -0
- package/client/browser-resolution-picker.tsx +189 -0
- package/client/browser-tabs.tsx +117 -0
- package/client/browser-toolbar-menu-web.test.ts +185 -0
- package/client/browser-toolbar-menu-web.ts +159 -0
- package/client/browser-toolbar-menu.tsx +255 -0
- package/client/browser-toolbar-submenu.tsx +93 -0
- package/client/browser-video-bytes.test.ts +44 -0
- package/client/browser-video-currency.ts +45 -0
- package/client/browser-video-decoder.test.ts +473 -0
- package/client/browser-video-decoder.ts +311 -0
- package/client/browser-video-surface.tsx +21 -0
- package/client/browser-video-visibility.test.ts +45 -0
- package/client/browser-viewer-recovery.test.ts +111 -0
- package/client/browser-viewer-recovery.ts +78 -0
- package/client/browser.tsx +1650 -1579
- package/client/favorite-star.tsx +23 -0
- package/client/frame-lifecycle.ts +62 -59
- package/client/sidebar.tsx +1 -2
- package/client/use-browser-auto-capture-density.test.ts +104 -0
- package/client/use-browser-auto-capture-density.ts +43 -0
- package/client/use-browser-canvas-input.ts +375 -0
- package/client/use-browser-capture-density.ts +87 -0
- package/client/use-browser-emulation-mode.ts +56 -0
- package/client/use-browser-fill-viewport.test.ts +108 -0
- package/client/use-browser-fill-viewport.ts +62 -0
- package/client/use-browser-frame-buffer.ts +66 -0
- package/client/use-browser-image-capture.ts +139 -0
- package/client/use-browser-scoped-mutation.test.ts +118 -0
- package/client/use-browser-scoped-mutation.ts +59 -0
- package/client/use-browser-video.test.ts +1060 -0
- package/client/use-browser-video.ts +481 -0
- package/client/use-browser-viewer-recovery.ts +26 -0
- package/client/use-resolution-favorites.ts +90 -0
- package/client/web.test.ts +125 -0
- package/client/web.ts +405 -0
- package/index.client.tsx +0 -10
- package/index.server.ts +32 -4
- package/npm-shrinkwrap.json +9 -9
- package/package.json +10 -5
- package/paseo-plugin.json +1 -1
- package/scripts/prepare-runtime.mjs +1 -1
- package/server/agent-browser-runtime.ts +1601 -314
- package/server/browser-cursor.test.ts +64 -0
- package/server/browser-cursor.ts +69 -0
- package/server/browser-frame-context.ts +22 -0
- package/server/browser-frame-epoch.test.ts +95 -0
- package/server/browser-frame-rejection.ts +17 -0
- package/server/browser-gesture.ts +120 -0
- package/server/browser-operation-diagnostics.test.ts +79 -0
- package/server/browser-operation-diagnostics.ts +103 -0
- package/server/browser-policy.ts +1316 -151
- package/server/browser-video-input-authority.test.ts +211 -0
- package/server/browser-video-policy.test.ts +564 -0
- package/server/browser-video-socket.test.ts +368 -0
- package/server/browser.ts +151 -16
- package/server/capture-transport-policy.test.ts +105 -0
- package/server/capture-transport-policy.ts +154 -0
- package/server/capture-transport-runtime.test.ts +213 -0
- package/server/daemon-launch-chain.test.ts +70 -0
- package/server/daemon-shutdown.test.ts +289 -0
- package/server/discrete-input-freshness.test.ts +364 -0
- package/server/discrete-input-native-lifetime.test.ts +150 -0
- package/server/display-loss-recovery.test.ts +294 -0
- package/server/emulation-restoration.test.ts +262 -0
- package/server/input-cleanup.ts +23 -0
- package/server/input-generation.test.ts +31 -0
- package/server/input-generation.ts +36 -0
- package/server/jpeg-capture-demand.test.ts +76 -0
- package/server/jpeg-capture-demand.ts +98 -0
- package/server/jpeg-frame.ts +29 -0
- package/server/keyboard-input.test.ts +61 -0
- package/server/keyboard-input.ts +104 -0
- package/server/live-input-policy.test.ts +958 -0
- package/server/live-input-runtime.test.ts +732 -0
- package/server/live-input-supervisor.test.ts +147 -0
- package/server/mcp-entry.ts +44 -2
- package/server/native-video-backpressure.test.ts +228 -0
- package/server/native-video-buffer.test.ts +141 -0
- package/server/native-video-buffer.ts +114 -0
- package/server/native-video-capture.test.ts +599 -0
- package/server/native-video-capture.ts +497 -0
- package/server/native-video-extension-source.test.ts +201 -0
- package/server/native-video-extension-source.ts +364 -0
- package/server/native-video-extension.test.ts +41 -0
- package/server/native-video-extension.ts +83 -0
- package/server/native-video-packet.ts +9 -0
- package/server/native-video-settings.test.ts +245 -0
- package/server/navigation-budget.ts +8 -0
- package/server/navigation-commit.ts +125 -0
- package/server/navigation-policy.test.ts +153 -0
- package/server/process-identity.ts +58 -0
- package/server/reattach-held-input.test.ts +143 -0
- package/server/runtime-frame-reuse.test.ts +295 -0
- package/server/runtime-jpeg-demand.test.ts +200 -0
- package/server/runtime-jpeg-shutdown.test.ts +210 -0
- package/server/runtime-jpeg-source-order.test.ts +201 -0
- package/server/runtime-launch-environment.test.ts +111 -0
- package/server/runtime-navigation.test.ts +327 -0
- package/server/runtime-owner-display.test.ts +152 -0
- package/server/runtime-owner.ts +340 -31
- package/server/runtime-page-state.test.ts +249 -0
- package/server/runtime-protocol.ts +31 -1
- package/server/runtime-target-retention.test.ts +181 -0
- package/server/runtime-video-opt-in.test.ts +82 -0
- package/server/runtime-video.test.ts +444 -0
- package/server/screencast-source-time.ts +46 -0
- package/server/stale-wheel-admission.test.ts +136 -0
- package/server/supervisor-client.ts +28 -4
- package/server/supervisor-heartbeat.test.ts +80 -0
- package/server/supervisor.ts +553 -55
- package/server/tab-close-confirmation.test.ts +97 -0
- package/server/video-source-recovery.test.ts +20 -0
- package/server/video-source-recovery.ts +64 -0
- package/server/viewer-capture-lifetime.ts +51 -0
- package/server/virtual-display.test.ts +184 -0
- package/server/virtual-display.ts +236 -0
- package/shared/browser-display-preferences.ts +46 -0
- package/shared/browser-video.ts +83 -0
- package/shared/browser.ts +286 -65
- package/shared/capture-density.ts +16 -0
- package/shared/capture-settings.ts +17 -0
- package/shared/device-presets.ts +90 -0
- package/shared/resolution-menu.test.ts +120 -0
- package/shared/resolution-menu.ts +72 -0
- package/shared/video-settings.test.ts +38 -0
- package/shared/video-settings.ts +34 -0
- package/shared/viewport-limits.ts +5 -0
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
2
3
|
import { chmod, mkdir, readFile, rm } from "node:fs/promises";
|
|
3
4
|
import { isAbsolute, resolve } from "node:path";
|
|
4
5
|
import { promisify } from "node:util";
|
|
6
|
+
import { canUseCaptureDensity, captureDensitySchema } from "../shared/capture-density";
|
|
7
|
+
|
|
8
|
+
import { captureDimensions, DEFAULT_JPEG_QUALITY } from "../shared/capture-settings";
|
|
9
|
+
import { MAX_VIEWPORT } from "../shared/viewport-limits";
|
|
5
10
|
import {
|
|
6
11
|
attachToTarget,
|
|
7
12
|
CdpConnection,
|
|
@@ -12,13 +17,56 @@ import {
|
|
|
12
17
|
CdpUnknownOutcomeError,
|
|
13
18
|
listPageTargets,
|
|
14
19
|
} from "./cdp";
|
|
20
|
+
import { readJpegFrameDimensions } from "./jpeg-frame";
|
|
21
|
+
|
|
22
|
+
export { readJpegFrameDimensions as jpegDimensions } from "./jpeg-frame";
|
|
23
|
+
|
|
24
|
+
import {
|
|
25
|
+
type BrowserCursor,
|
|
26
|
+
type BrowserGestureKeyEvent,
|
|
27
|
+
browserCursorSchema,
|
|
28
|
+
browserGestureKeySchema,
|
|
29
|
+
MAX_HELD_BROWSER_KEYS,
|
|
30
|
+
} from "../shared/browser";
|
|
31
|
+
import { resolveVideoEncoderSettings } from "../shared/video-settings";
|
|
32
|
+
import { browserCursorExpression } from "./browser-cursor";
|
|
33
|
+
import { GESTURE_IDLE_MS, GESTURE_LIFETIME_MS } from "./browser-gesture";
|
|
34
|
+
import { CAPTURE_MAX_AGE_MS, createCaptureTransportPolicy } from "./capture-transport-policy";
|
|
35
|
+
import { formatRuntimeInputGeneration } from "./input-generation";
|
|
36
|
+
import { createJpegCaptureDemand } from "./jpeg-capture-demand";
|
|
37
|
+
import { heldKeyModifiers, nativeKeyEvent } from "./keyboard-input";
|
|
38
|
+
import {
|
|
39
|
+
NativeVideoCapture,
|
|
40
|
+
type NativeVideoQuality,
|
|
41
|
+
type NativeVideoRead,
|
|
42
|
+
} from "./native-video-capture";
|
|
43
|
+
import { NATIVE_VIDEO_EXTENSION_ID, prepareNativeVideoExtension } from "./native-video-extension";
|
|
44
|
+
import {
|
|
45
|
+
NAVIGATION_ACK_TIMEOUT_MS,
|
|
46
|
+
NAVIGATION_COMMIT_TIMEOUT_MS,
|
|
47
|
+
NAVIGATION_METADATA_TIMEOUT_MS,
|
|
48
|
+
} from "./navigation-budget";
|
|
49
|
+
import { waitForNavigationCommit } from "./navigation-commit";
|
|
50
|
+
import {
|
|
51
|
+
type ProcessIdentity,
|
|
52
|
+
probeProcess,
|
|
53
|
+
processExists,
|
|
54
|
+
RUNTIME_OWNER_VARIABLE,
|
|
55
|
+
} from "./process-identity";
|
|
56
|
+
import { createScreencastSourceTime } from "./screencast-source-time";
|
|
57
|
+
import { createVideoSourceRecovery } from "./video-source-recovery";
|
|
15
58
|
|
|
16
59
|
const execFileAsync = promisify(execFile);
|
|
17
|
-
export const AGENT_BROWSER_VERSION = "0.
|
|
60
|
+
export const AGENT_BROWSER_VERSION = "0.38.2";
|
|
18
61
|
const PRIVATE_DIRECTORY_MODE = 0o700;
|
|
19
62
|
const PRIVATE_FILE_MODE = 0o600;
|
|
20
63
|
const DEFAULT_TIMEOUT_MS = 15_000;
|
|
21
64
|
const SCREENCAST_SETUP_RETRY_MS = 25;
|
|
65
|
+
/** Bound for the agent-browser session daemon to exit after close or SIGKILL. */
|
|
66
|
+
const DAEMON_EXIT_WAIT_MS = 5_000;
|
|
67
|
+
// Cursor decoration must not hold the serialized input/video fence while page
|
|
68
|
+
// scripts are busy. A missed sample leaves input and its cleanup unchanged.
|
|
69
|
+
const CURSOR_SAMPLE_TIMEOUT_MS = 250;
|
|
22
70
|
|
|
23
71
|
export class AgentBrowserUnavailableError extends Error {
|
|
24
72
|
override readonly name = "AgentBrowserUnavailableError";
|
|
@@ -34,6 +82,8 @@ export interface BrowserViewport {
|
|
|
34
82
|
width: number;
|
|
35
83
|
height: number;
|
|
36
84
|
deviceScaleFactor: number;
|
|
85
|
+
/** Capture density does not change the emulated layout or pointer coordinate system. */
|
|
86
|
+
captureScale?: number;
|
|
37
87
|
mobile: boolean;
|
|
38
88
|
touch: boolean;
|
|
39
89
|
userAgent?: string;
|
|
@@ -68,6 +118,8 @@ export interface RuntimePageState {
|
|
|
68
118
|
title: string;
|
|
69
119
|
canGoBack: boolean;
|
|
70
120
|
canGoForward: boolean;
|
|
121
|
+
/** Process-local attachment/document revision, including same-URL reloads. */
|
|
122
|
+
inputGeneration: string;
|
|
71
123
|
}
|
|
72
124
|
|
|
73
125
|
export interface RuntimeFrame {
|
|
@@ -94,7 +146,13 @@ export interface AgentBrowserRuntimeOptions {
|
|
|
94
146
|
session: string;
|
|
95
147
|
initialUrl?: string;
|
|
96
148
|
headed?: boolean;
|
|
149
|
+
/** Trusted per-instance private X11 context, only DISPLAY/XAUTHORITY; never mutates host environment. */
|
|
150
|
+
launchEnvironment?: Readonly<Record<string, string>>;
|
|
151
|
+
/** Trusted daemon-side opt-in. Only then is the capture extension materialized and loaded. */
|
|
152
|
+
nativeVideo?: boolean;
|
|
97
153
|
timeoutMs?: number;
|
|
154
|
+
/** Bound for the session daemon to exit after close or SIGKILL. */
|
|
155
|
+
daemonExitWaitMs?: number;
|
|
98
156
|
}
|
|
99
157
|
|
|
100
158
|
interface VersionResult {
|
|
@@ -108,14 +166,10 @@ interface NavigationHistory {
|
|
|
108
166
|
currentIndex: number;
|
|
109
167
|
entries: Array<{ id: number; url: string; title: string }>;
|
|
110
168
|
}
|
|
111
|
-
interface PageLifecycleEvent {
|
|
112
|
-
name: string;
|
|
113
|
-
loaderId: string;
|
|
114
|
-
}
|
|
115
169
|
|
|
116
170
|
interface ScreencastFrame {
|
|
117
171
|
data: string;
|
|
118
|
-
metadata: { deviceWidth: number; deviceHeight: number };
|
|
172
|
+
metadata: { deviceWidth: number; deviceHeight: number; timestamp?: number };
|
|
119
173
|
sessionId: number;
|
|
120
174
|
}
|
|
121
175
|
|
|
@@ -124,7 +178,11 @@ function requireAbsolute(path: string, label: string): string {
|
|
|
124
178
|
return resolve(path);
|
|
125
179
|
}
|
|
126
180
|
|
|
127
|
-
function runtimeEnvironment(
|
|
181
|
+
function runtimeEnvironment(
|
|
182
|
+
ipcDirectory: string,
|
|
183
|
+
launchEnvironment: Readonly<Record<string, string>> | undefined,
|
|
184
|
+
ownerNonce: string,
|
|
185
|
+
): NodeJS.ProcessEnv {
|
|
128
186
|
const environment: NodeJS.ProcessEnv = {};
|
|
129
187
|
for (const key of [
|
|
130
188
|
"PATH",
|
|
@@ -153,6 +211,27 @@ function runtimeEnvironment(ipcDirectory: string): NodeJS.ProcessEnv {
|
|
|
153
211
|
const value = process.env[key];
|
|
154
212
|
if (value !== undefined) environment[key] = value;
|
|
155
213
|
}
|
|
214
|
+
if (launchEnvironment !== undefined) {
|
|
215
|
+
const keys = Object.keys(launchEnvironment);
|
|
216
|
+
const display = launchEnvironment.DISPLAY;
|
|
217
|
+
const authority = launchEnvironment.XAUTHORITY;
|
|
218
|
+
if (
|
|
219
|
+
keys.length !== 2 ||
|
|
220
|
+
keys.some((key) => key !== "DISPLAY" && key !== "XAUTHORITY") ||
|
|
221
|
+
typeof display !== "string" ||
|
|
222
|
+
!/^:[0-9]{1,5}(?:\.[0-9]{1,2})?$/.test(display) ||
|
|
223
|
+
typeof authority !== "string" ||
|
|
224
|
+
!isAbsolute(authority)
|
|
225
|
+
) {
|
|
226
|
+
// Do not include display/authentication values in errors or child arguments.
|
|
227
|
+
throw new AgentBrowserIncompatibleError("Private browser display environment is invalid");
|
|
228
|
+
}
|
|
229
|
+
environment.DISPLAY = display;
|
|
230
|
+
environment.XAUTHORITY = authority;
|
|
231
|
+
// A private X11 display must not inherit an unrelated host Wayland session.
|
|
232
|
+
delete environment.WAYLAND_DISPLAY;
|
|
233
|
+
}
|
|
234
|
+
environment[RUNTIME_OWNER_VARIABLE] = ownerNonce;
|
|
156
235
|
environment.AGENT_BROWSER_SOCKET_DIR = ipcDirectory;
|
|
157
236
|
environment.AGENT_BROWSER_IDLE_TIMEOUT_MS = "0";
|
|
158
237
|
environment.AGENT_BROWSER_STREAM_PORT = "0";
|
|
@@ -183,26 +262,6 @@ function findString(value: unknown, keys: readonly string[]): string | null {
|
|
|
183
262
|
return null;
|
|
184
263
|
}
|
|
185
264
|
|
|
186
|
-
export function jpegDimensions(base64: string): { width: number; height: number } | null {
|
|
187
|
-
const bytes = Buffer.from(base64, "base64");
|
|
188
|
-
let offset = 2;
|
|
189
|
-
while (bytes[0] === 0xff && bytes[1] === 0xd8 && offset + 9 <= bytes.length) {
|
|
190
|
-
if (bytes[offset] !== 0xff) return null;
|
|
191
|
-
const marker = bytes[offset + 1];
|
|
192
|
-
// SOFn carries the frame size; DHT (c4), JPG (c8) and DAC (cc) share the range but do not.
|
|
193
|
-
if (
|
|
194
|
-
marker !== undefined &&
|
|
195
|
-
marker >= 0xc0 &&
|
|
196
|
-
marker <= 0xcf &&
|
|
197
|
-
![0xc4, 0xc8, 0xcc].includes(marker)
|
|
198
|
-
)
|
|
199
|
-
return { width: bytes.readUInt16BE(offset + 7), height: bytes.readUInt16BE(offset + 5) };
|
|
200
|
-
if (marker === 0xda) return null;
|
|
201
|
-
offset += 2 + bytes.readUInt16BE(offset + 2);
|
|
202
|
-
}
|
|
203
|
-
return null;
|
|
204
|
-
}
|
|
205
|
-
|
|
206
265
|
export class AgentBrowserRuntime {
|
|
207
266
|
readonly binaryPath: string;
|
|
208
267
|
readonly executablePath: string;
|
|
@@ -213,17 +272,58 @@ export class AgentBrowserRuntime {
|
|
|
213
272
|
private readonly headed: boolean;
|
|
214
273
|
private readonly timeoutMs: number;
|
|
215
274
|
private readonly environment: NodeJS.ProcessEnv;
|
|
275
|
+
private daemon: { pid: number; identity: ProcessIdentity | null } | null = null;
|
|
276
|
+
private readonly ownerNonce = randomUUID();
|
|
277
|
+
private readonly daemonExitWaitMs: number;
|
|
216
278
|
private connection: CdpConnection | null = null;
|
|
217
279
|
private page: CdpSession | null = null;
|
|
218
280
|
private targetId: string | null = null;
|
|
219
|
-
private viewport:
|
|
281
|
+
private viewport: DeviceEmulation | null = null;
|
|
282
|
+
private emulationAppliedPage: CdpSession | null = null;
|
|
283
|
+
private attachmentGeneration = 0;
|
|
284
|
+
private readonly capturePolicy = createCaptureTransportPolicy();
|
|
285
|
+
private readonly screencastSourceTime = createScreencastSourceTime();
|
|
220
286
|
private screencastFrame: RuntimeFrame | null = null;
|
|
287
|
+
private screencastFrameReceivedAt: number | null = null;
|
|
221
288
|
private screencastWaiters = new Set<(frame: RuntimeFrame | null) => void>();
|
|
222
289
|
private screencastActive = false;
|
|
223
|
-
private screencastQuality =
|
|
290
|
+
private screencastQuality: number = DEFAULT_JPEG_QUALITY;
|
|
291
|
+
private screencastStartFailed = false;
|
|
292
|
+
private readonly jpegDemand = createJpegCaptureDemand({
|
|
293
|
+
now: () => performance.now(),
|
|
294
|
+
retire: () => this.stopScreencastSession(),
|
|
295
|
+
onError: () => console.warn("Shared Browser could not retire unused JPEG capture"),
|
|
296
|
+
});
|
|
224
297
|
private heldButtons = new Set<MouseButton>();
|
|
225
|
-
|
|
298
|
+
/** Last native pointer position sent, so cancelled drags release where they were. */
|
|
299
|
+
private lastMousePoint = { x: 0, y: 0 };
|
|
300
|
+
private heldKeys = new Map<string, { key: string; code: string }>();
|
|
301
|
+
private heldTouches = false;
|
|
302
|
+
/** Attachment on which the current held state was published; cleanup never targets another. */
|
|
303
|
+
private heldOwner: CdpSession | null = null;
|
|
304
|
+
private activeTouches = new Map<number, { x: number; y: number; id: number }>();
|
|
305
|
+
private documentGeneration = 0;
|
|
306
|
+
private liveInput: {
|
|
307
|
+
id: string;
|
|
308
|
+
page: CdpSession;
|
|
309
|
+
generation: number;
|
|
310
|
+
documentGeneration: number;
|
|
311
|
+
idleUntil: number;
|
|
312
|
+
expiresAt: number;
|
|
313
|
+
timer: ReturnType<typeof setTimeout> | null;
|
|
314
|
+
} | null = null;
|
|
226
315
|
private stopping = false;
|
|
316
|
+
private shutdownInFlight: Promise<void> | null = null;
|
|
317
|
+
private shutDown = false;
|
|
318
|
+
/** A daemon was launched for this session, so absence of evidence is not proof of exit. */
|
|
319
|
+
private daemonLaunched = false;
|
|
320
|
+
private readonly nativeVideo: boolean;
|
|
321
|
+
private video: NativeVideoCapture | null = null;
|
|
322
|
+
private videoRetirement: Promise<void> | null = null;
|
|
323
|
+
private ownsBrowser = true;
|
|
324
|
+
private readonly videoRecovery = createVideoSourceRecovery();
|
|
325
|
+
private videoTransitionDepth = 0;
|
|
326
|
+
private videoTransitionEpoch = 0;
|
|
227
327
|
|
|
228
328
|
constructor(options: AgentBrowserRuntimeOptions) {
|
|
229
329
|
this.binaryPath = requireAbsolute(options.binaryPath, "agent-browser binary path");
|
|
@@ -236,22 +336,49 @@ export class AgentBrowserRuntime {
|
|
|
236
336
|
this.session = options.session;
|
|
237
337
|
this.headed = options.headed ?? false;
|
|
238
338
|
this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
239
|
-
this.
|
|
339
|
+
this.daemonExitWaitMs = options.daemonExitWaitMs ?? DAEMON_EXIT_WAIT_MS;
|
|
340
|
+
this.environment = runtimeEnvironment(
|
|
341
|
+
this.ipcDirectory,
|
|
342
|
+
options.launchEnvironment,
|
|
343
|
+
this.ownerNonce,
|
|
344
|
+
);
|
|
240
345
|
this.initialUrl = options.initialUrl ?? "about:blank";
|
|
346
|
+
this.nativeVideo = options.nativeVideo === true;
|
|
241
347
|
}
|
|
242
348
|
|
|
243
349
|
async launch(): Promise<void> {
|
|
350
|
+
this.assertRuntimeOpen();
|
|
244
351
|
if (this.connection?.isOpen) return;
|
|
245
352
|
this.invalidateScreencastFrame();
|
|
246
|
-
this.stopping = false;
|
|
247
353
|
await mkdir(this.profilePath, { recursive: true, mode: PRIVATE_DIRECTORY_MODE });
|
|
248
354
|
await mkdir(this.ipcDirectory, { recursive: true, mode: PRIVATE_DIRECTORY_MODE });
|
|
249
355
|
await Promise.all([
|
|
250
356
|
chmod(this.profilePath, PRIVATE_DIRECTORY_MODE),
|
|
251
357
|
chmod(this.ipcDirectory, PRIVATE_DIRECTORY_MODE),
|
|
252
358
|
]);
|
|
359
|
+
this.assertRuntimeOpen();
|
|
253
360
|
await this.assertVersion();
|
|
254
|
-
|
|
361
|
+
this.assertRuntimeOpen();
|
|
362
|
+
// Image-only sessions never materialize, load or allowlist the capture
|
|
363
|
+
// extension, so its tabCapture/debugger permissions do not exist in them.
|
|
364
|
+
const videoExtension = this.nativeVideo
|
|
365
|
+
? await prepareNativeVideoExtension(this.ipcDirectory)
|
|
366
|
+
: null;
|
|
367
|
+
this.assertRuntimeOpen();
|
|
368
|
+
const existingArguments = process.env.PASEO_SHARED_BROWSER_CHROMIUM_ARGS;
|
|
369
|
+
const chromiumArguments = [
|
|
370
|
+
existingArguments,
|
|
371
|
+
// agent-browser creates the launch page itself. Chromium's default
|
|
372
|
+
// startup window otherwise leaves an additional chrome://newtab page.
|
|
373
|
+
"--no-startup-window",
|
|
374
|
+
videoExtension ? `--allowlisted-extension-id=${NATIVE_VIDEO_EXTENSION_ID}` : undefined,
|
|
375
|
+
]
|
|
376
|
+
.filter(Boolean)
|
|
377
|
+
.join(",");
|
|
378
|
+
this.ownsBrowser = true;
|
|
379
|
+
// From here a nonce-owned daemon may exist even if open rejects (e.g. navigation failure).
|
|
380
|
+
this.daemonLaunched = true;
|
|
381
|
+
this.daemon = null;
|
|
255
382
|
const opened = await this.invoke([
|
|
256
383
|
"--session",
|
|
257
384
|
this.session,
|
|
@@ -259,25 +386,63 @@ export class AgentBrowserRuntime {
|
|
|
259
386
|
this.profilePath,
|
|
260
387
|
"--executable-path",
|
|
261
388
|
this.executablePath,
|
|
389
|
+
...(videoExtension ? ["--extension", videoExtension] : []),
|
|
390
|
+
"--hide-scrollbars",
|
|
391
|
+
"false",
|
|
262
392
|
...(this.headed ? ["--headed"] : []),
|
|
263
393
|
...(chromiumArguments ? ["--args", chromiumArguments] : []),
|
|
264
394
|
"--json",
|
|
265
395
|
"open",
|
|
266
396
|
this.initialUrl,
|
|
267
397
|
]);
|
|
398
|
+
this.assertRuntimeOpen();
|
|
268
399
|
const targetId = findString(opened, ["targetId", "target_id"]);
|
|
269
400
|
await this.protectIpcMetadata();
|
|
401
|
+
await this.captureDaemon();
|
|
402
|
+
this.assertRuntimeOpen();
|
|
270
403
|
await this.connectCdp(targetId);
|
|
271
404
|
}
|
|
272
405
|
|
|
406
|
+
/** Attach an independent page controller to an existing browser session. */
|
|
407
|
+
async attachTarget(targetId: string): Promise<void> {
|
|
408
|
+
this.assertRuntimeOpen();
|
|
409
|
+
this.ownsBrowser = false;
|
|
410
|
+
await this.connectCdp(targetId, false);
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/** Create a page in the existing profile without changing another controller's selected page. */
|
|
414
|
+
async createTarget(url = "about:blank"): Promise<string> {
|
|
415
|
+
const result = await this.requireConnection().send<{ targetId: string }>(
|
|
416
|
+
"Target.createTarget",
|
|
417
|
+
{ url },
|
|
418
|
+
{ mutation: true },
|
|
419
|
+
);
|
|
420
|
+
return result.targetId;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/** Close only the named page. The caller must keep another page in the browser. */
|
|
424
|
+
async closeTarget(targetId: string): Promise<void> {
|
|
425
|
+
const result = await this.requireConnection().send<{ success: boolean }>(
|
|
426
|
+
"Target.closeTarget",
|
|
427
|
+
{ targetId },
|
|
428
|
+
{ mutation: true },
|
|
429
|
+
);
|
|
430
|
+
if (!result.success) throw new CdpUnavailableError("Browser tab could not be closed");
|
|
431
|
+
}
|
|
432
|
+
|
|
273
433
|
async reconnect(): Promise<void> {
|
|
274
|
-
|
|
275
|
-
this.
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
434
|
+
this.assertRuntimeOpen();
|
|
435
|
+
await this.withVideoTransition(async () => {
|
|
436
|
+
const targetId = this.targetId;
|
|
437
|
+
this.attachmentGeneration += 1;
|
|
438
|
+
this.invalidateScreencastFrame();
|
|
439
|
+
this.connection?.close();
|
|
440
|
+
this.connection = null;
|
|
441
|
+
this.page = null;
|
|
442
|
+
await this.assertVersion();
|
|
443
|
+
this.assertRuntimeOpen();
|
|
444
|
+
await this.connectCdp(targetId);
|
|
445
|
+
});
|
|
281
446
|
}
|
|
282
447
|
|
|
283
448
|
async health(): Promise<RuntimeHealth> {
|
|
@@ -321,32 +486,73 @@ export class AgentBrowserRuntime {
|
|
|
321
486
|
}
|
|
322
487
|
|
|
323
488
|
async targets(): Promise<RuntimeTarget[]> {
|
|
324
|
-
return listPageTargets(this.requireConnection())
|
|
489
|
+
return (await listPageTargets(this.requireConnection())).filter(
|
|
490
|
+
(target) => !target.url.startsWith(`chrome-extension://${NATIVE_VIDEO_EXTENSION_ID}/`),
|
|
491
|
+
);
|
|
325
492
|
}
|
|
326
493
|
|
|
327
|
-
async selectTarget(targetId: string): Promise<void> {
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
494
|
+
async selectTarget(targetId: string, activate = true): Promise<void> {
|
|
495
|
+
await this.withVideoTransition(async () => {
|
|
496
|
+
const attachmentGeneration = ++this.attachmentGeneration;
|
|
497
|
+
const connection = this.requireConnection();
|
|
498
|
+
const target = (await this.targets()).find((candidate) => candidate.targetId === targetId);
|
|
499
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
500
|
+
if (!target) throw new CdpUnavailableError(`Unknown page target: ${targetId}`);
|
|
501
|
+
this.invalidateScreencastFrame();
|
|
502
|
+
const previous = this.page;
|
|
503
|
+
if (previous) {
|
|
504
|
+
if (this.screencastActive) {
|
|
505
|
+
await previous.send("Page.stopScreencast", {}, { mutation: true });
|
|
506
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
507
|
+
}
|
|
508
|
+
await previous.detach();
|
|
509
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
334
510
|
}
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
511
|
+
this.invalidateScreencastFrame();
|
|
512
|
+
if (activate) {
|
|
513
|
+
await connection.send("Target.activateTarget", { targetId }, { mutation: true });
|
|
514
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
515
|
+
}
|
|
516
|
+
// CDP emulation is session-owned: detaching the former session restores native
|
|
517
|
+
// window metrics. Publish a replacement only after its retained settings succeed.
|
|
518
|
+
this.page = null;
|
|
519
|
+
this.emulationAppliedPage = null;
|
|
520
|
+
const replacement = await attachToTarget(connection, targetId);
|
|
521
|
+
try {
|
|
522
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
523
|
+
await this.bindPageEvents(replacement);
|
|
524
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
525
|
+
await this.restoreConfiguredEmulation(replacement);
|
|
526
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
527
|
+
this.page = replacement;
|
|
528
|
+
this.targetId = targetId;
|
|
529
|
+
} catch (error) {
|
|
530
|
+
await replacement.detach().catch(() => undefined);
|
|
531
|
+
// Detaching a partial emulation can clear target metrics even if a newer
|
|
532
|
+
// attachment won. Its next access must restore intent before admitting pixels.
|
|
533
|
+
this.emulationAppliedPage = null;
|
|
534
|
+
this.invalidateScreencastFrame();
|
|
535
|
+
throw error;
|
|
536
|
+
}
|
|
537
|
+
if (this.screencastActive) await this.startScreencastSession(replacement);
|
|
538
|
+
});
|
|
343
539
|
}
|
|
344
540
|
|
|
345
|
-
private async navigationHistory(
|
|
346
|
-
|
|
541
|
+
private async navigationHistory(
|
|
542
|
+
page: CdpSession,
|
|
543
|
+
timeoutMs = this.timeoutMs,
|
|
544
|
+
): Promise<NavigationHistory> {
|
|
545
|
+
const deadline = Date.now() + timeoutMs;
|
|
347
546
|
for (;;) {
|
|
547
|
+
const remaining = deadline - Date.now();
|
|
548
|
+
if (remaining <= 0)
|
|
549
|
+
throw new CdpUnavailableError("Browser navigation history observation timed out");
|
|
348
550
|
try {
|
|
349
|
-
return await page.send<NavigationHistory>(
|
|
551
|
+
return await page.send<NavigationHistory>(
|
|
552
|
+
"Page.getNavigationHistory",
|
|
553
|
+
{},
|
|
554
|
+
{ timeoutMs: remaining },
|
|
555
|
+
);
|
|
350
556
|
} catch (error) {
|
|
351
557
|
if (!String(error).includes("Not attached to an active page") || Date.now() >= deadline) {
|
|
352
558
|
throw error;
|
|
@@ -357,113 +563,524 @@ export class AgentBrowserRuntime {
|
|
|
357
563
|
}
|
|
358
564
|
}
|
|
359
565
|
}
|
|
360
|
-
|
|
566
|
+
/** Read fresh metadata concurrently, discarding both replies if the document changes.
|
|
567
|
+
* One same-attachment reread shares the original monotonic timeout budget in
|
|
568
|
+
* milliseconds. Failed reads, replacement attachments and repeated document
|
|
569
|
+
* changes reject; this read-only recovery never repeats a native mutation. */
|
|
570
|
+
async state(options: { timeoutMs?: number } = {}): Promise<RuntimePageState> {
|
|
571
|
+
if (
|
|
572
|
+
options.timeoutMs !== undefined &&
|
|
573
|
+
(!Number.isInteger(options.timeoutMs) ||
|
|
574
|
+
options.timeoutMs < 1 ||
|
|
575
|
+
options.timeoutMs > DEFAULT_TIMEOUT_MS)
|
|
576
|
+
) {
|
|
577
|
+
throw new RangeError("Browser metadata timeout must be an integer from 1 to 15000ms");
|
|
578
|
+
}
|
|
579
|
+
const timeoutMs = Math.min(options.timeoutMs ?? this.timeoutMs, this.timeoutMs);
|
|
361
580
|
const page = await this.requirePage();
|
|
362
|
-
const
|
|
363
|
-
const
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
581
|
+
const attachmentGeneration = this.attachmentGeneration;
|
|
582
|
+
const deadline = performance.now() + timeoutMs;
|
|
583
|
+
|
|
584
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
585
|
+
this.assertRuntimeOpen();
|
|
586
|
+
const remaining = attempt === 0 ? timeoutMs : deadline - performance.now();
|
|
587
|
+
if (remaining <= 0) {
|
|
588
|
+
throw new CdpUnavailableError("Browser metadata observation timed out");
|
|
589
|
+
}
|
|
590
|
+
const documentGeneration = this.documentGeneration;
|
|
591
|
+
// Both observations must describe the same document. A navigation discards
|
|
592
|
+
// the entire pair; neither native history nor DOM metadata is cached.
|
|
593
|
+
const [history, evaluated] = await Promise.all([
|
|
594
|
+
this.navigationHistory(page, remaining),
|
|
595
|
+
page.send<{
|
|
596
|
+
result: { value?: { url?: string; title?: string } };
|
|
597
|
+
}>(
|
|
598
|
+
"Runtime.evaluate",
|
|
599
|
+
{
|
|
600
|
+
expression: "({url: location.href, title: document.title})",
|
|
601
|
+
returnByValue: true,
|
|
602
|
+
},
|
|
603
|
+
{ timeoutMs: remaining },
|
|
604
|
+
),
|
|
605
|
+
]);
|
|
606
|
+
this.assertRuntimeOpen();
|
|
607
|
+
if (page !== this.page || attachmentGeneration !== this.attachmentGeneration) {
|
|
608
|
+
throw new CdpUnavailableError("Browser metadata changed while being read");
|
|
609
|
+
}
|
|
610
|
+
// Successful replies can beat an overdue CDP timer in the event queue.
|
|
611
|
+
// Their metadata still cannot outlive the original shared phase budget.
|
|
612
|
+
if (performance.now() >= deadline) {
|
|
613
|
+
throw new CdpUnavailableError("Browser metadata observation timed out");
|
|
614
|
+
}
|
|
615
|
+
if (documentGeneration !== this.documentGeneration) {
|
|
616
|
+
if (attempt === 0) continue;
|
|
617
|
+
throw new CdpUnavailableError("Browser metadata changed while being read");
|
|
618
|
+
}
|
|
619
|
+
return {
|
|
620
|
+
url: evaluated.result.value?.url ?? history.entries[history.currentIndex]?.url ?? "",
|
|
621
|
+
title: evaluated.result.value?.title ?? history.entries[history.currentIndex]?.title ?? "",
|
|
622
|
+
canGoBack: history.currentIndex > 0,
|
|
623
|
+
canGoForward: history.currentIndex < history.entries.length - 1,
|
|
624
|
+
inputGeneration: formatRuntimeInputGeneration(attachmentGeneration, documentGeneration),
|
|
625
|
+
};
|
|
626
|
+
}
|
|
627
|
+
throw new CdpUnavailableError("Browser metadata changed while being read");
|
|
375
628
|
}
|
|
376
629
|
|
|
630
|
+
/** Await native ACK and its main-frame commit, without waiting for scripts or
|
|
631
|
+
* DOMContentLoaded. A missing commit remains uncertain and is never replayed. */
|
|
377
632
|
async navigate(url: string): Promise<void> {
|
|
378
|
-
await this.withInvalidatedScreencast(
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
),
|
|
409
|
-
),
|
|
410
|
-
30_000,
|
|
633
|
+
await this.withInvalidatedScreencast(
|
|
634
|
+
async (page) => {
|
|
635
|
+
const assertCurrent = this.navigationAttachmentGuard(page);
|
|
636
|
+
await waitForNavigationCommit(
|
|
637
|
+
page,
|
|
638
|
+
async () => {
|
|
639
|
+
const result = await page.send<{
|
|
640
|
+
frameId: string;
|
|
641
|
+
errorText?: string;
|
|
642
|
+
loaderId?: string;
|
|
643
|
+
isDownload?: boolean;
|
|
644
|
+
}>(
|
|
645
|
+
"Page.navigate",
|
|
646
|
+
{ url },
|
|
647
|
+
{ mutation: true, timeoutMs: Math.min(this.timeoutMs, NAVIGATION_ACK_TIMEOUT_MS) },
|
|
648
|
+
);
|
|
649
|
+
// Chromium can abort document navigation because the accepted
|
|
650
|
+
// response is a download. This does not replace the current page.
|
|
651
|
+
if (result.errorText && !result.isDownload) {
|
|
652
|
+
throw new Error(`Navigation failed: ${result.errorText}`);
|
|
653
|
+
}
|
|
654
|
+
return {
|
|
655
|
+
frameId: result.frameId,
|
|
656
|
+
...(result.loaderId ? { loaderId: result.loaderId } : {}),
|
|
657
|
+
url,
|
|
658
|
+
...(result.isDownload ? { download: true } : {}),
|
|
659
|
+
};
|
|
660
|
+
},
|
|
661
|
+
assertCurrent,
|
|
662
|
+
Math.min(this.timeoutMs, NAVIGATION_COMMIT_TIMEOUT_MS),
|
|
411
663
|
);
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
clearTimeout(timer);
|
|
416
|
-
page.off("Page.lifecycleEvent", onLifecycle);
|
|
417
|
-
}
|
|
418
|
-
});
|
|
664
|
+
},
|
|
665
|
+
{ preserveStream: true },
|
|
666
|
+
);
|
|
419
667
|
}
|
|
420
668
|
|
|
421
669
|
async back(): Promise<void> {
|
|
422
|
-
await this.
|
|
423
|
-
const history = await this.navigationHistory(page);
|
|
424
|
-
const entry = history.entries[history.currentIndex - 1];
|
|
425
|
-
if (!entry) throw new Error("Browser cannot go back");
|
|
426
|
-
await page.send("Page.navigateToHistoryEntry", { entryId: entry.id }, { mutation: true });
|
|
427
|
-
});
|
|
670
|
+
await this.navigateHistory(-1);
|
|
428
671
|
}
|
|
429
672
|
|
|
430
673
|
async forward(): Promise<void> {
|
|
431
|
-
await this.
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
674
|
+
await this.navigateHistory(1);
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
/** History's ACK has no loader receipt. Capture its root loader and target URL
|
|
678
|
+
* in one parallel bounded preflight, then admit only the matching new commit. */
|
|
679
|
+
private async navigateHistory(direction: -1 | 1): Promise<void> {
|
|
680
|
+
await this.withInvalidatedScreencast(
|
|
681
|
+
async (page) => {
|
|
682
|
+
const assertCurrent = this.navigationAttachmentGuard(page);
|
|
683
|
+
const [history, root] = await Promise.all([
|
|
684
|
+
this.navigationHistory(page, Math.min(this.timeoutMs, NAVIGATION_METADATA_TIMEOUT_MS)),
|
|
685
|
+
this.navigationRoot(page),
|
|
686
|
+
]);
|
|
687
|
+
assertCurrent();
|
|
688
|
+
const entry = history.entries[history.currentIndex + direction];
|
|
689
|
+
if (!entry)
|
|
690
|
+
throw new Error(
|
|
691
|
+
direction === -1 ? "Browser cannot go back" : "Browser cannot go forward",
|
|
692
|
+
);
|
|
693
|
+
await waitForNavigationCommit(
|
|
694
|
+
page,
|
|
695
|
+
async () => {
|
|
696
|
+
await page.send(
|
|
697
|
+
"Page.navigateToHistoryEntry",
|
|
698
|
+
{ entryId: entry.id },
|
|
699
|
+
{
|
|
700
|
+
mutation: true,
|
|
701
|
+
timeoutMs: Math.min(this.timeoutMs, NAVIGATION_ACK_TIMEOUT_MS),
|
|
702
|
+
},
|
|
703
|
+
);
|
|
704
|
+
return { frameId: root.id, previousLoaderId: root.loaderId, url: entry.url };
|
|
705
|
+
},
|
|
706
|
+
assertCurrent,
|
|
707
|
+
Math.min(this.timeoutMs, NAVIGATION_COMMIT_TIMEOUT_MS),
|
|
708
|
+
);
|
|
709
|
+
},
|
|
710
|
+
{ preserveStream: true },
|
|
711
|
+
);
|
|
437
712
|
}
|
|
438
713
|
|
|
439
714
|
async reload(ignoreCache = false): Promise<void> {
|
|
440
|
-
await this.withInvalidatedScreencast(
|
|
441
|
-
page
|
|
715
|
+
await this.withInvalidatedScreencast(
|
|
716
|
+
async (page) => {
|
|
717
|
+
const assertCurrent = this.navigationAttachmentGuard(page);
|
|
718
|
+
const root = await this.navigationRoot(page);
|
|
719
|
+
assertCurrent();
|
|
720
|
+
await waitForNavigationCommit(
|
|
721
|
+
page,
|
|
722
|
+
async () => {
|
|
723
|
+
await page.send(
|
|
724
|
+
"Page.reload",
|
|
725
|
+
{ ignoreCache },
|
|
726
|
+
{
|
|
727
|
+
mutation: true,
|
|
728
|
+
timeoutMs: Math.min(this.timeoutMs, NAVIGATION_ACK_TIMEOUT_MS),
|
|
729
|
+
},
|
|
730
|
+
);
|
|
731
|
+
return { frameId: root.id, previousLoaderId: root.loaderId };
|
|
732
|
+
},
|
|
733
|
+
assertCurrent,
|
|
734
|
+
Math.min(this.timeoutMs, NAVIGATION_COMMIT_TIMEOUT_MS),
|
|
735
|
+
);
|
|
736
|
+
},
|
|
737
|
+
{ preserveStream: true },
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
/** A root snapshot is observation only, bounded before mutation publication. */
|
|
742
|
+
private async navigationRoot(page: CdpSession): Promise<{ id: string; loaderId: string }> {
|
|
743
|
+
const result = await page.send<{ frameTree: { frame: { id: string; loaderId: string } } }>(
|
|
744
|
+
"Page.getFrameTree",
|
|
745
|
+
{},
|
|
746
|
+
{ timeoutMs: Math.min(this.timeoutMs, NAVIGATION_METADATA_TIMEOUT_MS) },
|
|
442
747
|
);
|
|
748
|
+
const root = result.frameTree.frame;
|
|
749
|
+
if (!root.id || !root.loaderId)
|
|
750
|
+
throw new CdpUnavailableError("Browser main frame is unavailable");
|
|
751
|
+
return root;
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
/** Keep commit observation on the original attachment, including shutdown. */
|
|
755
|
+
private navigationAttachmentGuard(page: CdpSession): () => void {
|
|
756
|
+
const attachmentGeneration = this.attachmentGeneration;
|
|
757
|
+
return () => {
|
|
758
|
+
this.assertAttachmentCurrent(page.connection, attachmentGeneration);
|
|
759
|
+
if (this.page !== page)
|
|
760
|
+
throw new CdpUnavailableError("Browser page changed during navigation");
|
|
761
|
+
};
|
|
443
762
|
}
|
|
444
763
|
|
|
445
764
|
async emulate(device: DeviceEmulation): Promise<void> {
|
|
446
765
|
this.assertViewport(device);
|
|
447
|
-
await this.
|
|
766
|
+
await this.withVideoTransition(async () => {
|
|
767
|
+
await this.withInvalidatedScreencast(async (page) => {
|
|
768
|
+
const attachmentGeneration = this.attachmentGeneration;
|
|
769
|
+
this.emulationAppliedPage = null;
|
|
770
|
+
await this.applyEmulation(page, device);
|
|
771
|
+
this.assertAttachmentCurrent(page.connection, attachmentGeneration);
|
|
772
|
+
if (this.page !== page)
|
|
773
|
+
throw new CdpUnavailableError("Browser page changed during emulation");
|
|
774
|
+
this.viewport = { ...device };
|
|
775
|
+
this.emulationAppliedPage = page;
|
|
776
|
+
});
|
|
777
|
+
});
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
/** Retain the full mode/UA/screen intent while rendering enough real source pixels for the chosen density. */
|
|
781
|
+
async setCaptureDensity(density: number, baseDeviceScaleFactor: number): Promise<void> {
|
|
782
|
+
const scale = captureDensitySchema.parse(density);
|
|
783
|
+
const viewport = this.requireViewport();
|
|
784
|
+
if (!canUseCaptureDensity(viewport, scale))
|
|
785
|
+
throw new RangeError("Capture density exceeds the supported image bounds");
|
|
786
|
+
if (
|
|
787
|
+
!Number.isFinite(baseDeviceScaleFactor) ||
|
|
788
|
+
baseDeviceScaleFactor < 1 ||
|
|
789
|
+
baseDeviceScaleFactor > 4
|
|
790
|
+
)
|
|
791
|
+
throw new RangeError("Invalid device pixel ratio");
|
|
792
|
+
await this.emulate({
|
|
793
|
+
...viewport,
|
|
794
|
+
captureScale: scale,
|
|
795
|
+
deviceScaleFactor: Math.max(baseDeviceScaleFactor, scale),
|
|
796
|
+
});
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
/** Apply the entire device contract to one CDP session without changing saved intent. */
|
|
800
|
+
private async applyEmulation(page: CdpSession, device: DeviceEmulation): Promise<void> {
|
|
801
|
+
await page.send(
|
|
802
|
+
"Emulation.setDeviceMetricsOverride",
|
|
803
|
+
{
|
|
804
|
+
width: device.width,
|
|
805
|
+
height: device.height,
|
|
806
|
+
deviceScaleFactor: device.deviceScaleFactor,
|
|
807
|
+
mobile: device.mobile,
|
|
808
|
+
screenWidth: device.screenWidth ?? device.width,
|
|
809
|
+
screenHeight: device.screenHeight ?? device.height,
|
|
810
|
+
screenOrientation: {
|
|
811
|
+
type: device.width > device.height ? "landscapePrimary" : "portraitPrimary",
|
|
812
|
+
angle: device.width > device.height ? 90 : 0,
|
|
813
|
+
},
|
|
814
|
+
},
|
|
815
|
+
{ mutation: true },
|
|
816
|
+
);
|
|
817
|
+
await page.send(
|
|
818
|
+
"Emulation.setTouchEmulationEnabled",
|
|
819
|
+
{ enabled: device.touch, maxTouchPoints: device.touch ? 5 : 1 },
|
|
820
|
+
{ mutation: true },
|
|
821
|
+
);
|
|
822
|
+
// A page controlled through its own session may sit behind another tab in the shared
|
|
823
|
+
// browser. Chromium holds a hidden page's mouseMoved acknowledgement for ~5 s (equal to the
|
|
824
|
+
// input idle bound), so every press there failed. Target-scoped focus emulation makes only
|
|
825
|
+
// this page report visible/focused: no tab is raised and no viewer's selection changes.
|
|
826
|
+
await page.send("Emulation.setFocusEmulationEnabled", { enabled: true }, { mutation: true });
|
|
827
|
+
if (device.userAgent) {
|
|
828
|
+
await page.send(
|
|
829
|
+
"Emulation.setUserAgentOverride",
|
|
830
|
+
{ userAgent: device.userAgent, platform: device.platform ?? "" },
|
|
831
|
+
{ mutation: true },
|
|
832
|
+
);
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
/** No frame/input may use a session whose emulated viewport has not been restored. */
|
|
837
|
+
private async restoreConfiguredEmulation(page: CdpSession): Promise<void> {
|
|
838
|
+
const device = this.viewport;
|
|
839
|
+
if (!device || this.emulationAppliedPage === page) return;
|
|
840
|
+
await this.withVideoTransition(async () => {
|
|
841
|
+
const attachmentGeneration = this.attachmentGeneration;
|
|
842
|
+
this.invalidateScreencastFrame();
|
|
843
|
+
this.emulationAppliedPage = null;
|
|
448
844
|
await this.applyEmulation(page, device);
|
|
449
|
-
this.
|
|
845
|
+
this.assertAttachmentCurrent(page.connection, attachmentGeneration);
|
|
846
|
+
if (device !== this.viewport) {
|
|
847
|
+
throw new CdpUnavailableError("Browser device settings changed during restoration");
|
|
848
|
+
}
|
|
849
|
+
this.emulationAppliedPage = page;
|
|
850
|
+
this.invalidateScreencastFrame();
|
|
851
|
+
});
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
/** Reject late configuration completions after a disconnect or newer target selection. */
|
|
855
|
+
private assertAttachmentCurrent(connection: CdpConnection, generation: number): void {
|
|
856
|
+
this.assertRuntimeOpen();
|
|
857
|
+
if (
|
|
858
|
+
connection !== this.connection ||
|
|
859
|
+
!connection.isOpen ||
|
|
860
|
+
generation !== this.attachmentGeneration
|
|
861
|
+
) {
|
|
862
|
+
throw new CdpUnavailableError("Browser attachment changed during emulation restoration");
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
/** Starts only exact restored native tab capture, lazily adding this viewer's quality encoder. */
|
|
867
|
+
async startVideo(quality: NativeVideoQuality = "high"): Promise<NativeVideoRead> {
|
|
868
|
+
return this.readVideo({ quality, waitMs: 0 });
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
/** Long-poll outside the input queue. The caller must fence returned inputGeneration before publication. */
|
|
872
|
+
async readVideo(input: {
|
|
873
|
+
quality: NativeVideoQuality;
|
|
874
|
+
bitrate?: number;
|
|
875
|
+
fps?: number;
|
|
876
|
+
streamId?: string | null;
|
|
877
|
+
afterSequence?: number;
|
|
878
|
+
waitMs?: number;
|
|
879
|
+
requestKeyFrame?: boolean;
|
|
880
|
+
}): Promise<NativeVideoRead & { inputGeneration: string }> {
|
|
881
|
+
if (!this.nativeVideo) {
|
|
882
|
+
return {
|
|
883
|
+
...this.videoReset(),
|
|
884
|
+
status: "unsupported",
|
|
885
|
+
reasonCode: "video-disabled",
|
|
886
|
+
reason: "Encoded video is not enabled on this host",
|
|
887
|
+
};
|
|
888
|
+
}
|
|
889
|
+
const settings = resolveVideoEncoderSettings(input);
|
|
890
|
+
const transitionEpoch = this.videoTransitionEpoch;
|
|
891
|
+
if (!this.videoMayStart(transitionEpoch)) return this.videoReset();
|
|
892
|
+
|
|
893
|
+
// Retirement uses the same stop barrier as viewport changes. The next read
|
|
894
|
+
// resumes from current configuration instead of treating idle expiry as a
|
|
895
|
+
// permanent codec failure or creating a replacement while the old track closes.
|
|
896
|
+
if (this.video?.idleExpired) {
|
|
897
|
+
this.retireVideoForRead();
|
|
898
|
+
return this.videoReset();
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
const page = await this.requirePage();
|
|
902
|
+
// requirePage can itself restore or reconnect. Retry on the next read rather
|
|
903
|
+
// than creating a helper from a configuration that changed across its await.
|
|
904
|
+
if (!this.videoMayStart(transitionEpoch)) return this.videoReset();
|
|
905
|
+
const connection = this.requireConnection();
|
|
906
|
+
const generation = this.attachmentGeneration;
|
|
907
|
+
const viewport = this.requireViewport();
|
|
908
|
+
const recoveryContext = { connection, page, viewport, attachmentGeneration: generation };
|
|
909
|
+
const blocked = this.videoRecovery.blocked(recoveryContext);
|
|
910
|
+
if (blocked === "source-dimensions") {
|
|
911
|
+
return {
|
|
912
|
+
...this.videoReset(),
|
|
913
|
+
status: "unsupported",
|
|
914
|
+
reason: "Native video dimensions do not match the configured viewport",
|
|
915
|
+
};
|
|
916
|
+
}
|
|
917
|
+
if (blocked) {
|
|
918
|
+
return this.videoReset();
|
|
919
|
+
}
|
|
920
|
+
if (!this.video) {
|
|
921
|
+
const pixels = captureDimensions(viewport, viewport.captureScale ?? 1);
|
|
922
|
+
this.video = new NativeVideoCapture({
|
|
923
|
+
connection,
|
|
924
|
+
targetId: page.targetId,
|
|
925
|
+
width: pixels.width,
|
|
926
|
+
height: pixels.height,
|
|
927
|
+
isCurrent: () =>
|
|
928
|
+
this.videoMayStart(transitionEpoch) &&
|
|
929
|
+
connection === this.connection &&
|
|
930
|
+
page === this.page &&
|
|
931
|
+
viewport === this.viewport &&
|
|
932
|
+
generation === this.attachmentGeneration &&
|
|
933
|
+
this.emulationAppliedPage === page,
|
|
934
|
+
});
|
|
935
|
+
}
|
|
936
|
+
const video = this.video;
|
|
937
|
+
const documentGeneration = this.documentGeneration;
|
|
938
|
+
const result = await video.read({ ...input, ...settings });
|
|
939
|
+
if (
|
|
940
|
+
!this.videoMayStart(transitionEpoch) ||
|
|
941
|
+
video.idleExpired ||
|
|
942
|
+
video !== this.video ||
|
|
943
|
+
connection !== this.connection ||
|
|
944
|
+
page !== this.page ||
|
|
945
|
+
viewport !== this.viewport ||
|
|
946
|
+
generation !== this.attachmentGeneration ||
|
|
947
|
+
documentGeneration !== this.documentGeneration
|
|
948
|
+
) {
|
|
949
|
+
return this.videoReset();
|
|
950
|
+
}
|
|
951
|
+
if (video.sourceFailure) {
|
|
952
|
+
// Store failure admission before opening the existing stop barrier. A
|
|
953
|
+
// concurrent read sees either that barrier or the cooldown, never a second
|
|
954
|
+
// track while the failed source/helper is still being retired.
|
|
955
|
+
this.videoRecovery.failed(recoveryContext, video.sourceFailure);
|
|
956
|
+
const permanent = video.sourceFailure === "source-dimensions";
|
|
957
|
+
this.retireVideoForRead();
|
|
958
|
+
if (
|
|
959
|
+
this.stopping ||
|
|
960
|
+
connection !== this.connection ||
|
|
961
|
+
page !== this.page ||
|
|
962
|
+
viewport !== this.viewport ||
|
|
963
|
+
generation !== this.attachmentGeneration
|
|
964
|
+
) {
|
|
965
|
+
return this.videoReset();
|
|
966
|
+
}
|
|
967
|
+
if (permanent) {
|
|
968
|
+
return {
|
|
969
|
+
...this.videoReset(),
|
|
970
|
+
status: "unsupported",
|
|
971
|
+
reason: "Native video dimensions do not match the configured viewport",
|
|
972
|
+
};
|
|
973
|
+
}
|
|
974
|
+
return this.videoReset();
|
|
975
|
+
}
|
|
976
|
+
return {
|
|
977
|
+
...result,
|
|
978
|
+
inputGeneration: formatRuntimeInputGeneration(
|
|
979
|
+
this.attachmentGeneration,
|
|
980
|
+
this.documentGeneration,
|
|
981
|
+
),
|
|
982
|
+
};
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
/** Explicit action barrier; pointer samples preserve the native track and encoder continuity. */
|
|
986
|
+
invalidateQueuedVideoFrames(): void {
|
|
987
|
+
this.video?.invalidateQueuedFrames();
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
/** Establish the stop barrier now, but do not hold a viewing RPC across slow
|
|
991
|
+
* helper teardown. stopVideo owns barrier release; no replacement can start
|
|
992
|
+
* until it settles. Explicit stop callers still await their cleanup. */
|
|
993
|
+
private retireVideoForRead(): void {
|
|
994
|
+
const retirement = this.stopVideo();
|
|
995
|
+
this.videoRetirement = retirement;
|
|
996
|
+
const complete = () => {
|
|
997
|
+
if (this.videoRetirement === retirement) this.videoRetirement = null;
|
|
998
|
+
};
|
|
999
|
+
void retirement.then(complete, () => {
|
|
1000
|
+
complete();
|
|
1001
|
+
console.warn("[shared-browser] Video retirement failed");
|
|
450
1002
|
});
|
|
451
1003
|
}
|
|
452
1004
|
|
|
453
|
-
|
|
1005
|
+
/** Stop admission too: a concurrent reader cannot replace a source still closing. */
|
|
1006
|
+
async stopVideo(): Promise<void> {
|
|
1007
|
+
if (this.videoRetirement) {
|
|
1008
|
+
await this.videoRetirement;
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
await this.withVideoTransition(async () => {});
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
/** Nested device/attachment phases keep one continuous barrier until all phases finish. */
|
|
1015
|
+
private async withVideoTransition<T>(transition: () => Promise<T>): Promise<T> {
|
|
1016
|
+
const releaseJpeg = this.stopping ? () => {} : this.jpegDemand.pin(false);
|
|
1017
|
+
this.videoTransitionDepth += 1;
|
|
1018
|
+
this.videoTransitionEpoch += 1;
|
|
1019
|
+
const video = this.video;
|
|
1020
|
+
this.video = null;
|
|
1021
|
+
try {
|
|
1022
|
+
if (video) await video.stop();
|
|
1023
|
+
return await transition();
|
|
1024
|
+
} finally {
|
|
1025
|
+
this.videoTransitionDepth -= 1;
|
|
1026
|
+
releaseJpeg();
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
private videoMayStart(epoch: number): boolean {
|
|
1031
|
+
return !this.stopping && this.videoTransitionDepth === 0 && epoch === this.videoTransitionEpoch;
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
private videoReset(): NativeVideoRead & { inputGeneration: string } {
|
|
1035
|
+
return {
|
|
1036
|
+
status: "reset",
|
|
1037
|
+
streamId: null,
|
|
1038
|
+
packets: [],
|
|
1039
|
+
inputGeneration: formatRuntimeInputGeneration(
|
|
1040
|
+
this.attachmentGeneration,
|
|
1041
|
+
this.documentGeneration,
|
|
1042
|
+
),
|
|
1043
|
+
};
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
/** Explicit viewing start remains supported, but does not keep unused JPEG work alive. */
|
|
1047
|
+
async startScreencast(quality: number = DEFAULT_JPEG_QUALITY): Promise<void> {
|
|
454
1048
|
if (!Number.isInteger(quality) || quality < 1 || quality > 100) {
|
|
455
1049
|
throw new RangeError("Screencast quality must be an integer from 1 to 100");
|
|
456
1050
|
}
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
1051
|
+
const release = this.jpegDemand.pin();
|
|
1052
|
+
try {
|
|
1053
|
+
await this.jpegDemand.serialize(async () => {
|
|
1054
|
+
const page = await this.requirePage();
|
|
1055
|
+
if (this.screencastActive && this.screencastQuality === quality) return;
|
|
1056
|
+
this.screencastQuality = quality;
|
|
1057
|
+
this.screencastStartFailed = false;
|
|
1058
|
+
this.screencastActive = true;
|
|
1059
|
+
try {
|
|
1060
|
+
await this.startScreencastSession(page);
|
|
1061
|
+
} catch (error) {
|
|
1062
|
+
this.screencastActive = false;
|
|
1063
|
+
this.screencastStartFailed = true;
|
|
1064
|
+
throw error;
|
|
1065
|
+
}
|
|
1066
|
+
});
|
|
1067
|
+
} finally {
|
|
1068
|
+
release();
|
|
1069
|
+
}
|
|
461
1070
|
}
|
|
462
1071
|
|
|
1072
|
+
/** Explicit detach cleanup and idle retirement use the same native transition FIFO. */
|
|
463
1073
|
async stopScreencast(): Promise<void> {
|
|
1074
|
+
this.jpegDemand.cancel();
|
|
1075
|
+
await this.jpegDemand.serialize(() => this.stopScreencastSession());
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
private async stopScreencastSession(): Promise<void> {
|
|
1079
|
+
const wasActive = this.screencastActive;
|
|
464
1080
|
this.screencastActive = false;
|
|
1081
|
+
this.screencastStartFailed = false;
|
|
465
1082
|
this.invalidateScreencastFrame();
|
|
466
|
-
if (!this.page) return;
|
|
1083
|
+
if (!wasActive || !this.page) return;
|
|
467
1084
|
try {
|
|
468
1085
|
await this.page.send("Page.stopScreencast", {}, { mutation: true });
|
|
469
1086
|
} finally {
|
|
@@ -471,71 +1088,288 @@ export class AgentBrowserRuntime {
|
|
|
471
1088
|
}
|
|
472
1089
|
}
|
|
473
1090
|
|
|
474
|
-
|
|
475
|
-
|
|
1091
|
+
/** Return truthful fresh stream/fallback pixels; mutation/session changes revoke pending captures. */
|
|
1092
|
+
async frame(
|
|
1093
|
+
maxBytes: number,
|
|
1094
|
+
quality: number = DEFAULT_JPEG_QUALITY,
|
|
1095
|
+
waitMs = 500,
|
|
1096
|
+
): Promise<RuntimeFrame> {
|
|
1097
|
+
if (!Number.isInteger(maxBytes) || maxBytes < 1) {
|
|
476
1098
|
throw new RangeError("maxBytes must be positive");
|
|
477
|
-
|
|
478
|
-
if (
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
//
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
1099
|
+
}
|
|
1100
|
+
if (!Number.isFinite(quality)) throw new RangeError("JPEG quality must be finite");
|
|
1101
|
+
const requestedQuality = Math.max(1, Math.min(100, Math.round(quality)));
|
|
1102
|
+
// All reads protect in-flight pixels, but screenshot-only qualities do not
|
|
1103
|
+
// renew the shared JPEG stream when a viewer changes their preference.
|
|
1104
|
+
const release = this.jpegDemand.pin(requestedQuality === this.screencastQuality);
|
|
1105
|
+
try {
|
|
1106
|
+
await this.jpegDemand.serialize(async () => {
|
|
1107
|
+
const page = await this.requirePage();
|
|
1108
|
+
// Nonmatching quality keeps the existing screenshot-only path. A video
|
|
1109
|
+
// viewer choosing JPEG100 must not start an unused JPEG95 stream.
|
|
1110
|
+
if (
|
|
1111
|
+
requestedQuality !== this.screencastQuality ||
|
|
1112
|
+
this.screencastActive ||
|
|
1113
|
+
this.screencastStartFailed
|
|
1114
|
+
) {
|
|
1115
|
+
return;
|
|
1116
|
+
}
|
|
1117
|
+
this.screencastActive = true;
|
|
1118
|
+
try {
|
|
1119
|
+
await this.startScreencastSession(page);
|
|
1120
|
+
} catch {
|
|
1121
|
+
// Shutdown is terminal, not an optional stream failure to fall back from.
|
|
1122
|
+
this.assertRuntimeOpen();
|
|
1123
|
+
// Stream support is optional. Preserve fresh screenshots, and avoid
|
|
1124
|
+
// retrying a failed setup at every 250ms poll in the same demand period.
|
|
1125
|
+
console.warn("Shared Browser JPEG screencast is unavailable; using screenshot fallback");
|
|
1126
|
+
this.screencastActive = false;
|
|
1127
|
+
this.screencastStartFailed = true;
|
|
1128
|
+
}
|
|
497
1129
|
});
|
|
498
|
-
|
|
499
|
-
if (
|
|
500
|
-
|
|
1130
|
+
this.assertRuntimeOpen();
|
|
1131
|
+
if (requestedQuality !== this.screencastQuality) {
|
|
1132
|
+
// Viewers choose their own quality; never restart the shared stream or
|
|
1133
|
+
// classify its health from another viewer's screenshot preference.
|
|
1134
|
+
const cached = this.capturePolicy.readScreenshot(
|
|
1135
|
+
performance.now(),
|
|
1136
|
+
requestedQuality,
|
|
1137
|
+
maxBytes,
|
|
1138
|
+
);
|
|
1139
|
+
return cached ?? (await this.captureScreenshot(maxBytes, requestedQuality, true));
|
|
1140
|
+
}
|
|
1141
|
+
// Expiry discards pixels without resetting fallback dwell/recovery evidence.
|
|
1142
|
+
// Mutations and connection changes use the stronger invalidation below.
|
|
1143
|
+
const generation = this.capturePolicy.currentGeneration();
|
|
1144
|
+
const receivedAt = this.screencastFrameReceivedAt;
|
|
1145
|
+
if (receivedAt === null || performance.now() - receivedAt > CAPTURE_MAX_AGE_MS) {
|
|
1146
|
+
this.screencastFrame = null;
|
|
1147
|
+
this.screencastFrameReceivedAt = null;
|
|
1148
|
+
}
|
|
1149
|
+
if (this.screencastActive && this.capturePolicy.canUseStream(performance.now())) {
|
|
1150
|
+
const streamed = this.screencastFrame ?? (await this.waitForFrame(waitMs));
|
|
1151
|
+
this.assertRuntimeOpen();
|
|
1152
|
+
if (!this.capturePolicy.isCurrent(generation)) {
|
|
1153
|
+
throw new CdpUnavailableError(
|
|
1154
|
+
"Browser capture was invalidated while waiting for a frame",
|
|
1155
|
+
);
|
|
1156
|
+
}
|
|
1157
|
+
if (streamed && streamed.byteLength <= maxBytes) {
|
|
1158
|
+
return streamed;
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
this.capturePolicy.enterFallback(performance.now());
|
|
1163
|
+
const cached = this.capturePolicy.readScreenshot(
|
|
1164
|
+
performance.now(),
|
|
1165
|
+
requestedQuality,
|
|
1166
|
+
maxBytes,
|
|
1167
|
+
);
|
|
1168
|
+
if (cached) {
|
|
1169
|
+
return cached;
|
|
1170
|
+
}
|
|
1171
|
+
return await this.captureScreenshot(maxBytes, requestedQuality);
|
|
1172
|
+
} finally {
|
|
1173
|
+
release();
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
/** Capture the configured viewport and cache only an unchanged-session/mutation completion. */
|
|
1178
|
+
private async captureScreenshot(
|
|
1179
|
+
maxBytes: number,
|
|
1180
|
+
requestedQuality: number,
|
|
1181
|
+
preserveStreamRecovery = false,
|
|
1182
|
+
): Promise<RuntimeFrame> {
|
|
1183
|
+
const page = await this.requirePage();
|
|
1184
|
+
const screenshotGeneration = this.capturePolicy.beginScreenshot({
|
|
1185
|
+
preserveRecovery: preserveStreamRecovery,
|
|
1186
|
+
});
|
|
1187
|
+
try {
|
|
1188
|
+
const viewport = this.requireViewport();
|
|
1189
|
+
const expectedPixels = captureDimensions(viewport, viewport.captureScale);
|
|
1190
|
+
const readClip = async () => {
|
|
1191
|
+
const metrics = await page.send<{ cssVisualViewport: { pageX: number; pageY: number } }>(
|
|
1192
|
+
"Page.getLayoutMetrics",
|
|
1193
|
+
);
|
|
1194
|
+
this.assertCaptureCurrent(page, screenshotGeneration);
|
|
501
1195
|
return {
|
|
502
|
-
|
|
503
|
-
|
|
1196
|
+
x: metrics.cssVisualViewport.pageX,
|
|
1197
|
+
y: metrics.cssVisualViewport.pageY,
|
|
504
1198
|
width: viewport.width,
|
|
505
1199
|
height: viewport.height,
|
|
1200
|
+
scale: (viewport.captureScale ?? 1) / viewport.deviceScaleFactor,
|
|
1201
|
+
};
|
|
1202
|
+
};
|
|
1203
|
+
// Explicitly capture the visible CSS viewport, independent of the native window size.
|
|
1204
|
+
// Preserve scroll position and normalize device pixel ratio to the frame/input contract.
|
|
1205
|
+
let clip = await readClip();
|
|
1206
|
+
let resynced = false;
|
|
1207
|
+
// Reduce quality gradually only when a detailed frame exceeds the byte budget.
|
|
1208
|
+
const initialQuality = requestedQuality;
|
|
1209
|
+
const candidates = [...new Set([initialQuality, 90, 85, 75, 65, 50, 35, 20, 10, 1])].filter(
|
|
1210
|
+
(candidate) => candidate <= initialQuality,
|
|
1211
|
+
);
|
|
1212
|
+
const capture = async (quality: number) => {
|
|
1213
|
+
this.assertCaptureCurrent(page, screenshotGeneration);
|
|
1214
|
+
const result = await page.send<{ data: string }>("Page.captureScreenshot", {
|
|
1215
|
+
format: "jpeg",
|
|
1216
|
+
quality,
|
|
1217
|
+
fromSurface: true,
|
|
1218
|
+
// Beyond-viewport capture replaces emulation and resets enabled touch.
|
|
1219
|
+
captureBeyondViewport: false,
|
|
1220
|
+
clip,
|
|
1221
|
+
});
|
|
1222
|
+
this.assertCaptureCurrent(page, screenshotGeneration);
|
|
1223
|
+
const dimensions = readJpegFrameDimensions(result.data);
|
|
1224
|
+
if (!dimensions) throw new Error("Chromium returned a malformed JPEG screenshot");
|
|
1225
|
+
return {
|
|
1226
|
+
data: result.data,
|
|
1227
|
+
dimensions,
|
|
1228
|
+
byteLength: Buffer.byteLength(result.data, "base64"),
|
|
1229
|
+
};
|
|
1230
|
+
};
|
|
1231
|
+
const matchesViewport = (dimensions: { width: number; height: number }) =>
|
|
1232
|
+
dimensions.width === expectedPixels.width && dimensions.height === expectedPixels.height;
|
|
1233
|
+
|
|
1234
|
+
for (const candidate of candidates) {
|
|
1235
|
+
const boundedQuality = Math.max(1, Math.min(100, Math.round(candidate)));
|
|
1236
|
+
let pixels = await capture(boundedQuality);
|
|
1237
|
+
if (!matchesViewport(pixels.dimensions) && !resynced) {
|
|
1238
|
+
// Retain upstream viewport recovery, with captureScale rather than DPR:
|
|
1239
|
+
// mobile CSS emulation and image density are independently configured.
|
|
1240
|
+
// Recapture pixels only; uncertain input is never replayed.
|
|
1241
|
+
resynced = true;
|
|
1242
|
+
const attachmentGeneration = this.attachmentGeneration;
|
|
1243
|
+
await this.applyEmulation(page, viewport);
|
|
1244
|
+
this.assertAttachmentCurrent(page.connection, attachmentGeneration);
|
|
1245
|
+
this.assertCaptureCurrent(page, screenshotGeneration);
|
|
1246
|
+
if (viewport !== this.viewport)
|
|
1247
|
+
throw new CdpUnavailableError("Browser viewport changed during resync");
|
|
1248
|
+
this.emulationAppliedPage = page;
|
|
1249
|
+
clip = await readClip();
|
|
1250
|
+
pixels = await capture(boundedQuality);
|
|
1251
|
+
}
|
|
1252
|
+
if (!matchesViewport(pixels.dimensions)) {
|
|
1253
|
+
throw new Error(
|
|
1254
|
+
`Chromium returned a ${pixels.dimensions.width}x${pixels.dimensions.height} JPEG instead of ${expectedPixels.width}x${expectedPixels.height}`,
|
|
1255
|
+
);
|
|
1256
|
+
}
|
|
1257
|
+
if (pixels.byteLength > maxBytes) continue;
|
|
1258
|
+
|
|
1259
|
+
const frame: RuntimeFrame = {
|
|
1260
|
+
dataBase64: pixels.data,
|
|
1261
|
+
byteLength: pixels.byteLength,
|
|
1262
|
+
width: pixels.dimensions.width,
|
|
1263
|
+
height: pixels.dimensions.height,
|
|
506
1264
|
transport: "screenshot",
|
|
507
1265
|
capturedAt: new Date().toISOString(),
|
|
508
|
-
}
|
|
1266
|
+
};
|
|
1267
|
+
this.capturePolicy.rememberScreenshot(
|
|
1268
|
+
screenshotGeneration,
|
|
1269
|
+
frame,
|
|
1270
|
+
performance.now(),
|
|
1271
|
+
requestedQuality,
|
|
1272
|
+
maxBytes,
|
|
1273
|
+
);
|
|
1274
|
+
return frame;
|
|
509
1275
|
}
|
|
1276
|
+
throw new Error(`JPEG screenshot exceeds ${maxBytes} bytes at minimum quality`);
|
|
1277
|
+
} finally {
|
|
1278
|
+
this.capturePolicy.endScreenshot(screenshotGeneration, performance.now());
|
|
510
1279
|
}
|
|
511
|
-
throw new Error(`JPEG screenshot exceeds ${maxBytes} bytes at minimum quality`);
|
|
512
1280
|
}
|
|
513
1281
|
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
1282
|
+
/** Refuse async pixels from a superseded session or mutation, without silently replaying capture. */
|
|
1283
|
+
private assertCaptureCurrent(page: CdpSession, generation: number): void {
|
|
1284
|
+
this.assertRuntimeOpen();
|
|
1285
|
+
if (page !== this.page || !this.capturePolicy.isCurrent(generation)) {
|
|
1286
|
+
throw new CdpUnavailableError("Browser capture was invalidated while taking a screenshot");
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
/** Invalidate pre-input pixels even if Chromium cannot confirm the mutation. */
|
|
1291
|
+
private async dispatchInput(
|
|
1292
|
+
method: string,
|
|
1293
|
+
params: Record<string, unknown>,
|
|
1294
|
+
gestureId?: string,
|
|
1295
|
+
mouseModifiers?: number,
|
|
1296
|
+
track?: () => void,
|
|
1297
|
+
): Promise<void> {
|
|
1298
|
+
try {
|
|
1299
|
+
const page = await this.requirePage();
|
|
1300
|
+
if (gestureId) await this.assertLiveInput(gestureId);
|
|
1301
|
+
// Authority is validated. Record cleanup intent only now, immediately before
|
|
1302
|
+
// native publication, so rejected unsent packets never alter it while an
|
|
1303
|
+
// uncertain acknowledgement still leaves it in place.
|
|
1304
|
+
if (track) {
|
|
1305
|
+
if (this.heldOwner && this.heldOwner !== page) this.discardHeldState();
|
|
1306
|
+
track();
|
|
1307
|
+
if (this.holdsInput()) this.heldOwner ??= page;
|
|
1308
|
+
}
|
|
1309
|
+
const originalInput = gestureId ? this.liveInput : null;
|
|
1310
|
+
// A physical DOM mouse event may arrive after Control/Shift was pressed
|
|
1311
|
+
// outside this canvas. Explicit zero also overrides a prior held-key mask.
|
|
1312
|
+
const modifiers = mouseModifiers ?? heldKeyModifiers(this.heldKeys.values());
|
|
1313
|
+
const nativeParams =
|
|
1314
|
+
method === "Input.dispatchMouseEvent" && (mouseModifiers !== undefined || modifiers !== 0)
|
|
1315
|
+
? { ...params, modifiers }
|
|
1316
|
+
: params;
|
|
1317
|
+
await page.send(method, nativeParams, { mutation: true });
|
|
1318
|
+
if (gestureId) {
|
|
1319
|
+
// An input can synchronously activate a link or submit a form. Its CDP
|
|
1320
|
+
// acknowledgment succeeds even when navigation ends the old channel.
|
|
1321
|
+
// Attachment replacement and uncertain sends retain their failure path.
|
|
1322
|
+
if (
|
|
1323
|
+
originalInput &&
|
|
1324
|
+
page === this.page &&
|
|
1325
|
+
originalInput.generation === this.attachmentGeneration &&
|
|
1326
|
+
originalInput.documentGeneration !== this.documentGeneration
|
|
1327
|
+
) {
|
|
1328
|
+
await this.endLiveInput(gestureId);
|
|
1329
|
+
return;
|
|
1330
|
+
}
|
|
1331
|
+
await this.assertLiveInput(gestureId);
|
|
1332
|
+
this.renewLiveInput(gestureId);
|
|
1333
|
+
}
|
|
1334
|
+
} finally {
|
|
1335
|
+
this.invalidateScreencastFrame();
|
|
1336
|
+
}
|
|
525
1337
|
}
|
|
526
1338
|
|
|
527
|
-
async mouseMove(x: number, y: number): Promise<void> {
|
|
1339
|
+
async mouseMove(x: number, y: number, gestureId?: string, modifiers?: number): Promise<void> {
|
|
528
1340
|
this.assertPoint(x, y);
|
|
529
|
-
|
|
1341
|
+
this.assertMouseModifiers(modifiers);
|
|
1342
|
+
await this.dispatchInput(
|
|
530
1343
|
"Input.dispatchMouseEvent",
|
|
531
1344
|
{
|
|
532
1345
|
type: "mouseMoved",
|
|
533
1346
|
x,
|
|
534
1347
|
y,
|
|
535
|
-
button
|
|
1348
|
+
// Native scrollbar dragging needs the held button as well as its mask.
|
|
1349
|
+
button: this.heldMoveButton(),
|
|
536
1350
|
buttons: this.buttonMask(),
|
|
537
1351
|
},
|
|
538
|
-
|
|
1352
|
+
gestureId,
|
|
1353
|
+
modifiers,
|
|
1354
|
+
() => {
|
|
1355
|
+
this.lastMousePoint = { x, y };
|
|
1356
|
+
},
|
|
1357
|
+
);
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
/** Fixed native mouse departure; never maps an outside point to a page control. */
|
|
1361
|
+
async mouseLeave(gestureId: string): Promise<void> {
|
|
1362
|
+
if (this.heldButtons.size > 0) throw new Error("Release held mouse buttons before leaving");
|
|
1363
|
+
await this.dispatchInput(
|
|
1364
|
+
"Input.dispatchMouseEvent",
|
|
1365
|
+
{
|
|
1366
|
+
type: "mouseMoved",
|
|
1367
|
+
x: -1,
|
|
1368
|
+
y: -1,
|
|
1369
|
+
button: "none",
|
|
1370
|
+
buttons: 0,
|
|
1371
|
+
},
|
|
1372
|
+
gestureId,
|
|
539
1373
|
);
|
|
540
1374
|
}
|
|
541
1375
|
|
|
@@ -544,9 +1378,12 @@ export class AgentBrowserRuntime {
|
|
|
544
1378
|
y: number,
|
|
545
1379
|
button: MouseButton = "left",
|
|
546
1380
|
clickCount = 1,
|
|
1381
|
+
gestureId?: string,
|
|
1382
|
+
modifiers?: number,
|
|
547
1383
|
): Promise<void> {
|
|
548
1384
|
this.assertPoint(x, y);
|
|
549
|
-
|
|
1385
|
+
this.assertMouseModifiers(modifiers);
|
|
1386
|
+
await this.dispatchInput(
|
|
550
1387
|
"Input.dispatchMouseEvent",
|
|
551
1388
|
{
|
|
552
1389
|
type: "mousePressed",
|
|
@@ -556,14 +1393,28 @@ export class AgentBrowserRuntime {
|
|
|
556
1393
|
buttons: this.buttonMask(button),
|
|
557
1394
|
clickCount,
|
|
558
1395
|
},
|
|
559
|
-
|
|
1396
|
+
gestureId,
|
|
1397
|
+
modifiers,
|
|
1398
|
+
() => {
|
|
1399
|
+
// Publication may succeed before an acknowledgement is lost, so a
|
|
1400
|
+
// possibly held button is tracked from the send boundary.
|
|
1401
|
+
this.heldButtons.add(button);
|
|
1402
|
+
this.lastMousePoint = { x, y };
|
|
1403
|
+
},
|
|
560
1404
|
);
|
|
561
|
-
this.heldButtons.add(button);
|
|
562
1405
|
}
|
|
563
1406
|
|
|
564
|
-
async mouseUp(
|
|
1407
|
+
async mouseUp(
|
|
1408
|
+
x: number,
|
|
1409
|
+
y: number,
|
|
1410
|
+
button: MouseButton = "left",
|
|
1411
|
+
clickCount = 1,
|
|
1412
|
+
gestureId?: string,
|
|
1413
|
+
modifiers?: number,
|
|
1414
|
+
): Promise<void> {
|
|
565
1415
|
this.assertPoint(x, y);
|
|
566
|
-
|
|
1416
|
+
this.assertMouseModifiers(modifiers);
|
|
1417
|
+
await this.dispatchInput(
|
|
567
1418
|
"Input.dispatchMouseEvent",
|
|
568
1419
|
{
|
|
569
1420
|
type: "mouseReleased",
|
|
@@ -573,14 +1424,26 @@ export class AgentBrowserRuntime {
|
|
|
573
1424
|
buttons: this.buttonMask(undefined, button),
|
|
574
1425
|
clickCount,
|
|
575
1426
|
},
|
|
576
|
-
|
|
1427
|
+
gestureId,
|
|
1428
|
+
modifiers,
|
|
1429
|
+
() => {
|
|
1430
|
+
this.lastMousePoint = { x, y };
|
|
1431
|
+
},
|
|
577
1432
|
);
|
|
578
1433
|
this.heldButtons.delete(button);
|
|
579
1434
|
}
|
|
580
1435
|
|
|
581
|
-
async wheel(
|
|
1436
|
+
async wheel(
|
|
1437
|
+
x: number,
|
|
1438
|
+
y: number,
|
|
1439
|
+
deltaX: number,
|
|
1440
|
+
deltaY: number,
|
|
1441
|
+
gestureId?: string,
|
|
1442
|
+
modifiers?: number,
|
|
1443
|
+
): Promise<void> {
|
|
582
1444
|
this.assertPoint(x, y);
|
|
583
|
-
|
|
1445
|
+
this.assertMouseModifiers(modifiers);
|
|
1446
|
+
await this.dispatchInput(
|
|
584
1447
|
"Input.dispatchMouseEvent",
|
|
585
1448
|
{
|
|
586
1449
|
type: "mouseWheel",
|
|
@@ -591,16 +1454,46 @@ export class AgentBrowserRuntime {
|
|
|
591
1454
|
button: "none",
|
|
592
1455
|
buttons: this.buttonMask(),
|
|
593
1456
|
},
|
|
594
|
-
|
|
1457
|
+
gestureId,
|
|
1458
|
+
modifiers,
|
|
595
1459
|
);
|
|
596
1460
|
}
|
|
597
1461
|
|
|
598
|
-
|
|
599
|
-
|
|
1462
|
+
/** Insert a committed paste/IME value on the original guarded attachment. */
|
|
1463
|
+
async insertText(text: string, gestureId?: string): Promise<void> {
|
|
1464
|
+
await this.dispatchInput("Input.insertText", { text }, gestureId);
|
|
1465
|
+
}
|
|
1466
|
+
|
|
1467
|
+
/** Track cleanup intent before publication, including an uncertain key-down reply. */
|
|
1468
|
+
async dispatchKey(event: BrowserGestureKeyEvent, gestureId: string): Promise<void> {
|
|
1469
|
+
const parsed = browserGestureKeySchema.parse(event);
|
|
1470
|
+
await this.assertLiveInput(gestureId);
|
|
1471
|
+
const held = this.heldKeys.has(parsed.code);
|
|
1472
|
+
if (parsed.type === "down" && held !== parsed.repeat) {
|
|
1473
|
+
throw new Error(
|
|
1474
|
+
parsed.repeat ? "Key repeat has no matching press" : "Key is already pressed",
|
|
1475
|
+
);
|
|
1476
|
+
}
|
|
1477
|
+
if (parsed.type === "up" && !held) throw new Error("Key release has no matching press");
|
|
1478
|
+
if (parsed.type === "down") this.assertHeldKeyCapacity(parsed.code);
|
|
1479
|
+
await this.dispatchInput(
|
|
1480
|
+
"Input.dispatchKeyEvent",
|
|
1481
|
+
nativeKeyEvent(parsed),
|
|
1482
|
+
gestureId,
|
|
1483
|
+
undefined,
|
|
1484
|
+
parsed.type === "down"
|
|
1485
|
+
? () => this.heldKeys.set(parsed.code, { key: parsed.key, code: parsed.code })
|
|
1486
|
+
: undefined,
|
|
1487
|
+
);
|
|
1488
|
+
if (parsed.type === "up") this.heldKeys.delete(parsed.code);
|
|
600
1489
|
}
|
|
601
1490
|
|
|
602
|
-
|
|
603
|
-
|
|
1491
|
+
/** Track intent before publication. Internal discrete presses can use the
|
|
1492
|
+
* same opaque channel as live input so a lost ACK retains original-page cleanup. */
|
|
1493
|
+
async keyDown(key: string, code = key, gestureId?: string): Promise<void> {
|
|
1494
|
+
if (gestureId) await this.assertLiveInput(gestureId);
|
|
1495
|
+
this.assertHeldKeyCapacity(code);
|
|
1496
|
+
await this.dispatchInput(
|
|
604
1497
|
"Input.dispatchKeyEvent",
|
|
605
1498
|
{
|
|
606
1499
|
type: "keyDown",
|
|
@@ -608,58 +1501,215 @@ export class AgentBrowserRuntime {
|
|
|
608
1501
|
code,
|
|
609
1502
|
text: key.length === 1 ? key : undefined,
|
|
610
1503
|
},
|
|
611
|
-
|
|
1504
|
+
gestureId,
|
|
1505
|
+
undefined,
|
|
1506
|
+
() => this.heldKeys.set(code, { key, code }),
|
|
612
1507
|
);
|
|
613
|
-
this.heldKeys.add(key);
|
|
614
1508
|
}
|
|
615
1509
|
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
);
|
|
622
|
-
this.heldKeys.delete(key);
|
|
1510
|
+
/** Release on a current channel; its end cleanup still owns the original
|
|
1511
|
+
* page if navigation or attachment loss prevents this ordinary key-up. */
|
|
1512
|
+
async keyUp(key: string, code = key, gestureId?: string): Promise<void> {
|
|
1513
|
+
await this.dispatchInput("Input.dispatchKeyEvent", { type: "keyUp", key, code }, gestureId);
|
|
1514
|
+
this.heldKeys.delete(code);
|
|
623
1515
|
}
|
|
624
1516
|
|
|
1517
|
+
/**
|
|
1518
|
+
* The caller supplies the complete active contact set. CDP touchMove does not
|
|
1519
|
+
* release omitted contacts, so publish explicit contact ends before moving the
|
|
1520
|
+
* survivors. Any uncertain publication retains cleanup intent until cancel.
|
|
1521
|
+
*/
|
|
625
1522
|
async touch(
|
|
626
1523
|
type: "touchStart" | "touchMove" | "touchEnd" | "touchCancel",
|
|
627
1524
|
points: Array<{ x: number; y: number; id?: number }>,
|
|
1525
|
+
gestureId?: string,
|
|
628
1526
|
): Promise<void> {
|
|
629
1527
|
for (const point of points) this.assertPoint(point.x, point.y);
|
|
630
|
-
|
|
1528
|
+
if (gestureId) await this.assertLiveInput(gestureId);
|
|
1529
|
+
const contacts = points.map((point, index) => ({ ...point, id: point.id ?? index }));
|
|
1530
|
+
const activeIds = new Set(contacts.map((point) => point.id));
|
|
1531
|
+
const removed =
|
|
1532
|
+
type === "touchMove"
|
|
1533
|
+
? [...this.activeTouches.values()].filter((point) => !activeIds.has(point.id))
|
|
1534
|
+
: [];
|
|
1535
|
+
const hold = () => {
|
|
1536
|
+
this.heldTouches = true;
|
|
1537
|
+
};
|
|
1538
|
+
if (removed.length > 0) {
|
|
1539
|
+
await this.dispatchInput(
|
|
1540
|
+
"Input.dispatchTouchEvent",
|
|
1541
|
+
{ type: "touchEnd", touchPoints: removed },
|
|
1542
|
+
gestureId,
|
|
1543
|
+
undefined,
|
|
1544
|
+
hold,
|
|
1545
|
+
);
|
|
1546
|
+
}
|
|
1547
|
+
await this.dispatchInput(
|
|
631
1548
|
"Input.dispatchTouchEvent",
|
|
632
|
-
{
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
1549
|
+
{ type, touchPoints: contacts },
|
|
1550
|
+
gestureId,
|
|
1551
|
+
undefined,
|
|
1552
|
+
hold,
|
|
1553
|
+
);
|
|
1554
|
+
// Navigation cleanup already discarded this channel's contacts. Do not
|
|
1555
|
+
// recreate held-touch state after its acknowledged start caused navigation.
|
|
1556
|
+
if (gestureId && this.liveInput?.id !== gestureId) return;
|
|
1557
|
+
this.activeTouches.clear();
|
|
1558
|
+
if (type === "touchStart" || type === "touchMove") {
|
|
1559
|
+
for (const point of contacts) this.activeTouches.set(point.id, point);
|
|
1560
|
+
}
|
|
1561
|
+
this.heldTouches = this.activeTouches.size > 0;
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
/**
|
|
1565
|
+
* Bind decoded geometry to its exact native attachment and document. Compare
|
|
1566
|
+
* after cleanup/reattach awaits so a same-URL reload cannot silently admit a
|
|
1567
|
+
* press against a replacement page. A mismatch publishes no physical input.
|
|
1568
|
+
*/
|
|
1569
|
+
async beginLiveInput(id: string, expectedInputGeneration: string): Promise<void> {
|
|
1570
|
+
if (this.liveInput) await this.endLiveInput(this.liveInput.id);
|
|
1571
|
+
const page = await this.requirePage();
|
|
1572
|
+
const actualInputGeneration = formatRuntimeInputGeneration(
|
|
1573
|
+
this.attachmentGeneration,
|
|
1574
|
+
this.documentGeneration,
|
|
641
1575
|
);
|
|
1576
|
+
if (expectedInputGeneration !== actualInputGeneration) {
|
|
1577
|
+
throw new CdpUnavailableError("Live browser input attachment changed before admission");
|
|
1578
|
+
}
|
|
1579
|
+
// No asynchronous boundary may separate this comparison from the binding.
|
|
1580
|
+
const now = Date.now();
|
|
1581
|
+
this.liveInput = {
|
|
1582
|
+
id,
|
|
1583
|
+
page,
|
|
1584
|
+
generation: this.attachmentGeneration,
|
|
1585
|
+
documentGeneration: this.documentGeneration,
|
|
1586
|
+
idleUntil: now + GESTURE_IDLE_MS,
|
|
1587
|
+
expiresAt: now + GESTURE_LIFETIME_MS,
|
|
1588
|
+
timer: null,
|
|
1589
|
+
};
|
|
1590
|
+
this.scheduleLiveInputExpiry();
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
/** Compare after every asynchronous transport boundary, including automatic reconnect/reattach. */
|
|
1594
|
+
async assertLiveInput(id: string): Promise<void> {
|
|
1595
|
+
const page = await this.requirePage();
|
|
1596
|
+
if (
|
|
1597
|
+
this.liveInput?.id === id &&
|
|
1598
|
+
(Date.now() >= this.liveInput.idleUntil || Date.now() >= this.liveInput.expiresAt)
|
|
1599
|
+
) {
|
|
1600
|
+
await this.endLiveInput(id);
|
|
1601
|
+
}
|
|
1602
|
+
if (
|
|
1603
|
+
!this.liveInput ||
|
|
1604
|
+
this.liveInput.id !== id ||
|
|
1605
|
+
this.liveInput.page !== page ||
|
|
1606
|
+
this.liveInput.generation !== this.attachmentGeneration ||
|
|
1607
|
+
this.liveInput.documentGeneration !== this.documentGeneration
|
|
1608
|
+
) {
|
|
1609
|
+
throw new CdpUnavailableError("Live browser input attachment changed");
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1612
|
+
|
|
1613
|
+
/** Cleanup is idempotent and targets the original attachment, never whatever is currently active. */
|
|
1614
|
+
async endLiveInput(id: string): Promise<void> {
|
|
1615
|
+
const current = this.liveInput;
|
|
1616
|
+
if (!current || current.id !== id) return;
|
|
1617
|
+
this.liveInput = null;
|
|
1618
|
+
if (current.timer) clearTimeout(current.timer);
|
|
1619
|
+
await this.releaseHeldInput(current.page);
|
|
642
1620
|
}
|
|
643
1621
|
|
|
644
|
-
|
|
645
|
-
|
|
1622
|
+
/** Only acknowledged input extends idle time; metadata/checks cannot keep a stranded press alive. */
|
|
1623
|
+
private renewLiveInput(id: string): void {
|
|
1624
|
+
if (this.liveInput?.id !== id) return;
|
|
1625
|
+
this.liveInput.idleUntil = Date.now() + GESTURE_IDLE_MS;
|
|
1626
|
+
this.scheduleLiveInputExpiry();
|
|
1627
|
+
}
|
|
1628
|
+
|
|
1629
|
+
/** Runtime-owned expiry survives plugin disconnect or a fenced supervisor IPC client. */
|
|
1630
|
+
private scheduleLiveInputExpiry(): void {
|
|
1631
|
+
const current = this.liveInput;
|
|
1632
|
+
if (!current) return;
|
|
1633
|
+
if (current.timer) clearTimeout(current.timer);
|
|
1634
|
+
const delay = Math.max(0, Math.min(current.idleUntil, current.expiresAt) - Date.now());
|
|
1635
|
+
current.timer = setTimeout(() => {
|
|
1636
|
+
if (this.liveInput !== current) return;
|
|
1637
|
+
void this.endLiveInput(current.id).catch(() => undefined);
|
|
1638
|
+
}, delay);
|
|
1639
|
+
current.timer.unref?.();
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1642
|
+
/** Fixed hit-test, safe cursor names only. Cursor failures cannot fail an acknowledged input. */
|
|
1643
|
+
async cursorAt(x: number, y: number, gestureId: string): Promise<BrowserCursor | null> {
|
|
1644
|
+
try {
|
|
1645
|
+
this.assertPoint(x, y);
|
|
1646
|
+
await this.assertLiveInput(gestureId);
|
|
1647
|
+
const page = this.liveInput!.page;
|
|
1648
|
+
const result = await page.send<{ result: { value?: unknown } }>(
|
|
1649
|
+
"Runtime.evaluate",
|
|
1650
|
+
{
|
|
1651
|
+
expression: browserCursorExpression(x, y),
|
|
1652
|
+
returnByValue: true,
|
|
1653
|
+
},
|
|
1654
|
+
{ timeoutMs: Math.min(this.timeoutMs, CURSOR_SAMPLE_TIMEOUT_MS) },
|
|
1655
|
+
);
|
|
1656
|
+
await this.assertLiveInput(gestureId);
|
|
1657
|
+
const parsed = browserCursorSchema.safeParse(result.result.value);
|
|
1658
|
+
return parsed.success ? parsed.data : null;
|
|
1659
|
+
} catch {
|
|
1660
|
+
return null;
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
private holdsInput(): boolean {
|
|
1665
|
+
return this.heldButtons.size > 0 || this.heldKeys.size > 0 || this.heldTouches;
|
|
1666
|
+
}
|
|
1667
|
+
|
|
1668
|
+
/** Forget held state whose attachment is gone; it is never replayed on a replacement. */
|
|
1669
|
+
private discardHeldState(): void {
|
|
1670
|
+
this.heldButtons.clear();
|
|
1671
|
+
this.heldKeys.clear();
|
|
1672
|
+
this.heldTouches = false;
|
|
1673
|
+
this.activeTouches.clear();
|
|
1674
|
+
this.heldOwner = null;
|
|
1675
|
+
}
|
|
1676
|
+
|
|
1677
|
+
/**
|
|
1678
|
+
* Release only what `page` itself holds. State published on another attachment
|
|
1679
|
+
* is left to that attachment's own cleanup, never retargeted here.
|
|
1680
|
+
*/
|
|
1681
|
+
async releaseHeldInput(page = this.page): Promise<void> {
|
|
646
1682
|
if (!page) {
|
|
647
|
-
this.
|
|
648
|
-
this.heldKeys.clear();
|
|
1683
|
+
this.discardHeldState();
|
|
649
1684
|
return;
|
|
650
1685
|
}
|
|
1686
|
+
if (this.heldOwner && this.heldOwner !== page) return;
|
|
1687
|
+
this.heldOwner = null;
|
|
651
1688
|
const buttons = [...this.heldButtons];
|
|
652
|
-
const
|
|
1689
|
+
const { x, y } = this.lastMousePoint;
|
|
1690
|
+
const keys = [...this.heldKeys.values()];
|
|
653
1691
|
this.heldButtons.clear();
|
|
654
1692
|
this.heldKeys.clear();
|
|
1693
|
+
const touchHeld = this.heldTouches;
|
|
1694
|
+
this.heldTouches = false;
|
|
1695
|
+
this.activeTouches.clear();
|
|
655
1696
|
await Promise.allSettled([
|
|
1697
|
+
...(touchHeld
|
|
1698
|
+
? [
|
|
1699
|
+
page.send(
|
|
1700
|
+
"Input.dispatchTouchEvent",
|
|
1701
|
+
{ type: "touchCancel", touchPoints: [] },
|
|
1702
|
+
{ mutation: true },
|
|
1703
|
+
),
|
|
1704
|
+
]
|
|
1705
|
+
: []),
|
|
656
1706
|
...buttons.map((button) =>
|
|
657
1707
|
page.send(
|
|
658
1708
|
"Input.dispatchMouseEvent",
|
|
659
1709
|
{
|
|
660
1710
|
type: "mouseReleased",
|
|
661
|
-
x
|
|
662
|
-
y
|
|
1711
|
+
x,
|
|
1712
|
+
y,
|
|
663
1713
|
button,
|
|
664
1714
|
buttons: 0,
|
|
665
1715
|
clickCount: 1,
|
|
@@ -667,13 +1717,13 @@ export class AgentBrowserRuntime {
|
|
|
667
1717
|
{ mutation: true },
|
|
668
1718
|
),
|
|
669
1719
|
),
|
|
670
|
-
...keys.map((key) =>
|
|
1720
|
+
...keys.map(({ key, code }) =>
|
|
671
1721
|
page.send(
|
|
672
1722
|
"Input.dispatchKeyEvent",
|
|
673
1723
|
{
|
|
674
1724
|
type: "keyUp",
|
|
675
1725
|
key,
|
|
676
|
-
code
|
|
1726
|
+
code,
|
|
677
1727
|
},
|
|
678
1728
|
{ mutation: true },
|
|
679
1729
|
),
|
|
@@ -681,35 +1731,112 @@ export class AgentBrowserRuntime {
|
|
|
681
1731
|
]);
|
|
682
1732
|
}
|
|
683
1733
|
|
|
1734
|
+
/**
|
|
1735
|
+
* Concurrent callers share one in-flight shutdown; a failed one is not remembered as
|
|
1736
|
+
* success, so a later call retries and reports the same unconfirmed state.
|
|
1737
|
+
*/
|
|
684
1738
|
async shutdown(force = false): Promise<void> {
|
|
685
|
-
if (this.
|
|
1739
|
+
if (this.shutDown) return;
|
|
1740
|
+
if (this.shutdownInFlight) return this.shutdownInFlight;
|
|
1741
|
+
const operation = this.performShutdown(force);
|
|
1742
|
+
this.shutdownInFlight = operation;
|
|
1743
|
+
try {
|
|
1744
|
+
await operation;
|
|
1745
|
+
this.shutDown = true;
|
|
1746
|
+
} finally {
|
|
1747
|
+
if (this.shutdownInFlight === operation) this.shutdownInFlight = null;
|
|
1748
|
+
}
|
|
1749
|
+
}
|
|
1750
|
+
|
|
1751
|
+
private async performShutdown(force: boolean): Promise<void> {
|
|
686
1752
|
this.stopping = true;
|
|
687
|
-
|
|
1753
|
+
this.jpegDemand.close();
|
|
1754
|
+
this.screencastActive = false;
|
|
688
1755
|
this.invalidateScreencastFrame();
|
|
689
|
-
|
|
1756
|
+
await this.withVideoTransition(async () => {
|
|
1757
|
+
if (this.liveInput) await this.endLiveInput(this.liveInput.id);
|
|
1758
|
+
await this.releaseHeldInput();
|
|
1759
|
+
this.invalidateScreencastFrame();
|
|
1760
|
+
// Only the controller that launched the browser owns its session daemon.
|
|
1761
|
+
if (this.ownsBrowser) await this.stopDaemon(force);
|
|
1762
|
+
this.connection?.close();
|
|
1763
|
+
this.connection = null;
|
|
1764
|
+
this.page = null;
|
|
1765
|
+
this.targetId = null;
|
|
1766
|
+
this.invalidateScreencastFrame();
|
|
1767
|
+
});
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1770
|
+
/** Close, then require observed exit; escalate to a verified-owned SIGKILL and observe that too. */
|
|
1771
|
+
private async stopDaemon(force: boolean): Promise<void> {
|
|
1772
|
+
let forced = force;
|
|
1773
|
+
if (!forced) {
|
|
690
1774
|
try {
|
|
691
1775
|
await this.invoke(["--session", this.session, "--json", "close"]);
|
|
692
1776
|
} catch {
|
|
693
|
-
|
|
694
|
-
await this.shutdown(true);
|
|
695
|
-
return;
|
|
1777
|
+
forced = true;
|
|
696
1778
|
}
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
if (
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
1779
|
+
// close returns before the session daemon exits. A same-session relaunch would reach
|
|
1780
|
+
// that daemon with its original launch environment (e.g. a stopped private display).
|
|
1781
|
+
if (!forced && !(await this.daemonExited())) forced = true;
|
|
1782
|
+
}
|
|
1783
|
+
if (forced) await this.terminateDaemon();
|
|
1784
|
+
if (forced || this.daemonLaunched) await this.removeIpcMetadata();
|
|
1785
|
+
this.daemonLaunched = false;
|
|
1786
|
+
this.daemon = null;
|
|
1787
|
+
}
|
|
1788
|
+
|
|
1789
|
+
/**
|
|
1790
|
+
* Records the PID-file daemon. Only a PID whose environment carries this runtime's own
|
|
1791
|
+
* nonce gets a verified identity; anything else is observable but never signalled.
|
|
1792
|
+
*/
|
|
1793
|
+
private async captureDaemon(): Promise<void> {
|
|
1794
|
+
const pid = await this.daemonPid();
|
|
1795
|
+
if (pid === null) return;
|
|
1796
|
+
const probe = await probeProcess(pid, this.ownerNonce);
|
|
1797
|
+
this.daemon = { pid, identity: probe.status === "owned" ? probe.identity : null };
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
/**
|
|
1801
|
+
* Observes the daemon this runtime launched. Only `gone` is a positive exit; `unknown`
|
|
1802
|
+
* (no PID record, unreadable identity) is never success and never authorises a signal.
|
|
1803
|
+
*/
|
|
1804
|
+
private async daemonState(): Promise<"gone" | "owned" | "alive" | "unknown"> {
|
|
1805
|
+
if (!this.daemonLaunched) return "gone";
|
|
1806
|
+
if (!this.daemon) {
|
|
1807
|
+
await this.captureDaemon();
|
|
1808
|
+
if (!this.daemon) return "unknown";
|
|
1809
|
+
}
|
|
1810
|
+
const { pid, identity } = this.daemon;
|
|
1811
|
+
if (!identity) return processExists(pid) ? "alive" : "gone";
|
|
1812
|
+
const probe = await probeProcess(pid, this.ownerNonce);
|
|
1813
|
+
if (probe.status === "absent" || probe.status === "foreign") return "gone";
|
|
1814
|
+
if (probe.status === "owned")
|
|
1815
|
+
return probe.identity.startTicks === identity.startTicks ? "owned" : "gone";
|
|
1816
|
+
return processExists(pid) ? "unknown" : "gone";
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
/** True only after positive observation of exit, within the bound. */
|
|
1820
|
+
private async daemonExited(): Promise<boolean> {
|
|
1821
|
+
for (const deadline = Date.now() + this.daemonExitWaitMs; Date.now() < deadline; ) {
|
|
1822
|
+
if ((await this.daemonState()) === "gone") return true;
|
|
1823
|
+
await new Promise((resolve) => setTimeout(resolve, SCREENCAST_SETUP_RETRY_MS));
|
|
1824
|
+
}
|
|
1825
|
+
return (await this.daemonState()) === "gone";
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
/** Signals only a daemon freshly re-verified as owned, then confirms its exit. */
|
|
1829
|
+
private async terminateDaemon(): Promise<void> {
|
|
1830
|
+
if ((await this.daemonState()) === "owned" && this.daemon) {
|
|
1831
|
+
try {
|
|
1832
|
+
process.kill(this.daemon.pid, "SIGKILL");
|
|
1833
|
+
} catch (error) {
|
|
1834
|
+
if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
|
|
705
1835
|
}
|
|
706
|
-
await this.removeIpcMetadata();
|
|
707
1836
|
}
|
|
708
|
-
|
|
709
|
-
this.
|
|
710
|
-
|
|
711
|
-
this.targetId = null;
|
|
712
|
-
this.invalidateScreencastFrame();
|
|
1837
|
+
// Unverified or unknown daemons are never signalled, but exit must still be observed.
|
|
1838
|
+
if (!(await this.daemonExited()))
|
|
1839
|
+
throw new AgentBrowserUnavailableError("agent-browser daemon exit was not confirmed");
|
|
713
1840
|
}
|
|
714
1841
|
|
|
715
1842
|
private async assertVersion(): Promise<void> {
|
|
@@ -759,8 +1886,18 @@ export class AgentBrowserRuntime {
|
|
|
759
1886
|
}
|
|
760
1887
|
}
|
|
761
1888
|
|
|
762
|
-
private async connectCdp(
|
|
1889
|
+
private async connectCdp(
|
|
1890
|
+
preferredTargetId: string | null = null,
|
|
1891
|
+
activate = this.ownsBrowser,
|
|
1892
|
+
): Promise<void> {
|
|
1893
|
+
this.assertRuntimeOpen();
|
|
1894
|
+
// A launch-selected page remains the intended target even if its first
|
|
1895
|
+
// attachment fails. Otherwise a retry could silently adopt another tab.
|
|
1896
|
+
if (this.targetId === null && preferredTargetId !== null) {
|
|
1897
|
+
this.targetId = preferredTargetId;
|
|
1898
|
+
}
|
|
763
1899
|
const response = await this.invoke(["--session", this.session, "--json", "get", "cdp-url"]);
|
|
1900
|
+
this.assertRuntimeOpen();
|
|
764
1901
|
const url = findString(response, ["cdpUrl", "cdp_url", "url"]);
|
|
765
1902
|
if (!url) throw new AgentBrowserIncompatibleError("agent-browser did not return a CDP URL");
|
|
766
1903
|
const endpoint = new URL(url);
|
|
@@ -771,77 +1908,102 @@ export class AgentBrowserRuntime {
|
|
|
771
1908
|
commandTimeoutMs: this.timeoutMs,
|
|
772
1909
|
connectTimeoutMs: this.timeoutMs,
|
|
773
1910
|
});
|
|
1911
|
+
if (this.stopping) {
|
|
1912
|
+
connection.close();
|
|
1913
|
+
this.assertRuntimeOpen();
|
|
1914
|
+
}
|
|
774
1915
|
this.connection = connection;
|
|
775
1916
|
connection.once("disconnect", () => {
|
|
776
1917
|
if (this.connection !== connection) return;
|
|
777
1918
|
this.page = null;
|
|
778
|
-
|
|
1919
|
+
// Keep the selected page identity. A transport loss does not authorize
|
|
1920
|
+
// adopting another tab, even when the original page still exists.
|
|
1921
|
+
this.emulationAppliedPage = null;
|
|
1922
|
+
this.attachmentGeneration += 1;
|
|
779
1923
|
this.invalidateScreencastFrame();
|
|
780
1924
|
});
|
|
781
1925
|
const targets = await listPageTargets(this.connection);
|
|
1926
|
+
this.assertRuntimeOpen();
|
|
1927
|
+
const expectedTargetId = preferredTargetId ?? this.targetId;
|
|
782
1928
|
const target =
|
|
783
|
-
|
|
1929
|
+
expectedTargetId === null
|
|
1930
|
+
? targets[0]
|
|
1931
|
+
: targets.find((candidate) => candidate.targetId === expectedTargetId);
|
|
1932
|
+
if (expectedTargetId !== null && !target) {
|
|
1933
|
+
throw new CdpUnavailableError("The selected browser page is no longer available");
|
|
1934
|
+
}
|
|
784
1935
|
if (!target) throw new CdpUnavailableError("Chromium has no page target");
|
|
785
|
-
await this.selectTarget(target.targetId);
|
|
1936
|
+
await this.selectTarget(target.targetId, activate);
|
|
786
1937
|
}
|
|
787
1938
|
|
|
788
1939
|
private async requirePage(): Promise<CdpSession> {
|
|
789
|
-
|
|
1940
|
+
this.assertRuntimeOpen();
|
|
1941
|
+
if (this.page && this.connection?.isOpen) {
|
|
1942
|
+
const page = this.page;
|
|
1943
|
+
await this.restoreConfiguredEmulation(page);
|
|
1944
|
+
this.assertRuntimeOpen();
|
|
1945
|
+
if (this.page !== page || !this.connection?.isOpen) {
|
|
1946
|
+
throw new CdpUnavailableError("Browser page changed during emulation restoration");
|
|
1947
|
+
}
|
|
1948
|
+
return page;
|
|
1949
|
+
}
|
|
790
1950
|
await this.reconnect();
|
|
1951
|
+
this.assertRuntimeOpen();
|
|
791
1952
|
if (!this.page) throw new CdpUnavailableError("No page target is attached");
|
|
792
1953
|
return this.page;
|
|
793
1954
|
}
|
|
794
1955
|
|
|
795
1956
|
private requireConnection(): CdpConnection {
|
|
1957
|
+
this.assertRuntimeOpen();
|
|
796
1958
|
if (!this.connection?.isOpen) throw new CdpUnavailableError("Browser runtime is disconnected");
|
|
797
1959
|
return this.connection;
|
|
798
1960
|
}
|
|
799
1961
|
|
|
1962
|
+
/** Shutdown cleanup uses captured attachments directly; acquisition cannot revive them. */
|
|
1963
|
+
private assertRuntimeOpen(): void {
|
|
1964
|
+
if (this.stopping) throw new CdpUnavailableError("Browser runtime is shutting down");
|
|
1965
|
+
}
|
|
1966
|
+
|
|
800
1967
|
private requireViewport(): BrowserViewport {
|
|
801
1968
|
if (!this.viewport) throw new AgentBrowserIncompatibleError("Viewport has not been configured");
|
|
802
1969
|
return this.viewport;
|
|
803
1970
|
}
|
|
804
1971
|
|
|
805
|
-
private async
|
|
806
|
-
|
|
1972
|
+
private async bindPageEvents(page: CdpSession): Promise<void> {
|
|
1973
|
+
// URL equality is insufficient for reload. Main-document and same-document
|
|
1974
|
+
// navigation revoke the channel without treating child-frame loads as a reset.
|
|
1975
|
+
let rootFrameId: string | undefined;
|
|
1976
|
+
page.on("Page.frameNavigated", (event: { frame: { id: string; parentId?: string } }) => {
|
|
1977
|
+
if (page !== this.page || event.frame.parentId) return;
|
|
1978
|
+
rootFrameId = event.frame.id;
|
|
1979
|
+
this.documentGeneration += 1;
|
|
1980
|
+
this.invalidateScreencastFrame();
|
|
1981
|
+
this.video?.invalidateQueuedFrames();
|
|
1982
|
+
void this.endLiveInput(this.liveInput?.id ?? "").catch(() => undefined);
|
|
1983
|
+
});
|
|
1984
|
+
page.on("Page.navigatedWithinDocument", (event: { frameId: string }) => {
|
|
1985
|
+
if (page !== this.page || event.frameId !== rootFrameId) return;
|
|
1986
|
+
this.documentGeneration += 1;
|
|
1987
|
+
this.invalidateScreencastFrame();
|
|
1988
|
+
this.video?.invalidateQueuedFrames();
|
|
1989
|
+
void this.endLiveInput(this.liveInput?.id ?? "").catch(() => undefined);
|
|
1990
|
+
});
|
|
1991
|
+
page.on("Page.screencastFrame", (event: ScreencastFrame) =>
|
|
1992
|
+
this.onScreencastFrame(page, event),
|
|
1993
|
+
);
|
|
807
1994
|
page.on("event", (event: CdpEvent) => {
|
|
808
|
-
if (event.method === "Inspector.targetCrashed"
|
|
1995
|
+
if (event.method === "Inspector.targetCrashed" && this.page === page) {
|
|
1996
|
+
this.documentGeneration += 1;
|
|
1997
|
+
void this.stopVideo();
|
|
1998
|
+
void this.endLiveInput(this.liveInput?.id ?? "").catch(() => undefined);
|
|
1999
|
+
this.emulationAppliedPage = null;
|
|
2000
|
+
this.invalidateScreencastFrame();
|
|
2001
|
+
}
|
|
809
2002
|
});
|
|
810
2003
|
await Promise.all([page.send("Page.enable"), page.send("Runtime.enable")]);
|
|
2004
|
+
const tree = await page.send<{ frameTree?: { frame: { id: string } } }>("Page.getFrameTree");
|
|
2005
|
+
rootFrameId = tree.frameTree?.frame.id ?? rootFrameId;
|
|
811
2006
|
await this.navigationHistory(page);
|
|
812
|
-
// Emulation is scoped to the CDP session, so every fresh attach starts without it.
|
|
813
|
-
if (this.viewport) await this.applyEmulation(page, this.viewport);
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
private async applyEmulation(page: CdpSession, device: DeviceEmulation): Promise<void> {
|
|
817
|
-
await page.send(
|
|
818
|
-
"Emulation.setDeviceMetricsOverride",
|
|
819
|
-
{
|
|
820
|
-
width: device.width,
|
|
821
|
-
height: device.height,
|
|
822
|
-
deviceScaleFactor: device.deviceScaleFactor,
|
|
823
|
-
mobile: device.mobile,
|
|
824
|
-
screenWidth: device.screenWidth ?? device.width,
|
|
825
|
-
screenHeight: device.screenHeight ?? device.height,
|
|
826
|
-
screenOrientation: {
|
|
827
|
-
type: device.width > device.height ? "landscapePrimary" : "portraitPrimary",
|
|
828
|
-
angle: device.width > device.height ? 90 : 0,
|
|
829
|
-
},
|
|
830
|
-
},
|
|
831
|
-
{ mutation: true },
|
|
832
|
-
);
|
|
833
|
-
await page.send(
|
|
834
|
-
"Emulation.setTouchEmulationEnabled",
|
|
835
|
-
{ enabled: device.touch, maxTouchPoints: device.touch ? 5 : 1 },
|
|
836
|
-
{ mutation: true },
|
|
837
|
-
);
|
|
838
|
-
if (device.userAgent) {
|
|
839
|
-
await page.send(
|
|
840
|
-
"Emulation.setUserAgentOverride",
|
|
841
|
-
{ userAgent: device.userAgent, platform: device.platform ?? "" },
|
|
842
|
-
{ mutation: true },
|
|
843
|
-
);
|
|
844
|
-
}
|
|
845
2007
|
}
|
|
846
2008
|
|
|
847
2009
|
private async startScreencastSession(page: CdpSession): Promise<void> {
|
|
@@ -849,23 +2011,34 @@ export class AgentBrowserRuntime {
|
|
|
849
2011
|
let candidate = page;
|
|
850
2012
|
for (;;) {
|
|
851
2013
|
try {
|
|
2014
|
+
this.assertRuntimeOpen();
|
|
852
2015
|
await Promise.all([candidate.send("Page.enable"), candidate.send("Runtime.enable")]);
|
|
2016
|
+
this.assertRuntimeOpen();
|
|
853
2017
|
await this.navigationHistory(candidate);
|
|
2018
|
+
this.assertRuntimeOpen();
|
|
854
2019
|
if (this.page !== candidate) {
|
|
855
2020
|
candidate = await this.requirePage();
|
|
856
2021
|
continue;
|
|
857
2022
|
}
|
|
2023
|
+
await this.restoreConfiguredEmulation(candidate);
|
|
2024
|
+
this.assertRuntimeOpen();
|
|
2025
|
+
if (this.page !== candidate) {
|
|
2026
|
+
throw new CdpUnavailableError("Browser page changed before screencast start");
|
|
2027
|
+
}
|
|
858
2028
|
await candidate.send(
|
|
859
2029
|
"Page.startScreencast",
|
|
860
2030
|
{ format: "jpeg", quality: this.screencastQuality, everyNthFrame: 1 },
|
|
861
2031
|
{ mutation: true },
|
|
862
2032
|
);
|
|
2033
|
+
this.assertRuntimeOpen();
|
|
863
2034
|
return;
|
|
864
2035
|
} catch (error) {
|
|
2036
|
+
this.assertRuntimeOpen();
|
|
865
2037
|
if (Date.now() >= deadline) throw error;
|
|
866
2038
|
try {
|
|
867
2039
|
candidate = await this.reattachPageForScreencast(candidate);
|
|
868
2040
|
} catch (reattachError) {
|
|
2041
|
+
this.assertRuntimeOpen();
|
|
869
2042
|
if (Date.now() >= deadline) throw reattachError;
|
|
870
2043
|
}
|
|
871
2044
|
const { promise, resolve } = Promise.withResolvers<void>();
|
|
@@ -876,66 +2049,140 @@ export class AgentBrowserRuntime {
|
|
|
876
2049
|
}
|
|
877
2050
|
|
|
878
2051
|
private async reattachPageForScreencast(previous: CdpSession): Promise<CdpSession> {
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
await connection.send(
|
|
884
|
-
"Target.activateTarget",
|
|
885
|
-
{ targetId: target.targetId },
|
|
886
|
-
{ mutation: true },
|
|
887
|
-
);
|
|
888
|
-
const replacement = await attachToTarget(connection, target.targetId);
|
|
889
|
-
await this.preparePage(replacement);
|
|
890
|
-
if (this.page === previous) {
|
|
891
|
-
this.page = replacement;
|
|
892
|
-
this.targetId = target.targetId;
|
|
893
|
-
} else {
|
|
894
|
-
await replacement.detach().catch(() => undefined);
|
|
2052
|
+
this.assertRuntimeOpen();
|
|
2053
|
+
if (this.page !== previous) {
|
|
2054
|
+
if (!this.page) throw new CdpUnavailableError("Browser page changed during reattachment");
|
|
2055
|
+
return this.page;
|
|
895
2056
|
}
|
|
896
|
-
|
|
897
|
-
|
|
2057
|
+
return this.withVideoTransition(async () => {
|
|
2058
|
+
const attachmentGeneration = ++this.attachmentGeneration;
|
|
2059
|
+
const connection = this.requireConnection();
|
|
2060
|
+
const targets = await listPageTargets(connection);
|
|
2061
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
2062
|
+
const target = targets.find((candidate) => candidate.targetId === previous.targetId);
|
|
2063
|
+
if (!target)
|
|
2064
|
+
throw new CdpUnavailableError("The selected browser page is no longer available");
|
|
2065
|
+
await connection.send(
|
|
2066
|
+
"Target.activateTarget",
|
|
2067
|
+
{ targetId: target.targetId },
|
|
2068
|
+
{ mutation: true },
|
|
2069
|
+
);
|
|
2070
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
2071
|
+
const replacement = await attachToTarget(connection, target.targetId);
|
|
2072
|
+
if (this.page !== previous || attachmentGeneration !== this.attachmentGeneration) {
|
|
2073
|
+
await replacement.detach().catch(() => undefined);
|
|
2074
|
+
// The losing controller's detach may clear target overrides. Repair the
|
|
2075
|
+
// current controller on its next access rather than trust earlier pixels.
|
|
2076
|
+
this.emulationAppliedPage = null;
|
|
2077
|
+
this.invalidateScreencastFrame();
|
|
2078
|
+
if (!this.page) throw new CdpUnavailableError("Browser page changed during reattachment");
|
|
2079
|
+
return this.page;
|
|
2080
|
+
}
|
|
2081
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
2082
|
+
|
|
2083
|
+
this.invalidateScreencastFrame();
|
|
2084
|
+
// Detaching drops the session that holds any pressed button; release there first.
|
|
2085
|
+
if (this.liveInput) await this.endLiveInput(this.liveInput.id);
|
|
2086
|
+
await this.releaseHeldInput(previous);
|
|
2087
|
+
this.page = null;
|
|
2088
|
+
this.emulationAppliedPage = null;
|
|
2089
|
+
// Detach first: the former session can otherwise clear the newly applied override.
|
|
2090
|
+
try {
|
|
2091
|
+
await previous.detach();
|
|
2092
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
2093
|
+
await this.bindPageEvents(replacement);
|
|
2094
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
2095
|
+
await this.restoreConfiguredEmulation(replacement);
|
|
2096
|
+
this.assertAttachmentCurrent(connection, attachmentGeneration);
|
|
2097
|
+
this.page = replacement;
|
|
2098
|
+
this.targetId = target.targetId;
|
|
2099
|
+
return replacement;
|
|
2100
|
+
} catch (error) {
|
|
2101
|
+
await replacement.detach().catch(() => undefined);
|
|
2102
|
+
// Detaching a partial emulation can clear target metrics even if a newer
|
|
2103
|
+
// attachment won. Its next access must restore intent before admitting pixels.
|
|
2104
|
+
this.emulationAppliedPage = null;
|
|
2105
|
+
this.invalidateScreencastFrame();
|
|
2106
|
+
throw error;
|
|
2107
|
+
}
|
|
2108
|
+
});
|
|
898
2109
|
}
|
|
899
2110
|
|
|
900
2111
|
private async withInvalidatedScreencast(
|
|
901
2112
|
mutation: (page: CdpSession) => Promise<unknown>,
|
|
2113
|
+
options: { preserveStream?: boolean } = {},
|
|
902
2114
|
): Promise<void> {
|
|
903
|
-
const
|
|
904
|
-
const restart = this.screencastActive;
|
|
905
|
-
this.invalidateScreencastFrame();
|
|
906
|
-
if (restart) await page.send("Page.stopScreencast", {}, { mutation: true });
|
|
2115
|
+
const releaseJpeg = this.jpegDemand.pin(false);
|
|
907
2116
|
try {
|
|
908
|
-
await
|
|
909
|
-
|
|
2117
|
+
const page = await this.requirePage();
|
|
2118
|
+
// Ordinary navigation is observed by the same native stream. Optional
|
|
2119
|
+
// capture teardown/setup must not consume the navigation ACK budget.
|
|
2120
|
+
const restart = this.screencastActive && !options.preserveStream;
|
|
910
2121
|
this.invalidateScreencastFrame();
|
|
911
|
-
if (restart
|
|
2122
|
+
if (restart) await page.send("Page.stopScreencast", {}, { mutation: true });
|
|
2123
|
+
this.assertRuntimeOpen();
|
|
2124
|
+
try {
|
|
2125
|
+
await mutation(page);
|
|
2126
|
+
this.invalidateQueuedVideoFrames();
|
|
2127
|
+
} finally {
|
|
2128
|
+
this.invalidateScreencastFrame();
|
|
2129
|
+
if (restart && this.page === page) {
|
|
2130
|
+
this.assertRuntimeOpen();
|
|
2131
|
+
await this.restoreConfiguredEmulation(page);
|
|
2132
|
+
await this.startScreencastSession(page);
|
|
2133
|
+
}
|
|
2134
|
+
}
|
|
2135
|
+
} finally {
|
|
2136
|
+
releaseJpeg();
|
|
912
2137
|
}
|
|
913
2138
|
}
|
|
914
2139
|
|
|
2140
|
+
/** Clear JPEG stream/screenshot receipts after input or transport changes. */
|
|
915
2141
|
private invalidateScreencastFrame(): void {
|
|
2142
|
+
this.capturePolicy.invalidate();
|
|
2143
|
+
this.screencastSourceTime.invalidate(Date.now());
|
|
916
2144
|
this.screencastFrame = null;
|
|
2145
|
+
this.screencastFrameReceivedAt = null;
|
|
917
2146
|
this.resolveFrameWaiters(null);
|
|
918
2147
|
}
|
|
919
2148
|
|
|
920
|
-
private
|
|
921
|
-
|
|
922
|
-
|
|
2149
|
+
private onScreencastFrame(page: CdpSession, event: ScreencastFrame): void {
|
|
2150
|
+
// A late frame belongs to its emitting session, even after page reattachment.
|
|
2151
|
+
void page
|
|
2152
|
+
.send("Page.screencastFrameAck", { sessionId: event.sessionId })
|
|
923
2153
|
.catch(() => undefined);
|
|
924
|
-
|
|
925
|
-
|
|
2154
|
+
if (
|
|
2155
|
+
this.stopping ||
|
|
2156
|
+
page !== this.page ||
|
|
2157
|
+
!this.screencastActive ||
|
|
2158
|
+
this.emulationAppliedPage !== page
|
|
2159
|
+
)
|
|
2160
|
+
return;
|
|
2161
|
+
const dimensions = readJpegFrameDimensions(event.data);
|
|
926
2162
|
const viewport = this.viewport;
|
|
927
|
-
|
|
2163
|
+
// Chromium can report the new device dimensions while emitting a clipped transition image.
|
|
2164
|
+
// Reject those pixels and use the bounded screenshot fallback rather than mislabeling them.
|
|
2165
|
+
if (!viewport || !dimensions) return;
|
|
2166
|
+
const expectedPixels = captureDimensions(viewport, viewport.captureScale);
|
|
2167
|
+
if (dimensions.width !== expectedPixels.width || dimensions.height !== expectedPixels.height)
|
|
2168
|
+
return;
|
|
2169
|
+
const now = performance.now();
|
|
2170
|
+
const source = this.screencastSourceTime.accept(event.metadata.timestamp, Date.now(), now);
|
|
2171
|
+
if (!source) return;
|
|
2172
|
+
const { width, height } = dimensions;
|
|
928
2173
|
const frame: RuntimeFrame = {
|
|
929
2174
|
dataBase64: event.data,
|
|
930
2175
|
byteLength: Buffer.byteLength(event.data, "base64"),
|
|
931
2176
|
width,
|
|
932
2177
|
height,
|
|
933
2178
|
transport: "cdp-screencast",
|
|
934
|
-
capturedAt:
|
|
2179
|
+
capturedAt: source.capturedAt,
|
|
935
2180
|
};
|
|
2181
|
+
this.capturePolicy.observeStream(`provider:${event.metadata.timestamp}`, now);
|
|
936
2182
|
this.screencastFrame = frame;
|
|
2183
|
+
this.screencastFrameReceivedAt = source.receivedAt;
|
|
937
2184
|
this.resolveFrameWaiters(frame);
|
|
938
|
-
}
|
|
2185
|
+
}
|
|
939
2186
|
|
|
940
2187
|
private waitForFrame(waitMs: number): Promise<RuntimeFrame | null> {
|
|
941
2188
|
const { promise, resolve } = Promise.withResolvers<RuntimeFrame | null>();
|
|
@@ -958,6 +2205,17 @@ export class AgentBrowserRuntime {
|
|
|
958
2205
|
}
|
|
959
2206
|
|
|
960
2207
|
private assertViewport(viewport: BrowserViewport): void {
|
|
2208
|
+
const captureScale = viewport.captureScale ?? 1;
|
|
2209
|
+
const pixels = captureDimensions(viewport, captureScale);
|
|
2210
|
+
if (
|
|
2211
|
+
!Number.isFinite(captureScale) ||
|
|
2212
|
+
captureScale < 1 ||
|
|
2213
|
+
captureScale > 2 ||
|
|
2214
|
+
pixels.width > MAX_VIEWPORT.width ||
|
|
2215
|
+
pixels.height > MAX_VIEWPORT.height
|
|
2216
|
+
) {
|
|
2217
|
+
throw new RangeError("Capture density exceeds the supported image bounds");
|
|
2218
|
+
}
|
|
961
2219
|
if (
|
|
962
2220
|
!Number.isInteger(viewport.width) ||
|
|
963
2221
|
viewport.width < 1 ||
|
|
@@ -986,6 +2244,35 @@ export class AgentBrowserRuntime {
|
|
|
986
2244
|
}
|
|
987
2245
|
}
|
|
988
2246
|
|
|
2247
|
+
/** CDP/Puppeteer priority for supported held buttons, independent of press order.
|
|
2248
|
+
* https://github.com/puppeteer/puppeteer/blob/main/packages/puppeteer-core/src/cdp/Input.ts
|
|
2249
|
+
*/
|
|
2250
|
+
private heldMoveButton(): MouseButton | "none" {
|
|
2251
|
+
for (const button of ["left", "right", "middle"] as const) {
|
|
2252
|
+
if (this.heldButtons.has(button)) {
|
|
2253
|
+
return button;
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
2256
|
+
return "none";
|
|
2257
|
+
}
|
|
2258
|
+
|
|
2259
|
+
/** Validate before recording held-button intent or sending a native packet. */
|
|
2260
|
+
private assertMouseModifiers(modifiers: number | undefined): void {
|
|
2261
|
+
if (
|
|
2262
|
+
modifiers !== undefined &&
|
|
2263
|
+
(!Number.isInteger(modifiers) || modifiers < 0 || modifiers > 15)
|
|
2264
|
+
) {
|
|
2265
|
+
throw new RangeError("Mouse modifiers must be an integer from 0 to 15");
|
|
2266
|
+
}
|
|
2267
|
+
}
|
|
2268
|
+
|
|
2269
|
+
/** Both live and discrete keyboard paths must bound release fanout before sending. */
|
|
2270
|
+
private assertHeldKeyCapacity(code: string): void {
|
|
2271
|
+
if (!this.heldKeys.has(code) && this.heldKeys.size >= MAX_HELD_BROWSER_KEYS) {
|
|
2272
|
+
throw new Error("Too many simultaneously held browser keys");
|
|
2273
|
+
}
|
|
2274
|
+
}
|
|
2275
|
+
|
|
989
2276
|
private buttonMask(add?: MouseButton, remove?: MouseButton): number {
|
|
990
2277
|
const active = new Set(this.heldButtons);
|
|
991
2278
|
if (add) active.add(add);
|