@omercnet/paseo-shared-browser 1.2.1 → 1.2.2-next.237.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/README.md +312 -25
  2. package/REMOTE_BROWSER_RESEARCH.md +395 -0
  3. package/STREAMING_DESIGN.md +191 -0
  4. package/client/browser-auto-capture-density.test.ts +102 -0
  5. package/client/browser-auto-capture-density.ts +30 -0
  6. package/client/browser-canvas-input.test.ts +198 -0
  7. package/client/browser-canvas-input.ts +202 -0
  8. package/client/browser-canvas-keyboard-web.test.ts +113 -0
  9. package/client/browser-canvas-keyboard-web.ts +230 -0
  10. package/client/browser-canvas-keyboard.test.ts +123 -0
  11. package/client/browser-canvas-keyboard.ts +126 -0
  12. package/client/browser-canvas-layout.test.ts +109 -0
  13. package/client/browser-canvas-layout.ts +51 -0
  14. package/client/browser-canvas-viewport-web.test.ts +77 -0
  15. package/client/browser-canvas-viewport-web.ts +63 -0
  16. package/client/browser-canvas-viewport.tsx +152 -0
  17. package/client/browser-capture-cadence.test.ts +9 -0
  18. package/client/browser-capture-cadence.ts +6 -0
  19. package/client/browser-capture-density-controls.tsx +75 -0
  20. package/client/browser-chrome.tsx +234 -0
  21. package/client/browser-client-presence.ts +184 -0
  22. package/client/browser-compose-controls.test.tsx +275 -0
  23. package/client/browser-compose-controls.tsx +174 -0
  24. package/client/browser-control-button.tsx +109 -0
  25. package/client/browser-control-dialog.tsx +71 -0
  26. package/client/browser-control-request.test.ts +27 -0
  27. package/client/browser-control-request.ts +57 -0
  28. package/client/browser-control-tooltip-web.ts +10 -0
  29. package/client/browser-dialog-body.tsx +22 -0
  30. package/client/browser-emulation-mode.test.ts +75 -0
  31. package/client/browser-emulation-mode.ts +98 -0
  32. package/client/browser-fill-viewport.test.ts +45 -0
  33. package/client/browser-fill-viewport.ts +28 -0
  34. package/client/browser-frame-buffer.test.ts +157 -0
  35. package/client/browser-frame-buffer.ts +148 -0
  36. package/client/browser-frame-image.tsx +62 -0
  37. package/client/browser-image-load.test.ts +23 -0
  38. package/client/browser-image-load.ts +26 -0
  39. package/client/browser-input-queue.test.ts +1201 -0
  40. package/client/browser-input-queue.ts +505 -0
  41. package/client/browser-menu-placement.test.ts +90 -0
  42. package/client/browser-menu-placement.ts +78 -0
  43. package/client/browser-panel-styles.ts +384 -0
  44. package/client/browser-quality-controls.test.ts +26 -0
  45. package/client/browser-quality-controls.tsx +139 -0
  46. package/client/browser-resolution-picker.tsx +189 -0
  47. package/client/browser-tabs.tsx +117 -0
  48. package/client/browser-toolbar-menu-web.test.ts +185 -0
  49. package/client/browser-toolbar-menu-web.ts +159 -0
  50. package/client/browser-toolbar-menu.tsx +255 -0
  51. package/client/browser-toolbar-submenu.tsx +93 -0
  52. package/client/browser-video-bytes.test.ts +44 -0
  53. package/client/browser-video-decoder.test.ts +473 -0
  54. package/client/browser-video-decoder.ts +311 -0
  55. package/client/browser-video-surface.tsx +21 -0
  56. package/client/browser-video-visibility.test.ts +45 -0
  57. package/client/browser-viewer-recovery.test.ts +111 -0
  58. package/client/browser-viewer-recovery.ts +78 -0
  59. package/client/browser.tsx +1642 -1569
  60. package/client/favorite-star.tsx +23 -0
  61. package/client/frame-lifecycle.ts +62 -59
  62. package/client/use-browser-auto-capture-density.test.ts +104 -0
  63. package/client/use-browser-auto-capture-density.ts +43 -0
  64. package/client/use-browser-canvas-input.ts +370 -0
  65. package/client/use-browser-capture-density.ts +87 -0
  66. package/client/use-browser-emulation-mode.ts +56 -0
  67. package/client/use-browser-fill-viewport.test.ts +108 -0
  68. package/client/use-browser-fill-viewport.ts +62 -0
  69. package/client/use-browser-frame-buffer.ts +66 -0
  70. package/client/use-browser-image-capture.ts +139 -0
  71. package/client/use-browser-scoped-mutation.test.ts +118 -0
  72. package/client/use-browser-scoped-mutation.ts +59 -0
  73. package/client/use-browser-video.test.ts +991 -0
  74. package/client/use-browser-video.ts +474 -0
  75. package/client/use-browser-viewer-recovery.ts +26 -0
  76. package/client/use-resolution-favorites.ts +90 -0
  77. package/client/web.test.ts +125 -0
  78. package/client/web.ts +405 -0
  79. package/index.client.tsx +0 -10
  80. package/index.server.ts +32 -0
  81. package/npm-shrinkwrap.json +1834 -0
  82. package/package.json +13 -8
  83. package/paseo-plugin.json +3 -1
  84. package/scripts/prepare-dependencies.mjs +1 -1
  85. package/scripts/prepare-runtime.mjs +1 -1
  86. package/server/agent-browser-runtime.ts +1598 -314
  87. package/server/browser-cursor.test.ts +64 -0
  88. package/server/browser-cursor.ts +69 -0
  89. package/server/browser-frame-context.ts +22 -0
  90. package/server/browser-frame-epoch.test.ts +95 -0
  91. package/server/browser-frame-rejection.ts +17 -0
  92. package/server/browser-gesture.ts +120 -0
  93. package/server/browser-operation-diagnostics.test.ts +79 -0
  94. package/server/browser-operation-diagnostics.ts +103 -0
  95. package/server/browser-policy.ts +1313 -151
  96. package/server/browser-video-input-authority.test.ts +208 -0
  97. package/server/browser-video-policy.test.ts +564 -0
  98. package/server/browser-video-socket.test.ts +368 -0
  99. package/server/browser.ts +151 -16
  100. package/server/capture-transport-policy.test.ts +105 -0
  101. package/server/capture-transport-policy.ts +154 -0
  102. package/server/capture-transport-runtime.test.ts +213 -0
  103. package/server/daemon-launch-chain.test.ts +70 -0
  104. package/server/daemon-shutdown.test.ts +289 -0
  105. package/server/discrete-input-freshness.test.ts +364 -0
  106. package/server/discrete-input-native-lifetime.test.ts +150 -0
  107. package/server/display-loss-recovery.test.ts +294 -0
  108. package/server/emulation-restoration.test.ts +262 -0
  109. package/server/input-cleanup.ts +23 -0
  110. package/server/input-generation.test.ts +31 -0
  111. package/server/input-generation.ts +36 -0
  112. package/server/jpeg-capture-demand.test.ts +76 -0
  113. package/server/jpeg-capture-demand.ts +98 -0
  114. package/server/jpeg-frame.ts +29 -0
  115. package/server/keyboard-input.test.ts +61 -0
  116. package/server/keyboard-input.ts +104 -0
  117. package/server/live-input-policy.test.ts +958 -0
  118. package/server/live-input-runtime.test.ts +732 -0
  119. package/server/live-input-supervisor.test.ts +147 -0
  120. package/server/mcp-entry.ts +44 -2
  121. package/server/native-video-backpressure.test.ts +228 -0
  122. package/server/native-video-buffer.test.ts +141 -0
  123. package/server/native-video-buffer.ts +114 -0
  124. package/server/native-video-capture.test.ts +599 -0
  125. package/server/native-video-capture.ts +497 -0
  126. package/server/native-video-extension-source.test.ts +201 -0
  127. package/server/native-video-extension-source.ts +364 -0
  128. package/server/native-video-extension.test.ts +41 -0
  129. package/server/native-video-extension.ts +83 -0
  130. package/server/native-video-packet.ts +9 -0
  131. package/server/native-video-settings.test.ts +245 -0
  132. package/server/navigation-budget.ts +8 -0
  133. package/server/navigation-commit.ts +125 -0
  134. package/server/navigation-policy.test.ts +153 -0
  135. package/server/process-identity.ts +58 -0
  136. package/server/reattach-held-input.test.ts +143 -0
  137. package/server/runtime-frame-reuse.test.ts +295 -0
  138. package/server/runtime-jpeg-demand.test.ts +200 -0
  139. package/server/runtime-jpeg-shutdown.test.ts +210 -0
  140. package/server/runtime-jpeg-source-order.test.ts +201 -0
  141. package/server/runtime-launch-environment.test.ts +111 -0
  142. package/server/runtime-navigation.test.ts +327 -0
  143. package/server/runtime-owner-display.test.ts +152 -0
  144. package/server/runtime-owner.ts +305 -31
  145. package/server/runtime-page-state.test.ts +249 -0
  146. package/server/runtime-protocol.ts +31 -1
  147. package/server/runtime-target-retention.test.ts +181 -0
  148. package/server/runtime-video-opt-in.test.ts +82 -0
  149. package/server/runtime-video.test.ts +444 -0
  150. package/server/screencast-source-time.ts +46 -0
  151. package/server/stale-wheel-admission.test.ts +136 -0
  152. package/server/supervisor-client.ts +28 -4
  153. package/server/supervisor-heartbeat.test.ts +80 -0
  154. package/server/supervisor.ts +553 -55
  155. package/server/video-source-recovery.test.ts +20 -0
  156. package/server/video-source-recovery.ts +64 -0
  157. package/server/viewer-capture-lifetime.ts +51 -0
  158. package/server/virtual-display.test.ts +184 -0
  159. package/server/virtual-display.ts +236 -0
  160. package/shared/browser-display-preferences.ts +46 -0
  161. package/shared/browser-video.ts +80 -0
  162. package/shared/browser.ts +286 -65
  163. package/shared/capture-density.ts +16 -0
  164. package/shared/capture-settings.ts +17 -0
  165. package/shared/device-presets.ts +90 -0
  166. package/shared/resolution-menu.test.ts +120 -0
  167. package/shared/resolution-menu.ts +72 -0
  168. package/shared/video-settings.test.ts +38 -0
  169. package/shared/video-settings.ts +34 -0
  170. package/shared/viewport-limits.ts +5 -0
@@ -1,7 +1,12 @@
1
1
  import { execFile } from "node:child_process";
2
+ import { randomUUID } from "node:crypto";
2
3
  import { chmod, mkdir, readFile, rm } from "node:fs/promises";
3
4
  import { isAbsolute, resolve } from "node:path";
4
5
  import { promisify } from "node:util";
6
+ import { canUseCaptureDensity, captureDensitySchema } from "../shared/capture-density";
7
+
8
+ import { captureDimensions, DEFAULT_JPEG_QUALITY } from "../shared/capture-settings";
9
+ import { MAX_VIEWPORT } from "../shared/viewport-limits";
5
10
  import {
6
11
  attachToTarget,
7
12
  CdpConnection,
@@ -12,13 +17,56 @@ import {
12
17
  CdpUnknownOutcomeError,
13
18
  listPageTargets,
14
19
  } from "./cdp";
20
+ import { readJpegFrameDimensions } from "./jpeg-frame";
21
+
22
+ export { readJpegFrameDimensions as jpegDimensions } from "./jpeg-frame";
23
+
24
+ import {
25
+ type BrowserCursor,
26
+ type BrowserGestureKeyEvent,
27
+ browserCursorSchema,
28
+ browserGestureKeySchema,
29
+ MAX_HELD_BROWSER_KEYS,
30
+ } from "../shared/browser";
31
+ import { resolveVideoEncoderSettings } from "../shared/video-settings";
32
+ import { browserCursorExpression } from "./browser-cursor";
33
+ import { GESTURE_IDLE_MS, GESTURE_LIFETIME_MS } from "./browser-gesture";
34
+ import { CAPTURE_MAX_AGE_MS, createCaptureTransportPolicy } from "./capture-transport-policy";
35
+ import { formatRuntimeInputGeneration } from "./input-generation";
36
+ import { createJpegCaptureDemand } from "./jpeg-capture-demand";
37
+ import { heldKeyModifiers, nativeKeyEvent } from "./keyboard-input";
38
+ import {
39
+ NativeVideoCapture,
40
+ type NativeVideoQuality,
41
+ type NativeVideoRead,
42
+ } from "./native-video-capture";
43
+ import { NATIVE_VIDEO_EXTENSION_ID, prepareNativeVideoExtension } from "./native-video-extension";
44
+ import {
45
+ NAVIGATION_ACK_TIMEOUT_MS,
46
+ NAVIGATION_COMMIT_TIMEOUT_MS,
47
+ NAVIGATION_METADATA_TIMEOUT_MS,
48
+ } from "./navigation-budget";
49
+ import { waitForNavigationCommit } from "./navigation-commit";
50
+ import {
51
+ type ProcessIdentity,
52
+ probeProcess,
53
+ processExists,
54
+ RUNTIME_OWNER_VARIABLE,
55
+ } from "./process-identity";
56
+ import { createScreencastSourceTime } from "./screencast-source-time";
57
+ import { createVideoSourceRecovery } from "./video-source-recovery";
15
58
 
16
59
  const execFileAsync = promisify(execFile);
17
- export const AGENT_BROWSER_VERSION = "0.37.1";
60
+ export const AGENT_BROWSER_VERSION = "0.38.2";
18
61
  const PRIVATE_DIRECTORY_MODE = 0o700;
19
62
  const PRIVATE_FILE_MODE = 0o600;
20
63
  const DEFAULT_TIMEOUT_MS = 15_000;
21
64
  const SCREENCAST_SETUP_RETRY_MS = 25;
65
+ /** Bound for the agent-browser session daemon to exit after close or SIGKILL. */
66
+ const DAEMON_EXIT_WAIT_MS = 5_000;
67
+ // Cursor decoration must not hold the serialized input/video fence while page
68
+ // scripts are busy. A missed sample leaves input and its cleanup unchanged.
69
+ const CURSOR_SAMPLE_TIMEOUT_MS = 250;
22
70
 
23
71
  export class AgentBrowserUnavailableError extends Error {
24
72
  override readonly name = "AgentBrowserUnavailableError";
@@ -34,6 +82,8 @@ export interface BrowserViewport {
34
82
  width: number;
35
83
  height: number;
36
84
  deviceScaleFactor: number;
85
+ /** Capture density does not change the emulated layout or pointer coordinate system. */
86
+ captureScale?: number;
37
87
  mobile: boolean;
38
88
  touch: boolean;
39
89
  userAgent?: string;
@@ -68,6 +118,8 @@ export interface RuntimePageState {
68
118
  title: string;
69
119
  canGoBack: boolean;
70
120
  canGoForward: boolean;
121
+ /** Process-local attachment/document revision, including same-URL reloads. */
122
+ inputGeneration: string;
71
123
  }
72
124
 
73
125
  export interface RuntimeFrame {
@@ -94,7 +146,13 @@ export interface AgentBrowserRuntimeOptions {
94
146
  session: string;
95
147
  initialUrl?: string;
96
148
  headed?: boolean;
149
+ /** Trusted per-instance private X11 context, only DISPLAY/XAUTHORITY; never mutates host environment. */
150
+ launchEnvironment?: Readonly<Record<string, string>>;
151
+ /** Trusted daemon-side opt-in. Only then is the capture extension materialized and loaded. */
152
+ nativeVideo?: boolean;
97
153
  timeoutMs?: number;
154
+ /** Bound for the session daemon to exit after close or SIGKILL. */
155
+ daemonExitWaitMs?: number;
98
156
  }
99
157
 
100
158
  interface VersionResult {
@@ -108,14 +166,10 @@ interface NavigationHistory {
108
166
  currentIndex: number;
109
167
  entries: Array<{ id: number; url: string; title: string }>;
110
168
  }
111
- interface PageLifecycleEvent {
112
- name: string;
113
- loaderId: string;
114
- }
115
169
 
116
170
  interface ScreencastFrame {
117
171
  data: string;
118
- metadata: { deviceWidth: number; deviceHeight: number };
172
+ metadata: { deviceWidth: number; deviceHeight: number; timestamp?: number };
119
173
  sessionId: number;
120
174
  }
121
175
 
@@ -124,7 +178,11 @@ function requireAbsolute(path: string, label: string): string {
124
178
  return resolve(path);
125
179
  }
126
180
 
127
- function runtimeEnvironment(ipcDirectory: string): NodeJS.ProcessEnv {
181
+ function runtimeEnvironment(
182
+ ipcDirectory: string,
183
+ launchEnvironment: Readonly<Record<string, string>> | undefined,
184
+ ownerNonce: string,
185
+ ): NodeJS.ProcessEnv {
128
186
  const environment: NodeJS.ProcessEnv = {};
129
187
  for (const key of [
130
188
  "PATH",
@@ -153,6 +211,27 @@ function runtimeEnvironment(ipcDirectory: string): NodeJS.ProcessEnv {
153
211
  const value = process.env[key];
154
212
  if (value !== undefined) environment[key] = value;
155
213
  }
214
+ if (launchEnvironment !== undefined) {
215
+ const keys = Object.keys(launchEnvironment);
216
+ const display = launchEnvironment.DISPLAY;
217
+ const authority = launchEnvironment.XAUTHORITY;
218
+ if (
219
+ keys.length !== 2 ||
220
+ keys.some((key) => key !== "DISPLAY" && key !== "XAUTHORITY") ||
221
+ typeof display !== "string" ||
222
+ !/^:[0-9]{1,5}(?:\.[0-9]{1,2})?$/.test(display) ||
223
+ typeof authority !== "string" ||
224
+ !isAbsolute(authority)
225
+ ) {
226
+ // Do not include display/authentication values in errors or child arguments.
227
+ throw new AgentBrowserIncompatibleError("Private browser display environment is invalid");
228
+ }
229
+ environment.DISPLAY = display;
230
+ environment.XAUTHORITY = authority;
231
+ // A private X11 display must not inherit an unrelated host Wayland session.
232
+ delete environment.WAYLAND_DISPLAY;
233
+ }
234
+ environment[RUNTIME_OWNER_VARIABLE] = ownerNonce;
156
235
  environment.AGENT_BROWSER_SOCKET_DIR = ipcDirectory;
157
236
  environment.AGENT_BROWSER_IDLE_TIMEOUT_MS = "0";
158
237
  environment.AGENT_BROWSER_STREAM_PORT = "0";
@@ -183,26 +262,6 @@ function findString(value: unknown, keys: readonly string[]): string | null {
183
262
  return null;
184
263
  }
185
264
 
186
- export function jpegDimensions(base64: string): { width: number; height: number } | null {
187
- const bytes = Buffer.from(base64, "base64");
188
- let offset = 2;
189
- while (bytes[0] === 0xff && bytes[1] === 0xd8 && offset + 9 <= bytes.length) {
190
- if (bytes[offset] !== 0xff) return null;
191
- const marker = bytes[offset + 1];
192
- // SOFn carries the frame size; DHT (c4), JPG (c8) and DAC (cc) share the range but do not.
193
- if (
194
- marker !== undefined &&
195
- marker >= 0xc0 &&
196
- marker <= 0xcf &&
197
- ![0xc4, 0xc8, 0xcc].includes(marker)
198
- )
199
- return { width: bytes.readUInt16BE(offset + 7), height: bytes.readUInt16BE(offset + 5) };
200
- if (marker === 0xda) return null;
201
- offset += 2 + bytes.readUInt16BE(offset + 2);
202
- }
203
- return null;
204
- }
205
-
206
265
  export class AgentBrowserRuntime {
207
266
  readonly binaryPath: string;
208
267
  readonly executablePath: string;
@@ -213,17 +272,58 @@ export class AgentBrowserRuntime {
213
272
  private readonly headed: boolean;
214
273
  private readonly timeoutMs: number;
215
274
  private readonly environment: NodeJS.ProcessEnv;
275
+ private daemon: { pid: number; identity: ProcessIdentity | null } | null = null;
276
+ private readonly ownerNonce = randomUUID();
277
+ private readonly daemonExitWaitMs: number;
216
278
  private connection: CdpConnection | null = null;
217
279
  private page: CdpSession | null = null;
218
280
  private targetId: string | null = null;
219
- private viewport: BrowserViewport | null = null;
281
+ private viewport: DeviceEmulation | null = null;
282
+ private emulationAppliedPage: CdpSession | null = null;
283
+ private attachmentGeneration = 0;
284
+ private readonly capturePolicy = createCaptureTransportPolicy();
285
+ private readonly screencastSourceTime = createScreencastSourceTime();
220
286
  private screencastFrame: RuntimeFrame | null = null;
287
+ private screencastFrameReceivedAt: number | null = null;
221
288
  private screencastWaiters = new Set<(frame: RuntimeFrame | null) => void>();
222
289
  private screencastActive = false;
223
- private screencastQuality = 65;
290
+ private screencastQuality: number = DEFAULT_JPEG_QUALITY;
291
+ private screencastStartFailed = false;
292
+ private readonly jpegDemand = createJpegCaptureDemand({
293
+ now: () => performance.now(),
294
+ retire: () => this.stopScreencastSession(),
295
+ onError: () => console.warn("Shared Browser could not retire unused JPEG capture"),
296
+ });
224
297
  private heldButtons = new Set<MouseButton>();
225
- private heldKeys = new Set<string>();
298
+ /** Last native pointer position sent, so cancelled drags release where they were. */
299
+ private lastMousePoint = { x: 0, y: 0 };
300
+ private heldKeys = new Map<string, { key: string; code: string }>();
301
+ private heldTouches = false;
302
+ /** Attachment on which the current held state was published; cleanup never targets another. */
303
+ private heldOwner: CdpSession | null = null;
304
+ private activeTouches = new Map<number, { x: number; y: number; id: number }>();
305
+ private documentGeneration = 0;
306
+ private liveInput: {
307
+ id: string;
308
+ page: CdpSession;
309
+ generation: number;
310
+ documentGeneration: number;
311
+ idleUntil: number;
312
+ expiresAt: number;
313
+ timer: ReturnType<typeof setTimeout> | null;
314
+ } | null = null;
226
315
  private stopping = false;
316
+ private shutdownInFlight: Promise<void> | null = null;
317
+ private shutDown = false;
318
+ /** A daemon was launched for this session, so absence of evidence is not proof of exit. */
319
+ private daemonLaunched = false;
320
+ private readonly nativeVideo: boolean;
321
+ private video: NativeVideoCapture | null = null;
322
+ private videoRetirement: Promise<void> | null = null;
323
+ private ownsBrowser = true;
324
+ private readonly videoRecovery = createVideoSourceRecovery();
325
+ private videoTransitionDepth = 0;
326
+ private videoTransitionEpoch = 0;
227
327
 
228
328
  constructor(options: AgentBrowserRuntimeOptions) {
229
329
  this.binaryPath = requireAbsolute(options.binaryPath, "agent-browser binary path");
@@ -236,22 +336,46 @@ export class AgentBrowserRuntime {
236
336
  this.session = options.session;
237
337
  this.headed = options.headed ?? false;
238
338
  this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
239
- this.environment = runtimeEnvironment(this.ipcDirectory);
339
+ this.daemonExitWaitMs = options.daemonExitWaitMs ?? DAEMON_EXIT_WAIT_MS;
340
+ this.environment = runtimeEnvironment(
341
+ this.ipcDirectory,
342
+ options.launchEnvironment,
343
+ this.ownerNonce,
344
+ );
240
345
  this.initialUrl = options.initialUrl ?? "about:blank";
346
+ this.nativeVideo = options.nativeVideo === true;
241
347
  }
242
348
 
243
349
  async launch(): Promise<void> {
350
+ this.assertRuntimeOpen();
244
351
  if (this.connection?.isOpen) return;
245
352
  this.invalidateScreencastFrame();
246
- this.stopping = false;
247
353
  await mkdir(this.profilePath, { recursive: true, mode: PRIVATE_DIRECTORY_MODE });
248
354
  await mkdir(this.ipcDirectory, { recursive: true, mode: PRIVATE_DIRECTORY_MODE });
249
355
  await Promise.all([
250
356
  chmod(this.profilePath, PRIVATE_DIRECTORY_MODE),
251
357
  chmod(this.ipcDirectory, PRIVATE_DIRECTORY_MODE),
252
358
  ]);
359
+ this.assertRuntimeOpen();
253
360
  await this.assertVersion();
254
- const chromiumArguments = process.env.PASEO_SHARED_BROWSER_CHROMIUM_ARGS;
361
+ this.assertRuntimeOpen();
362
+ // Image-only sessions never materialize, load or allowlist the capture
363
+ // extension, so its tabCapture/debugger permissions do not exist in them.
364
+ const videoExtension = this.nativeVideo
365
+ ? await prepareNativeVideoExtension(this.ipcDirectory)
366
+ : null;
367
+ this.assertRuntimeOpen();
368
+ const existingArguments = process.env.PASEO_SHARED_BROWSER_CHROMIUM_ARGS;
369
+ const chromiumArguments = [
370
+ existingArguments,
371
+ videoExtension ? `--allowlisted-extension-id=${NATIVE_VIDEO_EXTENSION_ID}` : undefined,
372
+ ]
373
+ .filter(Boolean)
374
+ .join(",");
375
+ this.ownsBrowser = true;
376
+ // From here a nonce-owned daemon may exist even if open rejects (e.g. navigation failure).
377
+ this.daemonLaunched = true;
378
+ this.daemon = null;
255
379
  const opened = await this.invoke([
256
380
  "--session",
257
381
  this.session,
@@ -259,25 +383,63 @@ export class AgentBrowserRuntime {
259
383
  this.profilePath,
260
384
  "--executable-path",
261
385
  this.executablePath,
386
+ ...(videoExtension ? ["--extension", videoExtension] : []),
387
+ "--hide-scrollbars",
388
+ "false",
262
389
  ...(this.headed ? ["--headed"] : []),
263
390
  ...(chromiumArguments ? ["--args", chromiumArguments] : []),
264
391
  "--json",
265
392
  "open",
266
393
  this.initialUrl,
267
394
  ]);
395
+ this.assertRuntimeOpen();
268
396
  const targetId = findString(opened, ["targetId", "target_id"]);
269
397
  await this.protectIpcMetadata();
398
+ await this.captureDaemon();
399
+ this.assertRuntimeOpen();
270
400
  await this.connectCdp(targetId);
271
401
  }
272
402
 
403
+ /** Attach an independent page controller to an existing browser session. */
404
+ async attachTarget(targetId: string): Promise<void> {
405
+ this.assertRuntimeOpen();
406
+ this.ownsBrowser = false;
407
+ await this.connectCdp(targetId, false);
408
+ }
409
+
410
+ /** Create a page in the existing profile without changing another controller's selected page. */
411
+ async createTarget(url = "about:blank"): Promise<string> {
412
+ const result = await this.requireConnection().send<{ targetId: string }>(
413
+ "Target.createTarget",
414
+ { url },
415
+ { mutation: true },
416
+ );
417
+ return result.targetId;
418
+ }
419
+
420
+ /** Close only the named page. The caller must keep another page in the browser. */
421
+ async closeTarget(targetId: string): Promise<void> {
422
+ const result = await this.requireConnection().send<{ success: boolean }>(
423
+ "Target.closeTarget",
424
+ { targetId },
425
+ { mutation: true },
426
+ );
427
+ if (!result.success) throw new CdpUnavailableError("Browser tab could not be closed");
428
+ }
429
+
273
430
  async reconnect(): Promise<void> {
274
- const targetId = this.targetId;
275
- this.invalidateScreencastFrame();
276
- this.connection?.close();
277
- this.connection = null;
278
- this.page = null;
279
- await this.assertVersion();
280
- await this.connectCdp(targetId);
431
+ this.assertRuntimeOpen();
432
+ await this.withVideoTransition(async () => {
433
+ const targetId = this.targetId;
434
+ this.attachmentGeneration += 1;
435
+ this.invalidateScreencastFrame();
436
+ this.connection?.close();
437
+ this.connection = null;
438
+ this.page = null;
439
+ await this.assertVersion();
440
+ this.assertRuntimeOpen();
441
+ await this.connectCdp(targetId);
442
+ });
281
443
  }
282
444
 
283
445
  async health(): Promise<RuntimeHealth> {
@@ -321,32 +483,73 @@ export class AgentBrowserRuntime {
321
483
  }
322
484
 
323
485
  async targets(): Promise<RuntimeTarget[]> {
324
- return listPageTargets(this.requireConnection());
486
+ return (await listPageTargets(this.requireConnection())).filter(
487
+ (target) => !target.url.startsWith(`chrome-extension://${NATIVE_VIDEO_EXTENSION_ID}/`),
488
+ );
325
489
  }
326
490
 
327
- async selectTarget(targetId: string): Promise<void> {
328
- const target = (await this.targets()).find((candidate) => candidate.targetId === targetId);
329
- if (!target) throw new CdpUnavailableError(`Unknown page target: ${targetId}`);
330
- this.invalidateScreencastFrame();
331
- if (this.page) {
332
- if (this.screencastActive) {
333
- await this.page.send("Page.stopScreencast", {}, { mutation: true });
491
+ async selectTarget(targetId: string, activate = true): Promise<void> {
492
+ await this.withVideoTransition(async () => {
493
+ const attachmentGeneration = ++this.attachmentGeneration;
494
+ const connection = this.requireConnection();
495
+ const target = (await this.targets()).find((candidate) => candidate.targetId === targetId);
496
+ this.assertAttachmentCurrent(connection, attachmentGeneration);
497
+ if (!target) throw new CdpUnavailableError(`Unknown page target: ${targetId}`);
498
+ this.invalidateScreencastFrame();
499
+ const previous = this.page;
500
+ if (previous) {
501
+ if (this.screencastActive) {
502
+ await previous.send("Page.stopScreencast", {}, { mutation: true });
503
+ this.assertAttachmentCurrent(connection, attachmentGeneration);
504
+ }
505
+ await previous.detach();
506
+ this.assertAttachmentCurrent(connection, attachmentGeneration);
334
507
  }
335
- await this.page.detach();
336
- }
337
- this.invalidateScreencastFrame();
338
- await this.requireConnection().send("Target.activateTarget", { targetId }, { mutation: true });
339
- this.page = await attachToTarget(this.requireConnection(), targetId);
340
- this.targetId = targetId;
341
- await this.preparePage(this.page);
342
- if (this.screencastActive) await this.startScreencastSession(this.page);
508
+ this.invalidateScreencastFrame();
509
+ if (activate) {
510
+ await connection.send("Target.activateTarget", { targetId }, { mutation: true });
511
+ this.assertAttachmentCurrent(connection, attachmentGeneration);
512
+ }
513
+ // CDP emulation is session-owned: detaching the former session restores native
514
+ // window metrics. Publish a replacement only after its retained settings succeed.
515
+ this.page = null;
516
+ this.emulationAppliedPage = null;
517
+ const replacement = await attachToTarget(connection, targetId);
518
+ try {
519
+ this.assertAttachmentCurrent(connection, attachmentGeneration);
520
+ await this.bindPageEvents(replacement);
521
+ this.assertAttachmentCurrent(connection, attachmentGeneration);
522
+ await this.restoreConfiguredEmulation(replacement);
523
+ this.assertAttachmentCurrent(connection, attachmentGeneration);
524
+ this.page = replacement;
525
+ this.targetId = targetId;
526
+ } catch (error) {
527
+ await replacement.detach().catch(() => undefined);
528
+ // Detaching a partial emulation can clear target metrics even if a newer
529
+ // attachment won. Its next access must restore intent before admitting pixels.
530
+ this.emulationAppliedPage = null;
531
+ this.invalidateScreencastFrame();
532
+ throw error;
533
+ }
534
+ if (this.screencastActive) await this.startScreencastSession(replacement);
535
+ });
343
536
  }
344
537
 
345
- private async navigationHistory(page: CdpSession): Promise<NavigationHistory> {
346
- const deadline = Date.now() + this.timeoutMs;
538
+ private async navigationHistory(
539
+ page: CdpSession,
540
+ timeoutMs = this.timeoutMs,
541
+ ): Promise<NavigationHistory> {
542
+ const deadline = Date.now() + timeoutMs;
347
543
  for (;;) {
544
+ const remaining = deadline - Date.now();
545
+ if (remaining <= 0)
546
+ throw new CdpUnavailableError("Browser navigation history observation timed out");
348
547
  try {
349
- return await page.send<NavigationHistory>("Page.getNavigationHistory");
548
+ return await page.send<NavigationHistory>(
549
+ "Page.getNavigationHistory",
550
+ {},
551
+ { timeoutMs: remaining },
552
+ );
350
553
  } catch (error) {
351
554
  if (!String(error).includes("Not attached to an active page") || Date.now() >= deadline) {
352
555
  throw error;
@@ -357,113 +560,524 @@ export class AgentBrowserRuntime {
357
560
  }
358
561
  }
359
562
  }
360
- async state(): Promise<RuntimePageState> {
563
+ /** Read fresh metadata concurrently, discarding both replies if the document changes.
564
+ * One same-attachment reread shares the original monotonic timeout budget in
565
+ * milliseconds. Failed reads, replacement attachments and repeated document
566
+ * changes reject; this read-only recovery never repeats a native mutation. */
567
+ async state(options: { timeoutMs?: number } = {}): Promise<RuntimePageState> {
568
+ if (
569
+ options.timeoutMs !== undefined &&
570
+ (!Number.isInteger(options.timeoutMs) ||
571
+ options.timeoutMs < 1 ||
572
+ options.timeoutMs > DEFAULT_TIMEOUT_MS)
573
+ ) {
574
+ throw new RangeError("Browser metadata timeout must be an integer from 1 to 15000ms");
575
+ }
576
+ const timeoutMs = Math.min(options.timeoutMs ?? this.timeoutMs, this.timeoutMs);
361
577
  const page = await this.requirePage();
362
- const history = await this.navigationHistory(page);
363
- const evaluated = await page.send<{
364
- result: { value?: { url?: string; title?: string } };
365
- }>("Runtime.evaluate", {
366
- expression: "({url: location.href, title: document.title})",
367
- returnByValue: true,
368
- });
369
- return {
370
- url: evaluated.result.value?.url ?? history.entries[history.currentIndex]?.url ?? "",
371
- title: evaluated.result.value?.title ?? history.entries[history.currentIndex]?.title ?? "",
372
- canGoBack: history.currentIndex > 0,
373
- canGoForward: history.currentIndex < history.entries.length - 1,
374
- };
578
+ const attachmentGeneration = this.attachmentGeneration;
579
+ const deadline = performance.now() + timeoutMs;
580
+
581
+ for (let attempt = 0; attempt < 2; attempt++) {
582
+ this.assertRuntimeOpen();
583
+ const remaining = attempt === 0 ? timeoutMs : deadline - performance.now();
584
+ if (remaining <= 0) {
585
+ throw new CdpUnavailableError("Browser metadata observation timed out");
586
+ }
587
+ const documentGeneration = this.documentGeneration;
588
+ // Both observations must describe the same document. A navigation discards
589
+ // the entire pair; neither native history nor DOM metadata is cached.
590
+ const [history, evaluated] = await Promise.all([
591
+ this.navigationHistory(page, remaining),
592
+ page.send<{
593
+ result: { value?: { url?: string; title?: string } };
594
+ }>(
595
+ "Runtime.evaluate",
596
+ {
597
+ expression: "({url: location.href, title: document.title})",
598
+ returnByValue: true,
599
+ },
600
+ { timeoutMs: remaining },
601
+ ),
602
+ ]);
603
+ this.assertRuntimeOpen();
604
+ if (page !== this.page || attachmentGeneration !== this.attachmentGeneration) {
605
+ throw new CdpUnavailableError("Browser metadata changed while being read");
606
+ }
607
+ // Successful replies can beat an overdue CDP timer in the event queue.
608
+ // Their metadata still cannot outlive the original shared phase budget.
609
+ if (performance.now() >= deadline) {
610
+ throw new CdpUnavailableError("Browser metadata observation timed out");
611
+ }
612
+ if (documentGeneration !== this.documentGeneration) {
613
+ if (attempt === 0) continue;
614
+ throw new CdpUnavailableError("Browser metadata changed while being read");
615
+ }
616
+ return {
617
+ url: evaluated.result.value?.url ?? history.entries[history.currentIndex]?.url ?? "",
618
+ title: evaluated.result.value?.title ?? history.entries[history.currentIndex]?.title ?? "",
619
+ canGoBack: history.currentIndex > 0,
620
+ canGoForward: history.currentIndex < history.entries.length - 1,
621
+ inputGeneration: formatRuntimeInputGeneration(attachmentGeneration, documentGeneration),
622
+ };
623
+ }
624
+ throw new CdpUnavailableError("Browser metadata changed while being read");
375
625
  }
376
626
 
627
+ /** Await native ACK and its main-frame commit, without waiting for scripts or
628
+ * DOMContentLoaded. A missing commit remains uncertain and is never replayed. */
377
629
  async navigate(url: string): Promise<void> {
378
- await this.withInvalidatedScreencast(async (page) => {
379
- await page.send("Page.setLifecycleEventsEnabled", { enabled: true });
380
- const observedLoaders = new Set<string>();
381
- const loaded = Promise.withResolvers<void>();
382
- let expectedLoaderId: string | undefined;
383
- const onLifecycle = (event: PageLifecycleEvent) => {
384
- if (event.name !== "DOMContentLoaded") return;
385
- if (expectedLoaderId === undefined) {
386
- observedLoaders.add(event.loaderId);
387
- return;
388
- }
389
- if (event.loaderId === expectedLoaderId) loaded.resolve();
390
- };
391
- page.on("Page.lifecycleEvent", onLifecycle);
392
- let timer: NodeJS.Timeout | undefined;
393
- try {
394
- const result = await page.send<{ errorText?: string; loaderId?: string }>(
395
- "Page.navigate",
396
- { url },
397
- { mutation: true, timeoutMs: 30_000 },
398
- );
399
- if (result.errorText) throw new Error(`Navigation failed: ${result.errorText}`);
400
- if (!result.loaderId) return;
401
- expectedLoaderId = result.loaderId;
402
- if (observedLoaders.has(expectedLoaderId)) return;
403
- timer = setTimeout(
404
- () =>
405
- loaded.reject(
406
- new CdpUnknownOutcomeError(
407
- "Page.navigate did not reach DOMContentLoaded; mutation outcome is unknown",
408
- ),
409
- ),
410
- 30_000,
630
+ await this.withInvalidatedScreencast(
631
+ async (page) => {
632
+ const assertCurrent = this.navigationAttachmentGuard(page);
633
+ await waitForNavigationCommit(
634
+ page,
635
+ async () => {
636
+ const result = await page.send<{
637
+ frameId: string;
638
+ errorText?: string;
639
+ loaderId?: string;
640
+ isDownload?: boolean;
641
+ }>(
642
+ "Page.navigate",
643
+ { url },
644
+ { mutation: true, timeoutMs: Math.min(this.timeoutMs, NAVIGATION_ACK_TIMEOUT_MS) },
645
+ );
646
+ // Chromium can abort document navigation because the accepted
647
+ // response is a download. This does not replace the current page.
648
+ if (result.errorText && !result.isDownload) {
649
+ throw new Error(`Navigation failed: ${result.errorText}`);
650
+ }
651
+ return {
652
+ frameId: result.frameId,
653
+ ...(result.loaderId ? { loaderId: result.loaderId } : {}),
654
+ url,
655
+ ...(result.isDownload ? { download: true } : {}),
656
+ };
657
+ },
658
+ assertCurrent,
659
+ Math.min(this.timeoutMs, NAVIGATION_COMMIT_TIMEOUT_MS),
411
660
  );
412
- timer.unref();
413
- await loaded.promise;
414
- } finally {
415
- clearTimeout(timer);
416
- page.off("Page.lifecycleEvent", onLifecycle);
417
- }
418
- });
661
+ },
662
+ { preserveStream: true },
663
+ );
419
664
  }
420
665
 
421
666
  async back(): Promise<void> {
422
- await this.withInvalidatedScreencast(async (page) => {
423
- const history = await this.navigationHistory(page);
424
- const entry = history.entries[history.currentIndex - 1];
425
- if (!entry) throw new Error("Browser cannot go back");
426
- await page.send("Page.navigateToHistoryEntry", { entryId: entry.id }, { mutation: true });
427
- });
667
+ await this.navigateHistory(-1);
428
668
  }
429
669
 
430
670
  async forward(): Promise<void> {
431
- await this.withInvalidatedScreencast(async (page) => {
432
- const history = await this.navigationHistory(page);
433
- const entry = history.entries[history.currentIndex + 1];
434
- if (!entry) throw new Error("Browser cannot go forward");
435
- await page.send("Page.navigateToHistoryEntry", { entryId: entry.id }, { mutation: true });
436
- });
671
+ await this.navigateHistory(1);
672
+ }
673
+
674
+ /** History's ACK has no loader receipt. Capture its root loader and target URL
675
+ * in one parallel bounded preflight, then admit only the matching new commit. */
676
+ private async navigateHistory(direction: -1 | 1): Promise<void> {
677
+ await this.withInvalidatedScreencast(
678
+ async (page) => {
679
+ const assertCurrent = this.navigationAttachmentGuard(page);
680
+ const [history, root] = await Promise.all([
681
+ this.navigationHistory(page, Math.min(this.timeoutMs, NAVIGATION_METADATA_TIMEOUT_MS)),
682
+ this.navigationRoot(page),
683
+ ]);
684
+ assertCurrent();
685
+ const entry = history.entries[history.currentIndex + direction];
686
+ if (!entry)
687
+ throw new Error(
688
+ direction === -1 ? "Browser cannot go back" : "Browser cannot go forward",
689
+ );
690
+ await waitForNavigationCommit(
691
+ page,
692
+ async () => {
693
+ await page.send(
694
+ "Page.navigateToHistoryEntry",
695
+ { entryId: entry.id },
696
+ {
697
+ mutation: true,
698
+ timeoutMs: Math.min(this.timeoutMs, NAVIGATION_ACK_TIMEOUT_MS),
699
+ },
700
+ );
701
+ return { frameId: root.id, previousLoaderId: root.loaderId, url: entry.url };
702
+ },
703
+ assertCurrent,
704
+ Math.min(this.timeoutMs, NAVIGATION_COMMIT_TIMEOUT_MS),
705
+ );
706
+ },
707
+ { preserveStream: true },
708
+ );
437
709
  }
438
710
 
439
711
  async reload(ignoreCache = false): Promise<void> {
440
- await this.withInvalidatedScreencast((page) =>
441
- page.send("Page.reload", { ignoreCache }, { mutation: true }),
712
+ await this.withInvalidatedScreencast(
713
+ async (page) => {
714
+ const assertCurrent = this.navigationAttachmentGuard(page);
715
+ const root = await this.navigationRoot(page);
716
+ assertCurrent();
717
+ await waitForNavigationCommit(
718
+ page,
719
+ async () => {
720
+ await page.send(
721
+ "Page.reload",
722
+ { ignoreCache },
723
+ {
724
+ mutation: true,
725
+ timeoutMs: Math.min(this.timeoutMs, NAVIGATION_ACK_TIMEOUT_MS),
726
+ },
727
+ );
728
+ return { frameId: root.id, previousLoaderId: root.loaderId };
729
+ },
730
+ assertCurrent,
731
+ Math.min(this.timeoutMs, NAVIGATION_COMMIT_TIMEOUT_MS),
732
+ );
733
+ },
734
+ { preserveStream: true },
735
+ );
736
+ }
737
+
738
+ /** A root snapshot is observation only, bounded before mutation publication. */
739
+ private async navigationRoot(page: CdpSession): Promise<{ id: string; loaderId: string }> {
740
+ const result = await page.send<{ frameTree: { frame: { id: string; loaderId: string } } }>(
741
+ "Page.getFrameTree",
742
+ {},
743
+ { timeoutMs: Math.min(this.timeoutMs, NAVIGATION_METADATA_TIMEOUT_MS) },
442
744
  );
745
+ const root = result.frameTree.frame;
746
+ if (!root.id || !root.loaderId)
747
+ throw new CdpUnavailableError("Browser main frame is unavailable");
748
+ return root;
749
+ }
750
+
751
+ /** Keep commit observation on the original attachment, including shutdown. */
752
+ private navigationAttachmentGuard(page: CdpSession): () => void {
753
+ const attachmentGeneration = this.attachmentGeneration;
754
+ return () => {
755
+ this.assertAttachmentCurrent(page.connection, attachmentGeneration);
756
+ if (this.page !== page)
757
+ throw new CdpUnavailableError("Browser page changed during navigation");
758
+ };
443
759
  }
444
760
 
445
761
  async emulate(device: DeviceEmulation): Promise<void> {
446
762
  this.assertViewport(device);
447
- await this.withInvalidatedScreencast(async (page) => {
763
+ await this.withVideoTransition(async () => {
764
+ await this.withInvalidatedScreencast(async (page) => {
765
+ const attachmentGeneration = this.attachmentGeneration;
766
+ this.emulationAppliedPage = null;
767
+ await this.applyEmulation(page, device);
768
+ this.assertAttachmentCurrent(page.connection, attachmentGeneration);
769
+ if (this.page !== page)
770
+ throw new CdpUnavailableError("Browser page changed during emulation");
771
+ this.viewport = { ...device };
772
+ this.emulationAppliedPage = page;
773
+ });
774
+ });
775
+ }
776
+
777
+ /** Retain the full mode/UA/screen intent while rendering enough real source pixels for the chosen density. */
778
+ async setCaptureDensity(density: number, baseDeviceScaleFactor: number): Promise<void> {
779
+ const scale = captureDensitySchema.parse(density);
780
+ const viewport = this.requireViewport();
781
+ if (!canUseCaptureDensity(viewport, scale))
782
+ throw new RangeError("Capture density exceeds the supported image bounds");
783
+ if (
784
+ !Number.isFinite(baseDeviceScaleFactor) ||
785
+ baseDeviceScaleFactor < 1 ||
786
+ baseDeviceScaleFactor > 4
787
+ )
788
+ throw new RangeError("Invalid device pixel ratio");
789
+ await this.emulate({
790
+ ...viewport,
791
+ captureScale: scale,
792
+ deviceScaleFactor: Math.max(baseDeviceScaleFactor, scale),
793
+ });
794
+ }
795
+
796
+ /** Apply the entire device contract to one CDP session without changing saved intent. */
797
+ private async applyEmulation(page: CdpSession, device: DeviceEmulation): Promise<void> {
798
+ await page.send(
799
+ "Emulation.setDeviceMetricsOverride",
800
+ {
801
+ width: device.width,
802
+ height: device.height,
803
+ deviceScaleFactor: device.deviceScaleFactor,
804
+ mobile: device.mobile,
805
+ screenWidth: device.screenWidth ?? device.width,
806
+ screenHeight: device.screenHeight ?? device.height,
807
+ screenOrientation: {
808
+ type: device.width > device.height ? "landscapePrimary" : "portraitPrimary",
809
+ angle: device.width > device.height ? 90 : 0,
810
+ },
811
+ },
812
+ { mutation: true },
813
+ );
814
+ await page.send(
815
+ "Emulation.setTouchEmulationEnabled",
816
+ { enabled: device.touch, maxTouchPoints: device.touch ? 5 : 1 },
817
+ { mutation: true },
818
+ );
819
+ // A page controlled through its own session may sit behind another tab in the shared
820
+ // browser. Chromium holds a hidden page's mouseMoved acknowledgement for ~5 s (equal to the
821
+ // input idle bound), so every press there failed. Target-scoped focus emulation makes only
822
+ // this page report visible/focused: no tab is raised and no viewer's selection changes.
823
+ await page.send("Emulation.setFocusEmulationEnabled", { enabled: true }, { mutation: true });
824
+ if (device.userAgent) {
825
+ await page.send(
826
+ "Emulation.setUserAgentOverride",
827
+ { userAgent: device.userAgent, platform: device.platform ?? "" },
828
+ { mutation: true },
829
+ );
830
+ }
831
+ }
832
+
833
+ /** No frame/input may use a session whose emulated viewport has not been restored. */
834
+ private async restoreConfiguredEmulation(page: CdpSession): Promise<void> {
835
+ const device = this.viewport;
836
+ if (!device || this.emulationAppliedPage === page) return;
837
+ await this.withVideoTransition(async () => {
838
+ const attachmentGeneration = this.attachmentGeneration;
839
+ this.invalidateScreencastFrame();
840
+ this.emulationAppliedPage = null;
448
841
  await this.applyEmulation(page, device);
449
- this.viewport = { ...device };
842
+ this.assertAttachmentCurrent(page.connection, attachmentGeneration);
843
+ if (device !== this.viewport) {
844
+ throw new CdpUnavailableError("Browser device settings changed during restoration");
845
+ }
846
+ this.emulationAppliedPage = page;
847
+ this.invalidateScreencastFrame();
848
+ });
849
+ }
850
+
851
+ /** Reject late configuration completions after a disconnect or newer target selection. */
852
+ private assertAttachmentCurrent(connection: CdpConnection, generation: number): void {
853
+ this.assertRuntimeOpen();
854
+ if (
855
+ connection !== this.connection ||
856
+ !connection.isOpen ||
857
+ generation !== this.attachmentGeneration
858
+ ) {
859
+ throw new CdpUnavailableError("Browser attachment changed during emulation restoration");
860
+ }
861
+ }
862
+
863
+ /** Starts only exact restored native tab capture, lazily adding this viewer's quality encoder. */
864
+ async startVideo(quality: NativeVideoQuality = "high"): Promise<NativeVideoRead> {
865
+ return this.readVideo({ quality, waitMs: 0 });
866
+ }
867
+
868
+ /** Long-poll outside the input queue. The caller must fence returned inputGeneration before publication. */
869
+ async readVideo(input: {
870
+ quality: NativeVideoQuality;
871
+ bitrate?: number;
872
+ fps?: number;
873
+ streamId?: string | null;
874
+ afterSequence?: number;
875
+ waitMs?: number;
876
+ requestKeyFrame?: boolean;
877
+ }): Promise<NativeVideoRead & { inputGeneration: string }> {
878
+ if (!this.nativeVideo) {
879
+ return {
880
+ ...this.videoReset(),
881
+ status: "unsupported",
882
+ reasonCode: "video-disabled",
883
+ reason: "Encoded video is not enabled on this host",
884
+ };
885
+ }
886
+ const settings = resolveVideoEncoderSettings(input);
887
+ const transitionEpoch = this.videoTransitionEpoch;
888
+ if (!this.videoMayStart(transitionEpoch)) return this.videoReset();
889
+
890
+ // Retirement uses the same stop barrier as viewport changes. The next read
891
+ // resumes from current configuration instead of treating idle expiry as a
892
+ // permanent codec failure or creating a replacement while the old track closes.
893
+ if (this.video?.idleExpired) {
894
+ this.retireVideoForRead();
895
+ return this.videoReset();
896
+ }
897
+
898
+ const page = await this.requirePage();
899
+ // requirePage can itself restore or reconnect. Retry on the next read rather
900
+ // than creating a helper from a configuration that changed across its await.
901
+ if (!this.videoMayStart(transitionEpoch)) return this.videoReset();
902
+ const connection = this.requireConnection();
903
+ const generation = this.attachmentGeneration;
904
+ const viewport = this.requireViewport();
905
+ const recoveryContext = { connection, page, viewport, attachmentGeneration: generation };
906
+ const blocked = this.videoRecovery.blocked(recoveryContext);
907
+ if (blocked === "source-dimensions") {
908
+ return {
909
+ ...this.videoReset(),
910
+ status: "unsupported",
911
+ reason: "Native video dimensions do not match the configured viewport",
912
+ };
913
+ }
914
+ if (blocked) {
915
+ return this.videoReset();
916
+ }
917
+ if (!this.video) {
918
+ const pixels = captureDimensions(viewport, viewport.captureScale ?? 1);
919
+ this.video = new NativeVideoCapture({
920
+ connection,
921
+ targetId: page.targetId,
922
+ width: pixels.width,
923
+ height: pixels.height,
924
+ isCurrent: () =>
925
+ this.videoMayStart(transitionEpoch) &&
926
+ connection === this.connection &&
927
+ page === this.page &&
928
+ viewport === this.viewport &&
929
+ generation === this.attachmentGeneration &&
930
+ this.emulationAppliedPage === page,
931
+ });
932
+ }
933
+ const video = this.video;
934
+ const documentGeneration = this.documentGeneration;
935
+ const result = await video.read({ ...input, ...settings });
936
+ if (
937
+ !this.videoMayStart(transitionEpoch) ||
938
+ video.idleExpired ||
939
+ video !== this.video ||
940
+ connection !== this.connection ||
941
+ page !== this.page ||
942
+ viewport !== this.viewport ||
943
+ generation !== this.attachmentGeneration ||
944
+ documentGeneration !== this.documentGeneration
945
+ ) {
946
+ return this.videoReset();
947
+ }
948
+ if (video.sourceFailure) {
949
+ // Store failure admission before opening the existing stop barrier. A
950
+ // concurrent read sees either that barrier or the cooldown, never a second
951
+ // track while the failed source/helper is still being retired.
952
+ this.videoRecovery.failed(recoveryContext, video.sourceFailure);
953
+ const permanent = video.sourceFailure === "source-dimensions";
954
+ this.retireVideoForRead();
955
+ if (
956
+ this.stopping ||
957
+ connection !== this.connection ||
958
+ page !== this.page ||
959
+ viewport !== this.viewport ||
960
+ generation !== this.attachmentGeneration
961
+ ) {
962
+ return this.videoReset();
963
+ }
964
+ if (permanent) {
965
+ return {
966
+ ...this.videoReset(),
967
+ status: "unsupported",
968
+ reason: "Native video dimensions do not match the configured viewport",
969
+ };
970
+ }
971
+ return this.videoReset();
972
+ }
973
+ return {
974
+ ...result,
975
+ inputGeneration: formatRuntimeInputGeneration(
976
+ this.attachmentGeneration,
977
+ this.documentGeneration,
978
+ ),
979
+ };
980
+ }
981
+
982
+ /** Explicit action barrier; pointer samples preserve the native track and encoder continuity. */
983
+ invalidateQueuedVideoFrames(): void {
984
+ this.video?.invalidateQueuedFrames();
985
+ }
986
+
987
+ /** Establish the stop barrier now, but do not hold a viewing RPC across slow
988
+ * helper teardown. stopVideo owns barrier release; no replacement can start
989
+ * until it settles. Explicit stop callers still await their cleanup. */
990
+ private retireVideoForRead(): void {
991
+ const retirement = this.stopVideo();
992
+ this.videoRetirement = retirement;
993
+ const complete = () => {
994
+ if (this.videoRetirement === retirement) this.videoRetirement = null;
995
+ };
996
+ void retirement.then(complete, () => {
997
+ complete();
998
+ console.warn("[shared-browser] Video retirement failed");
450
999
  });
451
1000
  }
452
1001
 
453
- async startScreencast(quality = 65): Promise<void> {
1002
+ /** Stop admission too: a concurrent reader cannot replace a source still closing. */
1003
+ async stopVideo(): Promise<void> {
1004
+ if (this.videoRetirement) {
1005
+ await this.videoRetirement;
1006
+ return;
1007
+ }
1008
+ await this.withVideoTransition(async () => {});
1009
+ }
1010
+
1011
+ /** Nested device/attachment phases keep one continuous barrier until all phases finish. */
1012
+ private async withVideoTransition<T>(transition: () => Promise<T>): Promise<T> {
1013
+ const releaseJpeg = this.stopping ? () => {} : this.jpegDemand.pin(false);
1014
+ this.videoTransitionDepth += 1;
1015
+ this.videoTransitionEpoch += 1;
1016
+ const video = this.video;
1017
+ this.video = null;
1018
+ try {
1019
+ if (video) await video.stop();
1020
+ return await transition();
1021
+ } finally {
1022
+ this.videoTransitionDepth -= 1;
1023
+ releaseJpeg();
1024
+ }
1025
+ }
1026
+
1027
+ private videoMayStart(epoch: number): boolean {
1028
+ return !this.stopping && this.videoTransitionDepth === 0 && epoch === this.videoTransitionEpoch;
1029
+ }
1030
+
1031
+ private videoReset(): NativeVideoRead & { inputGeneration: string } {
1032
+ return {
1033
+ status: "reset",
1034
+ streamId: null,
1035
+ packets: [],
1036
+ inputGeneration: formatRuntimeInputGeneration(
1037
+ this.attachmentGeneration,
1038
+ this.documentGeneration,
1039
+ ),
1040
+ };
1041
+ }
1042
+
1043
+ /** Explicit viewing start remains supported, but does not keep unused JPEG work alive. */
1044
+ async startScreencast(quality: number = DEFAULT_JPEG_QUALITY): Promise<void> {
454
1045
  if (!Number.isInteger(quality) || quality < 1 || quality > 100) {
455
1046
  throw new RangeError("Screencast quality must be an integer from 1 to 100");
456
1047
  }
457
- this.screencastActive = true;
458
- this.screencastQuality = quality;
459
- const page = await this.requirePage();
460
- await this.startScreencastSession(page);
1048
+ const release = this.jpegDemand.pin();
1049
+ try {
1050
+ await this.jpegDemand.serialize(async () => {
1051
+ const page = await this.requirePage();
1052
+ if (this.screencastActive && this.screencastQuality === quality) return;
1053
+ this.screencastQuality = quality;
1054
+ this.screencastStartFailed = false;
1055
+ this.screencastActive = true;
1056
+ try {
1057
+ await this.startScreencastSession(page);
1058
+ } catch (error) {
1059
+ this.screencastActive = false;
1060
+ this.screencastStartFailed = true;
1061
+ throw error;
1062
+ }
1063
+ });
1064
+ } finally {
1065
+ release();
1066
+ }
461
1067
  }
462
1068
 
1069
+ /** Explicit detach cleanup and idle retirement use the same native transition FIFO. */
463
1070
  async stopScreencast(): Promise<void> {
1071
+ this.jpegDemand.cancel();
1072
+ await this.jpegDemand.serialize(() => this.stopScreencastSession());
1073
+ }
1074
+
1075
+ private async stopScreencastSession(): Promise<void> {
1076
+ const wasActive = this.screencastActive;
464
1077
  this.screencastActive = false;
1078
+ this.screencastStartFailed = false;
465
1079
  this.invalidateScreencastFrame();
466
- if (!this.page) return;
1080
+ if (!wasActive || !this.page) return;
467
1081
  try {
468
1082
  await this.page.send("Page.stopScreencast", {}, { mutation: true });
469
1083
  } finally {
@@ -471,71 +1085,288 @@ export class AgentBrowserRuntime {
471
1085
  }
472
1086
  }
473
1087
 
474
- async frame(maxBytes: number, quality = 65, waitMs = 500): Promise<RuntimeFrame> {
475
- if (!Number.isInteger(maxBytes) || maxBytes < 1)
1088
+ /** Return truthful fresh stream/fallback pixels; mutation/session changes revoke pending captures. */
1089
+ async frame(
1090
+ maxBytes: number,
1091
+ quality: number = DEFAULT_JPEG_QUALITY,
1092
+ waitMs = 500,
1093
+ ): Promise<RuntimeFrame> {
1094
+ if (!Number.isInteger(maxBytes) || maxBytes < 1) {
476
1095
  throw new RangeError("maxBytes must be positive");
477
- const streamed = this.screencastFrame ?? (await this.waitForFrame(waitMs));
478
- if (streamed && streamed.byteLength <= maxBytes) return streamed;
479
- let frame = await this.screenshot(await this.requirePage(), maxBytes, quality);
480
- if (!this.sizeMismatch(frame)) return frame;
481
- // The browser stopped honouring the cached viewport. Re-apply it once; never loop.
482
- await this.emulate(this.requireViewport());
483
- frame = await this.screenshot(await this.requirePage(), maxBytes, quality);
484
- const mismatch = this.sizeMismatch(frame);
485
- if (mismatch) throw new Error(`Browser returned a ${mismatch} after viewport resync`);
486
- return frame;
487
- }
488
-
489
- private async screenshot(page: CdpSession, maxBytes: number, quality: number) {
490
- for (const candidate of [quality, 50, 35, 20, 10, 1]) {
491
- const boundedQuality = Math.max(1, Math.min(100, Math.round(candidate)));
492
- const result = await page.send<{ data: string }>("Page.captureScreenshot", {
493
- format: "jpeg",
494
- quality: boundedQuality,
495
- fromSurface: true,
496
- captureBeyondViewport: false,
1096
+ }
1097
+ if (!Number.isFinite(quality)) throw new RangeError("JPEG quality must be finite");
1098
+ const requestedQuality = Math.max(1, Math.min(100, Math.round(quality)));
1099
+ // All reads protect in-flight pixels, but screenshot-only qualities do not
1100
+ // renew the shared JPEG stream when a viewer changes their preference.
1101
+ const release = this.jpegDemand.pin(requestedQuality === this.screencastQuality);
1102
+ try {
1103
+ await this.jpegDemand.serialize(async () => {
1104
+ const page = await this.requirePage();
1105
+ // Nonmatching quality keeps the existing screenshot-only path. A video
1106
+ // viewer choosing JPEG100 must not start an unused JPEG95 stream.
1107
+ if (
1108
+ requestedQuality !== this.screencastQuality ||
1109
+ this.screencastActive ||
1110
+ this.screencastStartFailed
1111
+ ) {
1112
+ return;
1113
+ }
1114
+ this.screencastActive = true;
1115
+ try {
1116
+ await this.startScreencastSession(page);
1117
+ } catch {
1118
+ // Shutdown is terminal, not an optional stream failure to fall back from.
1119
+ this.assertRuntimeOpen();
1120
+ // Stream support is optional. Preserve fresh screenshots, and avoid
1121
+ // retrying a failed setup at every 250ms poll in the same demand period.
1122
+ console.warn("Shared Browser JPEG screencast is unavailable; using screenshot fallback");
1123
+ this.screencastActive = false;
1124
+ this.screencastStartFailed = true;
1125
+ }
497
1126
  });
498
- const byteLength = Buffer.byteLength(result.data, "base64");
499
- if (byteLength <= maxBytes) {
500
- const viewport = this.requireViewport();
1127
+ this.assertRuntimeOpen();
1128
+ if (requestedQuality !== this.screencastQuality) {
1129
+ // Viewers choose their own quality; never restart the shared stream or
1130
+ // classify its health from another viewer's screenshot preference.
1131
+ const cached = this.capturePolicy.readScreenshot(
1132
+ performance.now(),
1133
+ requestedQuality,
1134
+ maxBytes,
1135
+ );
1136
+ return cached ?? (await this.captureScreenshot(maxBytes, requestedQuality, true));
1137
+ }
1138
+ // Expiry discards pixels without resetting fallback dwell/recovery evidence.
1139
+ // Mutations and connection changes use the stronger invalidation below.
1140
+ const generation = this.capturePolicy.currentGeneration();
1141
+ const receivedAt = this.screencastFrameReceivedAt;
1142
+ if (receivedAt === null || performance.now() - receivedAt > CAPTURE_MAX_AGE_MS) {
1143
+ this.screencastFrame = null;
1144
+ this.screencastFrameReceivedAt = null;
1145
+ }
1146
+ if (this.screencastActive && this.capturePolicy.canUseStream(performance.now())) {
1147
+ const streamed = this.screencastFrame ?? (await this.waitForFrame(waitMs));
1148
+ this.assertRuntimeOpen();
1149
+ if (!this.capturePolicy.isCurrent(generation)) {
1150
+ throw new CdpUnavailableError(
1151
+ "Browser capture was invalidated while waiting for a frame",
1152
+ );
1153
+ }
1154
+ if (streamed && streamed.byteLength <= maxBytes) {
1155
+ return streamed;
1156
+ }
1157
+ }
1158
+
1159
+ this.capturePolicy.enterFallback(performance.now());
1160
+ const cached = this.capturePolicy.readScreenshot(
1161
+ performance.now(),
1162
+ requestedQuality,
1163
+ maxBytes,
1164
+ );
1165
+ if (cached) {
1166
+ return cached;
1167
+ }
1168
+ return await this.captureScreenshot(maxBytes, requestedQuality);
1169
+ } finally {
1170
+ release();
1171
+ }
1172
+ }
1173
+
1174
+ /** Capture the configured viewport and cache only an unchanged-session/mutation completion. */
1175
+ private async captureScreenshot(
1176
+ maxBytes: number,
1177
+ requestedQuality: number,
1178
+ preserveStreamRecovery = false,
1179
+ ): Promise<RuntimeFrame> {
1180
+ const page = await this.requirePage();
1181
+ const screenshotGeneration = this.capturePolicy.beginScreenshot({
1182
+ preserveRecovery: preserveStreamRecovery,
1183
+ });
1184
+ try {
1185
+ const viewport = this.requireViewport();
1186
+ const expectedPixels = captureDimensions(viewport, viewport.captureScale);
1187
+ const readClip = async () => {
1188
+ const metrics = await page.send<{ cssVisualViewport: { pageX: number; pageY: number } }>(
1189
+ "Page.getLayoutMetrics",
1190
+ );
1191
+ this.assertCaptureCurrent(page, screenshotGeneration);
501
1192
  return {
502
- dataBase64: result.data,
503
- byteLength,
1193
+ x: metrics.cssVisualViewport.pageX,
1194
+ y: metrics.cssVisualViewport.pageY,
504
1195
  width: viewport.width,
505
1196
  height: viewport.height,
1197
+ scale: (viewport.captureScale ?? 1) / viewport.deviceScaleFactor,
1198
+ };
1199
+ };
1200
+ // Explicitly capture the visible CSS viewport, independent of the native window size.
1201
+ // Preserve scroll position and normalize device pixel ratio to the frame/input contract.
1202
+ let clip = await readClip();
1203
+ let resynced = false;
1204
+ // Reduce quality gradually only when a detailed frame exceeds the byte budget.
1205
+ const initialQuality = requestedQuality;
1206
+ const candidates = [...new Set([initialQuality, 90, 85, 75, 65, 50, 35, 20, 10, 1])].filter(
1207
+ (candidate) => candidate <= initialQuality,
1208
+ );
1209
+ const capture = async (quality: number) => {
1210
+ this.assertCaptureCurrent(page, screenshotGeneration);
1211
+ const result = await page.send<{ data: string }>("Page.captureScreenshot", {
1212
+ format: "jpeg",
1213
+ quality,
1214
+ fromSurface: true,
1215
+ // Beyond-viewport capture replaces emulation and resets enabled touch.
1216
+ captureBeyondViewport: false,
1217
+ clip,
1218
+ });
1219
+ this.assertCaptureCurrent(page, screenshotGeneration);
1220
+ const dimensions = readJpegFrameDimensions(result.data);
1221
+ if (!dimensions) throw new Error("Chromium returned a malformed JPEG screenshot");
1222
+ return {
1223
+ data: result.data,
1224
+ dimensions,
1225
+ byteLength: Buffer.byteLength(result.data, "base64"),
1226
+ };
1227
+ };
1228
+ const matchesViewport = (dimensions: { width: number; height: number }) =>
1229
+ dimensions.width === expectedPixels.width && dimensions.height === expectedPixels.height;
1230
+
1231
+ for (const candidate of candidates) {
1232
+ const boundedQuality = Math.max(1, Math.min(100, Math.round(candidate)));
1233
+ let pixels = await capture(boundedQuality);
1234
+ if (!matchesViewport(pixels.dimensions) && !resynced) {
1235
+ // Retain upstream viewport recovery, with captureScale rather than DPR:
1236
+ // mobile CSS emulation and image density are independently configured.
1237
+ // Recapture pixels only; uncertain input is never replayed.
1238
+ resynced = true;
1239
+ const attachmentGeneration = this.attachmentGeneration;
1240
+ await this.applyEmulation(page, viewport);
1241
+ this.assertAttachmentCurrent(page.connection, attachmentGeneration);
1242
+ this.assertCaptureCurrent(page, screenshotGeneration);
1243
+ if (viewport !== this.viewport)
1244
+ throw new CdpUnavailableError("Browser viewport changed during resync");
1245
+ this.emulationAppliedPage = page;
1246
+ clip = await readClip();
1247
+ pixels = await capture(boundedQuality);
1248
+ }
1249
+ if (!matchesViewport(pixels.dimensions)) {
1250
+ throw new Error(
1251
+ `Chromium returned a ${pixels.dimensions.width}x${pixels.dimensions.height} JPEG instead of ${expectedPixels.width}x${expectedPixels.height}`,
1252
+ );
1253
+ }
1254
+ if (pixels.byteLength > maxBytes) continue;
1255
+
1256
+ const frame: RuntimeFrame = {
1257
+ dataBase64: pixels.data,
1258
+ byteLength: pixels.byteLength,
1259
+ width: pixels.dimensions.width,
1260
+ height: pixels.dimensions.height,
506
1261
  transport: "screenshot",
507
1262
  capturedAt: new Date().toISOString(),
508
- } satisfies RuntimeFrame;
1263
+ };
1264
+ this.capturePolicy.rememberScreenshot(
1265
+ screenshotGeneration,
1266
+ frame,
1267
+ performance.now(),
1268
+ requestedQuality,
1269
+ maxBytes,
1270
+ );
1271
+ return frame;
509
1272
  }
1273
+ throw new Error(`JPEG screenshot exceeds ${maxBytes} bytes at minimum quality`);
1274
+ } finally {
1275
+ this.capturePolicy.endScreenshot(screenshotGeneration, performance.now());
510
1276
  }
511
- throw new Error(`JPEG screenshot exceeds ${maxBytes} bytes at minimum quality`);
512
1277
  }
513
1278
 
514
- private sizeMismatch(frame: RuntimeFrame): string | null {
515
- const { width, height, deviceScaleFactor: scale } = this.requireViewport();
516
- const size = jpegDimensions(frame.dataBase64);
517
- // Chromium rounds fractional device pixels, so allow one pixel of slack.
518
- if (
519
- size &&
520
- Math.abs(size.width - width * scale) <= 1 &&
521
- Math.abs(size.height - height * scale) <= 1
522
- )
523
- return null;
524
- return `${size ? `${size.width}x${size.height}` : "unreadable"} JPEG for a ${width}x${height} viewport at ${scale}x`;
1279
+ /** Refuse async pixels from a superseded session or mutation, without silently replaying capture. */
1280
+ private assertCaptureCurrent(page: CdpSession, generation: number): void {
1281
+ this.assertRuntimeOpen();
1282
+ if (page !== this.page || !this.capturePolicy.isCurrent(generation)) {
1283
+ throw new CdpUnavailableError("Browser capture was invalidated while taking a screenshot");
1284
+ }
1285
+ }
1286
+
1287
+ /** Invalidate pre-input pixels even if Chromium cannot confirm the mutation. */
1288
+ private async dispatchInput(
1289
+ method: string,
1290
+ params: Record<string, unknown>,
1291
+ gestureId?: string,
1292
+ mouseModifiers?: number,
1293
+ track?: () => void,
1294
+ ): Promise<void> {
1295
+ try {
1296
+ const page = await this.requirePage();
1297
+ if (gestureId) await this.assertLiveInput(gestureId);
1298
+ // Authority is validated. Record cleanup intent only now, immediately before
1299
+ // native publication, so rejected unsent packets never alter it while an
1300
+ // uncertain acknowledgement still leaves it in place.
1301
+ if (track) {
1302
+ if (this.heldOwner && this.heldOwner !== page) this.discardHeldState();
1303
+ track();
1304
+ if (this.holdsInput()) this.heldOwner ??= page;
1305
+ }
1306
+ const originalInput = gestureId ? this.liveInput : null;
1307
+ // A physical DOM mouse event may arrive after Control/Shift was pressed
1308
+ // outside this canvas. Explicit zero also overrides a prior held-key mask.
1309
+ const modifiers = mouseModifiers ?? heldKeyModifiers(this.heldKeys.values());
1310
+ const nativeParams =
1311
+ method === "Input.dispatchMouseEvent" && (mouseModifiers !== undefined || modifiers !== 0)
1312
+ ? { ...params, modifiers }
1313
+ : params;
1314
+ await page.send(method, nativeParams, { mutation: true });
1315
+ if (gestureId) {
1316
+ // An input can synchronously activate a link or submit a form. Its CDP
1317
+ // acknowledgment succeeds even when navigation ends the old channel.
1318
+ // Attachment replacement and uncertain sends retain their failure path.
1319
+ if (
1320
+ originalInput &&
1321
+ page === this.page &&
1322
+ originalInput.generation === this.attachmentGeneration &&
1323
+ originalInput.documentGeneration !== this.documentGeneration
1324
+ ) {
1325
+ await this.endLiveInput(gestureId);
1326
+ return;
1327
+ }
1328
+ await this.assertLiveInput(gestureId);
1329
+ this.renewLiveInput(gestureId);
1330
+ }
1331
+ } finally {
1332
+ this.invalidateScreencastFrame();
1333
+ }
525
1334
  }
526
1335
 
527
- async mouseMove(x: number, y: number): Promise<void> {
1336
+ async mouseMove(x: number, y: number, gestureId?: string, modifiers?: number): Promise<void> {
528
1337
  this.assertPoint(x, y);
529
- await (await this.requirePage()).send(
1338
+ this.assertMouseModifiers(modifiers);
1339
+ await this.dispatchInput(
530
1340
  "Input.dispatchMouseEvent",
531
1341
  {
532
1342
  type: "mouseMoved",
533
1343
  x,
534
1344
  y,
535
- button: "none",
1345
+ // Native scrollbar dragging needs the held button as well as its mask.
1346
+ button: this.heldMoveButton(),
536
1347
  buttons: this.buttonMask(),
537
1348
  },
538
- { mutation: true },
1349
+ gestureId,
1350
+ modifiers,
1351
+ () => {
1352
+ this.lastMousePoint = { x, y };
1353
+ },
1354
+ );
1355
+ }
1356
+
1357
+ /** Fixed native mouse departure; never maps an outside point to a page control. */
1358
+ async mouseLeave(gestureId: string): Promise<void> {
1359
+ if (this.heldButtons.size > 0) throw new Error("Release held mouse buttons before leaving");
1360
+ await this.dispatchInput(
1361
+ "Input.dispatchMouseEvent",
1362
+ {
1363
+ type: "mouseMoved",
1364
+ x: -1,
1365
+ y: -1,
1366
+ button: "none",
1367
+ buttons: 0,
1368
+ },
1369
+ gestureId,
539
1370
  );
540
1371
  }
541
1372
 
@@ -544,9 +1375,12 @@ export class AgentBrowserRuntime {
544
1375
  y: number,
545
1376
  button: MouseButton = "left",
546
1377
  clickCount = 1,
1378
+ gestureId?: string,
1379
+ modifiers?: number,
547
1380
  ): Promise<void> {
548
1381
  this.assertPoint(x, y);
549
- await (await this.requirePage()).send(
1382
+ this.assertMouseModifiers(modifiers);
1383
+ await this.dispatchInput(
550
1384
  "Input.dispatchMouseEvent",
551
1385
  {
552
1386
  type: "mousePressed",
@@ -556,14 +1390,28 @@ export class AgentBrowserRuntime {
556
1390
  buttons: this.buttonMask(button),
557
1391
  clickCount,
558
1392
  },
559
- { mutation: true },
1393
+ gestureId,
1394
+ modifiers,
1395
+ () => {
1396
+ // Publication may succeed before an acknowledgement is lost, so a
1397
+ // possibly held button is tracked from the send boundary.
1398
+ this.heldButtons.add(button);
1399
+ this.lastMousePoint = { x, y };
1400
+ },
560
1401
  );
561
- this.heldButtons.add(button);
562
1402
  }
563
1403
 
564
- async mouseUp(x: number, y: number, button: MouseButton = "left", clickCount = 1): Promise<void> {
1404
+ async mouseUp(
1405
+ x: number,
1406
+ y: number,
1407
+ button: MouseButton = "left",
1408
+ clickCount = 1,
1409
+ gestureId?: string,
1410
+ modifiers?: number,
1411
+ ): Promise<void> {
565
1412
  this.assertPoint(x, y);
566
- await (await this.requirePage()).send(
1413
+ this.assertMouseModifiers(modifiers);
1414
+ await this.dispatchInput(
567
1415
  "Input.dispatchMouseEvent",
568
1416
  {
569
1417
  type: "mouseReleased",
@@ -573,14 +1421,26 @@ export class AgentBrowserRuntime {
573
1421
  buttons: this.buttonMask(undefined, button),
574
1422
  clickCount,
575
1423
  },
576
- { mutation: true },
1424
+ gestureId,
1425
+ modifiers,
1426
+ () => {
1427
+ this.lastMousePoint = { x, y };
1428
+ },
577
1429
  );
578
1430
  this.heldButtons.delete(button);
579
1431
  }
580
1432
 
581
- async wheel(x: number, y: number, deltaX: number, deltaY: number): Promise<void> {
1433
+ async wheel(
1434
+ x: number,
1435
+ y: number,
1436
+ deltaX: number,
1437
+ deltaY: number,
1438
+ gestureId?: string,
1439
+ modifiers?: number,
1440
+ ): Promise<void> {
582
1441
  this.assertPoint(x, y);
583
- await (await this.requirePage()).send(
1442
+ this.assertMouseModifiers(modifiers);
1443
+ await this.dispatchInput(
584
1444
  "Input.dispatchMouseEvent",
585
1445
  {
586
1446
  type: "mouseWheel",
@@ -591,16 +1451,46 @@ export class AgentBrowserRuntime {
591
1451
  button: "none",
592
1452
  buttons: this.buttonMask(),
593
1453
  },
594
- { mutation: true },
1454
+ gestureId,
1455
+ modifiers,
595
1456
  );
596
1457
  }
597
1458
 
598
- async insertText(text: string): Promise<void> {
599
- await (await this.requirePage()).send("Input.insertText", { text }, { mutation: true });
1459
+ /** Insert a committed paste/IME value on the original guarded attachment. */
1460
+ async insertText(text: string, gestureId?: string): Promise<void> {
1461
+ await this.dispatchInput("Input.insertText", { text }, gestureId);
1462
+ }
1463
+
1464
+ /** Track cleanup intent before publication, including an uncertain key-down reply. */
1465
+ async dispatchKey(event: BrowserGestureKeyEvent, gestureId: string): Promise<void> {
1466
+ const parsed = browserGestureKeySchema.parse(event);
1467
+ await this.assertLiveInput(gestureId);
1468
+ const held = this.heldKeys.has(parsed.code);
1469
+ if (parsed.type === "down" && held !== parsed.repeat) {
1470
+ throw new Error(
1471
+ parsed.repeat ? "Key repeat has no matching press" : "Key is already pressed",
1472
+ );
1473
+ }
1474
+ if (parsed.type === "up" && !held) throw new Error("Key release has no matching press");
1475
+ if (parsed.type === "down") this.assertHeldKeyCapacity(parsed.code);
1476
+ await this.dispatchInput(
1477
+ "Input.dispatchKeyEvent",
1478
+ nativeKeyEvent(parsed),
1479
+ gestureId,
1480
+ undefined,
1481
+ parsed.type === "down"
1482
+ ? () => this.heldKeys.set(parsed.code, { key: parsed.key, code: parsed.code })
1483
+ : undefined,
1484
+ );
1485
+ if (parsed.type === "up") this.heldKeys.delete(parsed.code);
600
1486
  }
601
1487
 
602
- async keyDown(key: string, code = key): Promise<void> {
603
- await (await this.requirePage()).send(
1488
+ /** Track intent before publication. Internal discrete presses can use the
1489
+ * same opaque channel as live input so a lost ACK retains original-page cleanup. */
1490
+ async keyDown(key: string, code = key, gestureId?: string): Promise<void> {
1491
+ if (gestureId) await this.assertLiveInput(gestureId);
1492
+ this.assertHeldKeyCapacity(code);
1493
+ await this.dispatchInput(
604
1494
  "Input.dispatchKeyEvent",
605
1495
  {
606
1496
  type: "keyDown",
@@ -608,58 +1498,215 @@ export class AgentBrowserRuntime {
608
1498
  code,
609
1499
  text: key.length === 1 ? key : undefined,
610
1500
  },
611
- { mutation: true },
1501
+ gestureId,
1502
+ undefined,
1503
+ () => this.heldKeys.set(code, { key, code }),
612
1504
  );
613
- this.heldKeys.add(key);
614
1505
  }
615
1506
 
616
- async keyUp(key: string, code = key): Promise<void> {
617
- await (await this.requirePage()).send(
618
- "Input.dispatchKeyEvent",
619
- { type: "keyUp", key, code },
620
- { mutation: true },
621
- );
622
- this.heldKeys.delete(key);
1507
+ /** Release on a current channel; its end cleanup still owns the original
1508
+ * page if navigation or attachment loss prevents this ordinary key-up. */
1509
+ async keyUp(key: string, code = key, gestureId?: string): Promise<void> {
1510
+ await this.dispatchInput("Input.dispatchKeyEvent", { type: "keyUp", key, code }, gestureId);
1511
+ this.heldKeys.delete(code);
623
1512
  }
624
1513
 
1514
+ /**
1515
+ * The caller supplies the complete active contact set. CDP touchMove does not
1516
+ * release omitted contacts, so publish explicit contact ends before moving the
1517
+ * survivors. Any uncertain publication retains cleanup intent until cancel.
1518
+ */
625
1519
  async touch(
626
1520
  type: "touchStart" | "touchMove" | "touchEnd" | "touchCancel",
627
1521
  points: Array<{ x: number; y: number; id?: number }>,
1522
+ gestureId?: string,
628
1523
  ): Promise<void> {
629
1524
  for (const point of points) this.assertPoint(point.x, point.y);
630
- await (await this.requirePage()).send(
1525
+ if (gestureId) await this.assertLiveInput(gestureId);
1526
+ const contacts = points.map((point, index) => ({ ...point, id: point.id ?? index }));
1527
+ const activeIds = new Set(contacts.map((point) => point.id));
1528
+ const removed =
1529
+ type === "touchMove"
1530
+ ? [...this.activeTouches.values()].filter((point) => !activeIds.has(point.id))
1531
+ : [];
1532
+ const hold = () => {
1533
+ this.heldTouches = true;
1534
+ };
1535
+ if (removed.length > 0) {
1536
+ await this.dispatchInput(
1537
+ "Input.dispatchTouchEvent",
1538
+ { type: "touchEnd", touchPoints: removed },
1539
+ gestureId,
1540
+ undefined,
1541
+ hold,
1542
+ );
1543
+ }
1544
+ await this.dispatchInput(
631
1545
  "Input.dispatchTouchEvent",
632
- {
633
- type,
634
- touchPoints: points.map((point, index) => ({
635
- x: point.x,
636
- y: point.y,
637
- id: point.id ?? index,
638
- })),
639
- },
640
- { mutation: true },
1546
+ { type, touchPoints: contacts },
1547
+ gestureId,
1548
+ undefined,
1549
+ hold,
1550
+ );
1551
+ // Navigation cleanup already discarded this channel's contacts. Do not
1552
+ // recreate held-touch state after its acknowledged start caused navigation.
1553
+ if (gestureId && this.liveInput?.id !== gestureId) return;
1554
+ this.activeTouches.clear();
1555
+ if (type === "touchStart" || type === "touchMove") {
1556
+ for (const point of contacts) this.activeTouches.set(point.id, point);
1557
+ }
1558
+ this.heldTouches = this.activeTouches.size > 0;
1559
+ }
1560
+
1561
+ /**
1562
+ * Bind decoded geometry to its exact native attachment and document. Compare
1563
+ * after cleanup/reattach awaits so a same-URL reload cannot silently admit a
1564
+ * press against a replacement page. A mismatch publishes no physical input.
1565
+ */
1566
+ async beginLiveInput(id: string, expectedInputGeneration: string): Promise<void> {
1567
+ if (this.liveInput) await this.endLiveInput(this.liveInput.id);
1568
+ const page = await this.requirePage();
1569
+ const actualInputGeneration = formatRuntimeInputGeneration(
1570
+ this.attachmentGeneration,
1571
+ this.documentGeneration,
641
1572
  );
1573
+ if (expectedInputGeneration !== actualInputGeneration) {
1574
+ throw new CdpUnavailableError("Live browser input attachment changed before admission");
1575
+ }
1576
+ // No asynchronous boundary may separate this comparison from the binding.
1577
+ const now = Date.now();
1578
+ this.liveInput = {
1579
+ id,
1580
+ page,
1581
+ generation: this.attachmentGeneration,
1582
+ documentGeneration: this.documentGeneration,
1583
+ idleUntil: now + GESTURE_IDLE_MS,
1584
+ expiresAt: now + GESTURE_LIFETIME_MS,
1585
+ timer: null,
1586
+ };
1587
+ this.scheduleLiveInputExpiry();
1588
+ }
1589
+
1590
+ /** Compare after every asynchronous transport boundary, including automatic reconnect/reattach. */
1591
+ async assertLiveInput(id: string): Promise<void> {
1592
+ const page = await this.requirePage();
1593
+ if (
1594
+ this.liveInput?.id === id &&
1595
+ (Date.now() >= this.liveInput.idleUntil || Date.now() >= this.liveInput.expiresAt)
1596
+ ) {
1597
+ await this.endLiveInput(id);
1598
+ }
1599
+ if (
1600
+ !this.liveInput ||
1601
+ this.liveInput.id !== id ||
1602
+ this.liveInput.page !== page ||
1603
+ this.liveInput.generation !== this.attachmentGeneration ||
1604
+ this.liveInput.documentGeneration !== this.documentGeneration
1605
+ ) {
1606
+ throw new CdpUnavailableError("Live browser input attachment changed");
1607
+ }
1608
+ }
1609
+
1610
+ /** Cleanup is idempotent and targets the original attachment, never whatever is currently active. */
1611
+ async endLiveInput(id: string): Promise<void> {
1612
+ const current = this.liveInput;
1613
+ if (!current || current.id !== id) return;
1614
+ this.liveInput = null;
1615
+ if (current.timer) clearTimeout(current.timer);
1616
+ await this.releaseHeldInput(current.page);
642
1617
  }
643
1618
 
644
- async releaseHeldInput(): Promise<void> {
645
- const page = this.page;
1619
+ /** Only acknowledged input extends idle time; metadata/checks cannot keep a stranded press alive. */
1620
+ private renewLiveInput(id: string): void {
1621
+ if (this.liveInput?.id !== id) return;
1622
+ this.liveInput.idleUntil = Date.now() + GESTURE_IDLE_MS;
1623
+ this.scheduleLiveInputExpiry();
1624
+ }
1625
+
1626
+ /** Runtime-owned expiry survives plugin disconnect or a fenced supervisor IPC client. */
1627
+ private scheduleLiveInputExpiry(): void {
1628
+ const current = this.liveInput;
1629
+ if (!current) return;
1630
+ if (current.timer) clearTimeout(current.timer);
1631
+ const delay = Math.max(0, Math.min(current.idleUntil, current.expiresAt) - Date.now());
1632
+ current.timer = setTimeout(() => {
1633
+ if (this.liveInput !== current) return;
1634
+ void this.endLiveInput(current.id).catch(() => undefined);
1635
+ }, delay);
1636
+ current.timer.unref?.();
1637
+ }
1638
+
1639
+ /** Fixed hit-test, safe cursor names only. Cursor failures cannot fail an acknowledged input. */
1640
+ async cursorAt(x: number, y: number, gestureId: string): Promise<BrowserCursor | null> {
1641
+ try {
1642
+ this.assertPoint(x, y);
1643
+ await this.assertLiveInput(gestureId);
1644
+ const page = this.liveInput!.page;
1645
+ const result = await page.send<{ result: { value?: unknown } }>(
1646
+ "Runtime.evaluate",
1647
+ {
1648
+ expression: browserCursorExpression(x, y),
1649
+ returnByValue: true,
1650
+ },
1651
+ { timeoutMs: Math.min(this.timeoutMs, CURSOR_SAMPLE_TIMEOUT_MS) },
1652
+ );
1653
+ await this.assertLiveInput(gestureId);
1654
+ const parsed = browserCursorSchema.safeParse(result.result.value);
1655
+ return parsed.success ? parsed.data : null;
1656
+ } catch {
1657
+ return null;
1658
+ }
1659
+ }
1660
+
1661
+ private holdsInput(): boolean {
1662
+ return this.heldButtons.size > 0 || this.heldKeys.size > 0 || this.heldTouches;
1663
+ }
1664
+
1665
+ /** Forget held state whose attachment is gone; it is never replayed on a replacement. */
1666
+ private discardHeldState(): void {
1667
+ this.heldButtons.clear();
1668
+ this.heldKeys.clear();
1669
+ this.heldTouches = false;
1670
+ this.activeTouches.clear();
1671
+ this.heldOwner = null;
1672
+ }
1673
+
1674
+ /**
1675
+ * Release only what `page` itself holds. State published on another attachment
1676
+ * is left to that attachment's own cleanup, never retargeted here.
1677
+ */
1678
+ async releaseHeldInput(page = this.page): Promise<void> {
646
1679
  if (!page) {
647
- this.heldButtons.clear();
648
- this.heldKeys.clear();
1680
+ this.discardHeldState();
649
1681
  return;
650
1682
  }
1683
+ if (this.heldOwner && this.heldOwner !== page) return;
1684
+ this.heldOwner = null;
651
1685
  const buttons = [...this.heldButtons];
652
- const keys = [...this.heldKeys];
1686
+ const { x, y } = this.lastMousePoint;
1687
+ const keys = [...this.heldKeys.values()];
653
1688
  this.heldButtons.clear();
654
1689
  this.heldKeys.clear();
1690
+ const touchHeld = this.heldTouches;
1691
+ this.heldTouches = false;
1692
+ this.activeTouches.clear();
655
1693
  await Promise.allSettled([
1694
+ ...(touchHeld
1695
+ ? [
1696
+ page.send(
1697
+ "Input.dispatchTouchEvent",
1698
+ { type: "touchCancel", touchPoints: [] },
1699
+ { mutation: true },
1700
+ ),
1701
+ ]
1702
+ : []),
656
1703
  ...buttons.map((button) =>
657
1704
  page.send(
658
1705
  "Input.dispatchMouseEvent",
659
1706
  {
660
1707
  type: "mouseReleased",
661
- x: 0,
662
- y: 0,
1708
+ x,
1709
+ y,
663
1710
  button,
664
1711
  buttons: 0,
665
1712
  clickCount: 1,
@@ -667,13 +1714,13 @@ export class AgentBrowserRuntime {
667
1714
  { mutation: true },
668
1715
  ),
669
1716
  ),
670
- ...keys.map((key) =>
1717
+ ...keys.map(({ key, code }) =>
671
1718
  page.send(
672
1719
  "Input.dispatchKeyEvent",
673
1720
  {
674
1721
  type: "keyUp",
675
1722
  key,
676
- code: key,
1723
+ code,
677
1724
  },
678
1725
  { mutation: true },
679
1726
  ),
@@ -681,35 +1728,112 @@ export class AgentBrowserRuntime {
681
1728
  ]);
682
1729
  }
683
1730
 
1731
+ /**
1732
+ * Concurrent callers share one in-flight shutdown; a failed one is not remembered as
1733
+ * success, so a later call retries and reports the same unconfirmed state.
1734
+ */
684
1735
  async shutdown(force = false): Promise<void> {
685
- if (this.stopping) return;
1736
+ if (this.shutDown) return;
1737
+ if (this.shutdownInFlight) return this.shutdownInFlight;
1738
+ const operation = this.performShutdown(force);
1739
+ this.shutdownInFlight = operation;
1740
+ try {
1741
+ await operation;
1742
+ this.shutDown = true;
1743
+ } finally {
1744
+ if (this.shutdownInFlight === operation) this.shutdownInFlight = null;
1745
+ }
1746
+ }
1747
+
1748
+ private async performShutdown(force: boolean): Promise<void> {
686
1749
  this.stopping = true;
687
- await this.releaseHeldInput();
1750
+ this.jpegDemand.close();
1751
+ this.screencastActive = false;
688
1752
  this.invalidateScreencastFrame();
689
- if (!force) {
1753
+ await this.withVideoTransition(async () => {
1754
+ if (this.liveInput) await this.endLiveInput(this.liveInput.id);
1755
+ await this.releaseHeldInput();
1756
+ this.invalidateScreencastFrame();
1757
+ // Only the controller that launched the browser owns its session daemon.
1758
+ if (this.ownsBrowser) await this.stopDaemon(force);
1759
+ this.connection?.close();
1760
+ this.connection = null;
1761
+ this.page = null;
1762
+ this.targetId = null;
1763
+ this.invalidateScreencastFrame();
1764
+ });
1765
+ }
1766
+
1767
+ /** Close, then require observed exit; escalate to a verified-owned SIGKILL and observe that too. */
1768
+ private async stopDaemon(force: boolean): Promise<void> {
1769
+ let forced = force;
1770
+ if (!forced) {
690
1771
  try {
691
1772
  await this.invoke(["--session", this.session, "--json", "close"]);
692
1773
  } catch {
693
- this.stopping = false;
694
- await this.shutdown(true);
695
- return;
1774
+ forced = true;
696
1775
  }
697
- } else {
698
- const pid = await this.daemonPid();
699
- if (pid !== null) {
700
- try {
701
- process.kill(pid, "SIGKILL");
702
- } catch (error) {
703
- if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
704
- }
1776
+ // close returns before the session daemon exits. A same-session relaunch would reach
1777
+ // that daemon with its original launch environment (e.g. a stopped private display).
1778
+ if (!forced && !(await this.daemonExited())) forced = true;
1779
+ }
1780
+ if (forced) await this.terminateDaemon();
1781
+ if (forced || this.daemonLaunched) await this.removeIpcMetadata();
1782
+ this.daemonLaunched = false;
1783
+ this.daemon = null;
1784
+ }
1785
+
1786
+ /**
1787
+ * Records the PID-file daemon. Only a PID whose environment carries this runtime's own
1788
+ * nonce gets a verified identity; anything else is observable but never signalled.
1789
+ */
1790
+ private async captureDaemon(): Promise<void> {
1791
+ const pid = await this.daemonPid();
1792
+ if (pid === null) return;
1793
+ const probe = await probeProcess(pid, this.ownerNonce);
1794
+ this.daemon = { pid, identity: probe.status === "owned" ? probe.identity : null };
1795
+ }
1796
+
1797
+ /**
1798
+ * Observes the daemon this runtime launched. Only `gone` is a positive exit; `unknown`
1799
+ * (no PID record, unreadable identity) is never success and never authorises a signal.
1800
+ */
1801
+ private async daemonState(): Promise<"gone" | "owned" | "alive" | "unknown"> {
1802
+ if (!this.daemonLaunched) return "gone";
1803
+ if (!this.daemon) {
1804
+ await this.captureDaemon();
1805
+ if (!this.daemon) return "unknown";
1806
+ }
1807
+ const { pid, identity } = this.daemon;
1808
+ if (!identity) return processExists(pid) ? "alive" : "gone";
1809
+ const probe = await probeProcess(pid, this.ownerNonce);
1810
+ if (probe.status === "absent" || probe.status === "foreign") return "gone";
1811
+ if (probe.status === "owned")
1812
+ return probe.identity.startTicks === identity.startTicks ? "owned" : "gone";
1813
+ return processExists(pid) ? "unknown" : "gone";
1814
+ }
1815
+
1816
+ /** True only after positive observation of exit, within the bound. */
1817
+ private async daemonExited(): Promise<boolean> {
1818
+ for (const deadline = Date.now() + this.daemonExitWaitMs; Date.now() < deadline; ) {
1819
+ if ((await this.daemonState()) === "gone") return true;
1820
+ await new Promise((resolve) => setTimeout(resolve, SCREENCAST_SETUP_RETRY_MS));
1821
+ }
1822
+ return (await this.daemonState()) === "gone";
1823
+ }
1824
+
1825
+ /** Signals only a daemon freshly re-verified as owned, then confirms its exit. */
1826
+ private async terminateDaemon(): Promise<void> {
1827
+ if ((await this.daemonState()) === "owned" && this.daemon) {
1828
+ try {
1829
+ process.kill(this.daemon.pid, "SIGKILL");
1830
+ } catch (error) {
1831
+ if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
705
1832
  }
706
- await this.removeIpcMetadata();
707
1833
  }
708
- this.connection?.close();
709
- this.connection = null;
710
- this.page = null;
711
- this.targetId = null;
712
- this.invalidateScreencastFrame();
1834
+ // Unverified or unknown daemons are never signalled, but exit must still be observed.
1835
+ if (!(await this.daemonExited()))
1836
+ throw new AgentBrowserUnavailableError("agent-browser daemon exit was not confirmed");
713
1837
  }
714
1838
 
715
1839
  private async assertVersion(): Promise<void> {
@@ -759,8 +1883,18 @@ export class AgentBrowserRuntime {
759
1883
  }
760
1884
  }
761
1885
 
762
- private async connectCdp(preferredTargetId: string | null = null): Promise<void> {
1886
+ private async connectCdp(
1887
+ preferredTargetId: string | null = null,
1888
+ activate = this.ownsBrowser,
1889
+ ): Promise<void> {
1890
+ this.assertRuntimeOpen();
1891
+ // A launch-selected page remains the intended target even if its first
1892
+ // attachment fails. Otherwise a retry could silently adopt another tab.
1893
+ if (this.targetId === null && preferredTargetId !== null) {
1894
+ this.targetId = preferredTargetId;
1895
+ }
763
1896
  const response = await this.invoke(["--session", this.session, "--json", "get", "cdp-url"]);
1897
+ this.assertRuntimeOpen();
764
1898
  const url = findString(response, ["cdpUrl", "cdp_url", "url"]);
765
1899
  if (!url) throw new AgentBrowserIncompatibleError("agent-browser did not return a CDP URL");
766
1900
  const endpoint = new URL(url);
@@ -771,77 +1905,102 @@ export class AgentBrowserRuntime {
771
1905
  commandTimeoutMs: this.timeoutMs,
772
1906
  connectTimeoutMs: this.timeoutMs,
773
1907
  });
1908
+ if (this.stopping) {
1909
+ connection.close();
1910
+ this.assertRuntimeOpen();
1911
+ }
774
1912
  this.connection = connection;
775
1913
  connection.once("disconnect", () => {
776
1914
  if (this.connection !== connection) return;
777
1915
  this.page = null;
778
- this.targetId = null;
1916
+ // Keep the selected page identity. A transport loss does not authorize
1917
+ // adopting another tab, even when the original page still exists.
1918
+ this.emulationAppliedPage = null;
1919
+ this.attachmentGeneration += 1;
779
1920
  this.invalidateScreencastFrame();
780
1921
  });
781
1922
  const targets = await listPageTargets(this.connection);
1923
+ this.assertRuntimeOpen();
1924
+ const expectedTargetId = preferredTargetId ?? this.targetId;
782
1925
  const target =
783
- targets.find((candidate) => candidate.targetId === preferredTargetId) ?? targets[0];
1926
+ expectedTargetId === null
1927
+ ? targets[0]
1928
+ : targets.find((candidate) => candidate.targetId === expectedTargetId);
1929
+ if (expectedTargetId !== null && !target) {
1930
+ throw new CdpUnavailableError("The selected browser page is no longer available");
1931
+ }
784
1932
  if (!target) throw new CdpUnavailableError("Chromium has no page target");
785
- await this.selectTarget(target.targetId);
1933
+ await this.selectTarget(target.targetId, activate);
786
1934
  }
787
1935
 
788
1936
  private async requirePage(): Promise<CdpSession> {
789
- if (this.page && this.connection?.isOpen) return this.page;
1937
+ this.assertRuntimeOpen();
1938
+ if (this.page && this.connection?.isOpen) {
1939
+ const page = this.page;
1940
+ await this.restoreConfiguredEmulation(page);
1941
+ this.assertRuntimeOpen();
1942
+ if (this.page !== page || !this.connection?.isOpen) {
1943
+ throw new CdpUnavailableError("Browser page changed during emulation restoration");
1944
+ }
1945
+ return page;
1946
+ }
790
1947
  await this.reconnect();
1948
+ this.assertRuntimeOpen();
791
1949
  if (!this.page) throw new CdpUnavailableError("No page target is attached");
792
1950
  return this.page;
793
1951
  }
794
1952
 
795
1953
  private requireConnection(): CdpConnection {
1954
+ this.assertRuntimeOpen();
796
1955
  if (!this.connection?.isOpen) throw new CdpUnavailableError("Browser runtime is disconnected");
797
1956
  return this.connection;
798
1957
  }
799
1958
 
1959
+ /** Shutdown cleanup uses captured attachments directly; acquisition cannot revive them. */
1960
+ private assertRuntimeOpen(): void {
1961
+ if (this.stopping) throw new CdpUnavailableError("Browser runtime is shutting down");
1962
+ }
1963
+
800
1964
  private requireViewport(): BrowserViewport {
801
1965
  if (!this.viewport) throw new AgentBrowserIncompatibleError("Viewport has not been configured");
802
1966
  return this.viewport;
803
1967
  }
804
1968
 
805
- private async preparePage(page: CdpSession): Promise<void> {
806
- page.on("Page.screencastFrame", this.onScreencastFrame);
1969
+ private async bindPageEvents(page: CdpSession): Promise<void> {
1970
+ // URL equality is insufficient for reload. Main-document and same-document
1971
+ // navigation revoke the channel without treating child-frame loads as a reset.
1972
+ let rootFrameId: string | undefined;
1973
+ page.on("Page.frameNavigated", (event: { frame: { id: string; parentId?: string } }) => {
1974
+ if (page !== this.page || event.frame.parentId) return;
1975
+ rootFrameId = event.frame.id;
1976
+ this.documentGeneration += 1;
1977
+ this.invalidateScreencastFrame();
1978
+ this.video?.invalidateQueuedFrames();
1979
+ void this.endLiveInput(this.liveInput?.id ?? "").catch(() => undefined);
1980
+ });
1981
+ page.on("Page.navigatedWithinDocument", (event: { frameId: string }) => {
1982
+ if (page !== this.page || event.frameId !== rootFrameId) return;
1983
+ this.documentGeneration += 1;
1984
+ this.invalidateScreencastFrame();
1985
+ this.video?.invalidateQueuedFrames();
1986
+ void this.endLiveInput(this.liveInput?.id ?? "").catch(() => undefined);
1987
+ });
1988
+ page.on("Page.screencastFrame", (event: ScreencastFrame) =>
1989
+ this.onScreencastFrame(page, event),
1990
+ );
807
1991
  page.on("event", (event: CdpEvent) => {
808
- if (event.method === "Inspector.targetCrashed") this.invalidateScreencastFrame();
1992
+ if (event.method === "Inspector.targetCrashed" && this.page === page) {
1993
+ this.documentGeneration += 1;
1994
+ void this.stopVideo();
1995
+ void this.endLiveInput(this.liveInput?.id ?? "").catch(() => undefined);
1996
+ this.emulationAppliedPage = null;
1997
+ this.invalidateScreencastFrame();
1998
+ }
809
1999
  });
810
2000
  await Promise.all([page.send("Page.enable"), page.send("Runtime.enable")]);
2001
+ const tree = await page.send<{ frameTree?: { frame: { id: string } } }>("Page.getFrameTree");
2002
+ rootFrameId = tree.frameTree?.frame.id ?? rootFrameId;
811
2003
  await this.navigationHistory(page);
812
- // Emulation is scoped to the CDP session, so every fresh attach starts without it.
813
- if (this.viewport) await this.applyEmulation(page, this.viewport);
814
- }
815
-
816
- private async applyEmulation(page: CdpSession, device: DeviceEmulation): Promise<void> {
817
- await page.send(
818
- "Emulation.setDeviceMetricsOverride",
819
- {
820
- width: device.width,
821
- height: device.height,
822
- deviceScaleFactor: device.deviceScaleFactor,
823
- mobile: device.mobile,
824
- screenWidth: device.screenWidth ?? device.width,
825
- screenHeight: device.screenHeight ?? device.height,
826
- screenOrientation: {
827
- type: device.width > device.height ? "landscapePrimary" : "portraitPrimary",
828
- angle: device.width > device.height ? 90 : 0,
829
- },
830
- },
831
- { mutation: true },
832
- );
833
- await page.send(
834
- "Emulation.setTouchEmulationEnabled",
835
- { enabled: device.touch, maxTouchPoints: device.touch ? 5 : 1 },
836
- { mutation: true },
837
- );
838
- if (device.userAgent) {
839
- await page.send(
840
- "Emulation.setUserAgentOverride",
841
- { userAgent: device.userAgent, platform: device.platform ?? "" },
842
- { mutation: true },
843
- );
844
- }
845
2004
  }
846
2005
 
847
2006
  private async startScreencastSession(page: CdpSession): Promise<void> {
@@ -849,23 +2008,34 @@ export class AgentBrowserRuntime {
849
2008
  let candidate = page;
850
2009
  for (;;) {
851
2010
  try {
2011
+ this.assertRuntimeOpen();
852
2012
  await Promise.all([candidate.send("Page.enable"), candidate.send("Runtime.enable")]);
2013
+ this.assertRuntimeOpen();
853
2014
  await this.navigationHistory(candidate);
2015
+ this.assertRuntimeOpen();
854
2016
  if (this.page !== candidate) {
855
2017
  candidate = await this.requirePage();
856
2018
  continue;
857
2019
  }
2020
+ await this.restoreConfiguredEmulation(candidate);
2021
+ this.assertRuntimeOpen();
2022
+ if (this.page !== candidate) {
2023
+ throw new CdpUnavailableError("Browser page changed before screencast start");
2024
+ }
858
2025
  await candidate.send(
859
2026
  "Page.startScreencast",
860
2027
  { format: "jpeg", quality: this.screencastQuality, everyNthFrame: 1 },
861
2028
  { mutation: true },
862
2029
  );
2030
+ this.assertRuntimeOpen();
863
2031
  return;
864
2032
  } catch (error) {
2033
+ this.assertRuntimeOpen();
865
2034
  if (Date.now() >= deadline) throw error;
866
2035
  try {
867
2036
  candidate = await this.reattachPageForScreencast(candidate);
868
2037
  } catch (reattachError) {
2038
+ this.assertRuntimeOpen();
869
2039
  if (Date.now() >= deadline) throw reattachError;
870
2040
  }
871
2041
  const { promise, resolve } = Promise.withResolvers<void>();
@@ -876,66 +2046,140 @@ export class AgentBrowserRuntime {
876
2046
  }
877
2047
 
878
2048
  private async reattachPageForScreencast(previous: CdpSession): Promise<CdpSession> {
879
- const connection = this.requireConnection();
880
- const targets = await listPageTargets(connection);
881
- const target = targets.find((candidate) => candidate.targetId === this.targetId) ?? targets[0];
882
- if (!target) throw new CdpUnavailableError("Chromium has no page target");
883
- await connection.send(
884
- "Target.activateTarget",
885
- { targetId: target.targetId },
886
- { mutation: true },
887
- );
888
- const replacement = await attachToTarget(connection, target.targetId);
889
- await this.preparePage(replacement);
890
- if (this.page === previous) {
891
- this.page = replacement;
892
- this.targetId = target.targetId;
893
- } else {
894
- await replacement.detach().catch(() => undefined);
2049
+ this.assertRuntimeOpen();
2050
+ if (this.page !== previous) {
2051
+ if (!this.page) throw new CdpUnavailableError("Browser page changed during reattachment");
2052
+ return this.page;
895
2053
  }
896
- await previous.detach().catch(() => undefined);
897
- return this.page ?? replacement;
2054
+ return this.withVideoTransition(async () => {
2055
+ const attachmentGeneration = ++this.attachmentGeneration;
2056
+ const connection = this.requireConnection();
2057
+ const targets = await listPageTargets(connection);
2058
+ this.assertAttachmentCurrent(connection, attachmentGeneration);
2059
+ const target = targets.find((candidate) => candidate.targetId === previous.targetId);
2060
+ if (!target)
2061
+ throw new CdpUnavailableError("The selected browser page is no longer available");
2062
+ await connection.send(
2063
+ "Target.activateTarget",
2064
+ { targetId: target.targetId },
2065
+ { mutation: true },
2066
+ );
2067
+ this.assertAttachmentCurrent(connection, attachmentGeneration);
2068
+ const replacement = await attachToTarget(connection, target.targetId);
2069
+ if (this.page !== previous || attachmentGeneration !== this.attachmentGeneration) {
2070
+ await replacement.detach().catch(() => undefined);
2071
+ // The losing controller's detach may clear target overrides. Repair the
2072
+ // current controller on its next access rather than trust earlier pixels.
2073
+ this.emulationAppliedPage = null;
2074
+ this.invalidateScreencastFrame();
2075
+ if (!this.page) throw new CdpUnavailableError("Browser page changed during reattachment");
2076
+ return this.page;
2077
+ }
2078
+ this.assertAttachmentCurrent(connection, attachmentGeneration);
2079
+
2080
+ this.invalidateScreencastFrame();
2081
+ // Detaching drops the session that holds any pressed button; release there first.
2082
+ if (this.liveInput) await this.endLiveInput(this.liveInput.id);
2083
+ await this.releaseHeldInput(previous);
2084
+ this.page = null;
2085
+ this.emulationAppliedPage = null;
2086
+ // Detach first: the former session can otherwise clear the newly applied override.
2087
+ try {
2088
+ await previous.detach();
2089
+ this.assertAttachmentCurrent(connection, attachmentGeneration);
2090
+ await this.bindPageEvents(replacement);
2091
+ this.assertAttachmentCurrent(connection, attachmentGeneration);
2092
+ await this.restoreConfiguredEmulation(replacement);
2093
+ this.assertAttachmentCurrent(connection, attachmentGeneration);
2094
+ this.page = replacement;
2095
+ this.targetId = target.targetId;
2096
+ return replacement;
2097
+ } catch (error) {
2098
+ await replacement.detach().catch(() => undefined);
2099
+ // Detaching a partial emulation can clear target metrics even if a newer
2100
+ // attachment won. Its next access must restore intent before admitting pixels.
2101
+ this.emulationAppliedPage = null;
2102
+ this.invalidateScreencastFrame();
2103
+ throw error;
2104
+ }
2105
+ });
898
2106
  }
899
2107
 
900
2108
  private async withInvalidatedScreencast(
901
2109
  mutation: (page: CdpSession) => Promise<unknown>,
2110
+ options: { preserveStream?: boolean } = {},
902
2111
  ): Promise<void> {
903
- const page = await this.requirePage();
904
- const restart = this.screencastActive;
905
- this.invalidateScreencastFrame();
906
- if (restart) await page.send("Page.stopScreencast", {}, { mutation: true });
2112
+ const releaseJpeg = this.jpegDemand.pin(false);
907
2113
  try {
908
- await mutation(page);
909
- } finally {
2114
+ const page = await this.requirePage();
2115
+ // Ordinary navigation is observed by the same native stream. Optional
2116
+ // capture teardown/setup must not consume the navigation ACK budget.
2117
+ const restart = this.screencastActive && !options.preserveStream;
910
2118
  this.invalidateScreencastFrame();
911
- if (restart && this.page === page) await this.startScreencastSession(page);
2119
+ if (restart) await page.send("Page.stopScreencast", {}, { mutation: true });
2120
+ this.assertRuntimeOpen();
2121
+ try {
2122
+ await mutation(page);
2123
+ this.invalidateQueuedVideoFrames();
2124
+ } finally {
2125
+ this.invalidateScreencastFrame();
2126
+ if (restart && this.page === page) {
2127
+ this.assertRuntimeOpen();
2128
+ await this.restoreConfiguredEmulation(page);
2129
+ await this.startScreencastSession(page);
2130
+ }
2131
+ }
2132
+ } finally {
2133
+ releaseJpeg();
912
2134
  }
913
2135
  }
914
2136
 
2137
+ /** Clear JPEG stream/screenshot receipts after input or transport changes. */
915
2138
  private invalidateScreencastFrame(): void {
2139
+ this.capturePolicy.invalidate();
2140
+ this.screencastSourceTime.invalidate(Date.now());
916
2141
  this.screencastFrame = null;
2142
+ this.screencastFrameReceivedAt = null;
917
2143
  this.resolveFrameWaiters(null);
918
2144
  }
919
2145
 
920
- private readonly onScreencastFrame = (event: ScreencastFrame): void => {
921
- void this.page
922
- ?.send("Page.screencastFrameAck", { sessionId: event.sessionId })
2146
+ private onScreencastFrame(page: CdpSession, event: ScreencastFrame): void {
2147
+ // A late frame belongs to its emitting session, even after page reattachment.
2148
+ void page
2149
+ .send("Page.screencastFrameAck", { sessionId: event.sessionId })
923
2150
  .catch(() => undefined);
924
- const width = Math.round(event.metadata.deviceWidth);
925
- const height = Math.round(event.metadata.deviceHeight);
2151
+ if (
2152
+ this.stopping ||
2153
+ page !== this.page ||
2154
+ !this.screencastActive ||
2155
+ this.emulationAppliedPage !== page
2156
+ )
2157
+ return;
2158
+ const dimensions = readJpegFrameDimensions(event.data);
926
2159
  const viewport = this.viewport;
927
- if (!viewport || width !== viewport.width || height !== viewport.height) return;
2160
+ // Chromium can report the new device dimensions while emitting a clipped transition image.
2161
+ // Reject those pixels and use the bounded screenshot fallback rather than mislabeling them.
2162
+ if (!viewport || !dimensions) return;
2163
+ const expectedPixels = captureDimensions(viewport, viewport.captureScale);
2164
+ if (dimensions.width !== expectedPixels.width || dimensions.height !== expectedPixels.height)
2165
+ return;
2166
+ const now = performance.now();
2167
+ const source = this.screencastSourceTime.accept(event.metadata.timestamp, Date.now(), now);
2168
+ if (!source) return;
2169
+ const { width, height } = dimensions;
928
2170
  const frame: RuntimeFrame = {
929
2171
  dataBase64: event.data,
930
2172
  byteLength: Buffer.byteLength(event.data, "base64"),
931
2173
  width,
932
2174
  height,
933
2175
  transport: "cdp-screencast",
934
- capturedAt: new Date().toISOString(),
2176
+ capturedAt: source.capturedAt,
935
2177
  };
2178
+ this.capturePolicy.observeStream(`provider:${event.metadata.timestamp}`, now);
936
2179
  this.screencastFrame = frame;
2180
+ this.screencastFrameReceivedAt = source.receivedAt;
937
2181
  this.resolveFrameWaiters(frame);
938
- };
2182
+ }
939
2183
 
940
2184
  private waitForFrame(waitMs: number): Promise<RuntimeFrame | null> {
941
2185
  const { promise, resolve } = Promise.withResolvers<RuntimeFrame | null>();
@@ -958,6 +2202,17 @@ export class AgentBrowserRuntime {
958
2202
  }
959
2203
 
960
2204
  private assertViewport(viewport: BrowserViewport): void {
2205
+ const captureScale = viewport.captureScale ?? 1;
2206
+ const pixels = captureDimensions(viewport, captureScale);
2207
+ if (
2208
+ !Number.isFinite(captureScale) ||
2209
+ captureScale < 1 ||
2210
+ captureScale > 2 ||
2211
+ pixels.width > MAX_VIEWPORT.width ||
2212
+ pixels.height > MAX_VIEWPORT.height
2213
+ ) {
2214
+ throw new RangeError("Capture density exceeds the supported image bounds");
2215
+ }
961
2216
  if (
962
2217
  !Number.isInteger(viewport.width) ||
963
2218
  viewport.width < 1 ||
@@ -986,6 +2241,35 @@ export class AgentBrowserRuntime {
986
2241
  }
987
2242
  }
988
2243
 
2244
+ /** CDP/Puppeteer priority for supported held buttons, independent of press order.
2245
+ * https://github.com/puppeteer/puppeteer/blob/main/packages/puppeteer-core/src/cdp/Input.ts
2246
+ */
2247
+ private heldMoveButton(): MouseButton | "none" {
2248
+ for (const button of ["left", "right", "middle"] as const) {
2249
+ if (this.heldButtons.has(button)) {
2250
+ return button;
2251
+ }
2252
+ }
2253
+ return "none";
2254
+ }
2255
+
2256
+ /** Validate before recording held-button intent or sending a native packet. */
2257
+ private assertMouseModifiers(modifiers: number | undefined): void {
2258
+ if (
2259
+ modifiers !== undefined &&
2260
+ (!Number.isInteger(modifiers) || modifiers < 0 || modifiers > 15)
2261
+ ) {
2262
+ throw new RangeError("Mouse modifiers must be an integer from 0 to 15");
2263
+ }
2264
+ }
2265
+
2266
+ /** Both live and discrete keyboard paths must bound release fanout before sending. */
2267
+ private assertHeldKeyCapacity(code: string): void {
2268
+ if (!this.heldKeys.has(code) && this.heldKeys.size >= MAX_HELD_BROWSER_KEYS) {
2269
+ throw new Error("Too many simultaneously held browser keys");
2270
+ }
2271
+ }
2272
+
989
2273
  private buttonMask(add?: MouseButton, remove?: MouseButton): number {
990
2274
  const active = new Set(this.heldButtons);
991
2275
  if (add) active.add(add);