@omercnet/paseo-shared-browser 1.2.2-next.232.1 → 1.2.2-next.237.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/README.md +312 -25
  2. package/REMOTE_BROWSER_RESEARCH.md +395 -0
  3. package/STREAMING_DESIGN.md +191 -0
  4. package/client/browser-auto-capture-density.test.ts +102 -0
  5. package/client/browser-auto-capture-density.ts +30 -0
  6. package/client/browser-canvas-input.test.ts +198 -0
  7. package/client/browser-canvas-input.ts +202 -0
  8. package/client/browser-canvas-keyboard-web.test.ts +113 -0
  9. package/client/browser-canvas-keyboard-web.ts +230 -0
  10. package/client/browser-canvas-keyboard.test.ts +123 -0
  11. package/client/browser-canvas-keyboard.ts +126 -0
  12. package/client/browser-canvas-layout.test.ts +109 -0
  13. package/client/browser-canvas-layout.ts +51 -0
  14. package/client/browser-canvas-viewport-web.test.ts +77 -0
  15. package/client/browser-canvas-viewport-web.ts +63 -0
  16. package/client/browser-canvas-viewport.tsx +152 -0
  17. package/client/browser-capture-cadence.test.ts +9 -0
  18. package/client/browser-capture-cadence.ts +6 -0
  19. package/client/browser-capture-density-controls.tsx +75 -0
  20. package/client/browser-chrome.tsx +234 -0
  21. package/client/browser-client-presence.ts +184 -0
  22. package/client/browser-compose-controls.test.tsx +275 -0
  23. package/client/browser-compose-controls.tsx +174 -0
  24. package/client/browser-control-button.tsx +109 -0
  25. package/client/browser-control-dialog.tsx +71 -0
  26. package/client/browser-control-request.test.ts +27 -0
  27. package/client/browser-control-request.ts +57 -0
  28. package/client/browser-control-tooltip-web.ts +10 -0
  29. package/client/browser-dialog-body.tsx +22 -0
  30. package/client/browser-emulation-mode.test.ts +75 -0
  31. package/client/browser-emulation-mode.ts +98 -0
  32. package/client/browser-fill-viewport.test.ts +45 -0
  33. package/client/browser-fill-viewport.ts +28 -0
  34. package/client/browser-frame-buffer.test.ts +157 -0
  35. package/client/browser-frame-buffer.ts +148 -0
  36. package/client/browser-frame-image.tsx +62 -0
  37. package/client/browser-image-load.test.ts +23 -0
  38. package/client/browser-image-load.ts +26 -0
  39. package/client/browser-input-queue.test.ts +1201 -0
  40. package/client/browser-input-queue.ts +505 -0
  41. package/client/browser-menu-placement.test.ts +90 -0
  42. package/client/browser-menu-placement.ts +78 -0
  43. package/client/browser-panel-styles.ts +384 -0
  44. package/client/browser-quality-controls.test.ts +26 -0
  45. package/client/browser-quality-controls.tsx +139 -0
  46. package/client/browser-resolution-picker.tsx +189 -0
  47. package/client/browser-tabs.tsx +117 -0
  48. package/client/browser-toolbar-menu-web.test.ts +185 -0
  49. package/client/browser-toolbar-menu-web.ts +159 -0
  50. package/client/browser-toolbar-menu.tsx +255 -0
  51. package/client/browser-toolbar-submenu.tsx +93 -0
  52. package/client/browser-video-bytes.test.ts +44 -0
  53. package/client/browser-video-decoder.test.ts +473 -0
  54. package/client/browser-video-decoder.ts +311 -0
  55. package/client/browser-video-surface.tsx +21 -0
  56. package/client/browser-video-visibility.test.ts +45 -0
  57. package/client/browser-viewer-recovery.test.ts +111 -0
  58. package/client/browser-viewer-recovery.ts +78 -0
  59. package/client/browser.tsx +1642 -1569
  60. package/client/favorite-star.tsx +23 -0
  61. package/client/frame-lifecycle.ts +62 -59
  62. package/client/use-browser-auto-capture-density.test.ts +104 -0
  63. package/client/use-browser-auto-capture-density.ts +43 -0
  64. package/client/use-browser-canvas-input.ts +370 -0
  65. package/client/use-browser-capture-density.ts +87 -0
  66. package/client/use-browser-emulation-mode.ts +56 -0
  67. package/client/use-browser-fill-viewport.test.ts +108 -0
  68. package/client/use-browser-fill-viewport.ts +62 -0
  69. package/client/use-browser-frame-buffer.ts +66 -0
  70. package/client/use-browser-image-capture.ts +139 -0
  71. package/client/use-browser-scoped-mutation.test.ts +118 -0
  72. package/client/use-browser-scoped-mutation.ts +59 -0
  73. package/client/use-browser-video.test.ts +991 -0
  74. package/client/use-browser-video.ts +474 -0
  75. package/client/use-browser-viewer-recovery.ts +26 -0
  76. package/client/use-resolution-favorites.ts +90 -0
  77. package/client/web.test.ts +125 -0
  78. package/client/web.ts +405 -0
  79. package/index.client.tsx +0 -10
  80. package/index.server.ts +32 -0
  81. package/package.json +9 -5
  82. package/paseo-plugin.json +3 -1
  83. package/scripts/prepare-runtime.mjs +1 -1
  84. package/server/agent-browser-runtime.ts +1598 -314
  85. package/server/browser-cursor.test.ts +64 -0
  86. package/server/browser-cursor.ts +69 -0
  87. package/server/browser-frame-context.ts +22 -0
  88. package/server/browser-frame-epoch.test.ts +95 -0
  89. package/server/browser-frame-rejection.ts +17 -0
  90. package/server/browser-gesture.ts +120 -0
  91. package/server/browser-operation-diagnostics.test.ts +79 -0
  92. package/server/browser-operation-diagnostics.ts +103 -0
  93. package/server/browser-policy.ts +1313 -151
  94. package/server/browser-video-input-authority.test.ts +208 -0
  95. package/server/browser-video-policy.test.ts +564 -0
  96. package/server/browser-video-socket.test.ts +368 -0
  97. package/server/browser.ts +151 -16
  98. package/server/capture-transport-policy.test.ts +105 -0
  99. package/server/capture-transport-policy.ts +154 -0
  100. package/server/capture-transport-runtime.test.ts +213 -0
  101. package/server/daemon-launch-chain.test.ts +70 -0
  102. package/server/daemon-shutdown.test.ts +289 -0
  103. package/server/discrete-input-freshness.test.ts +364 -0
  104. package/server/discrete-input-native-lifetime.test.ts +150 -0
  105. package/server/display-loss-recovery.test.ts +294 -0
  106. package/server/emulation-restoration.test.ts +262 -0
  107. package/server/input-cleanup.ts +23 -0
  108. package/server/input-generation.test.ts +31 -0
  109. package/server/input-generation.ts +36 -0
  110. package/server/jpeg-capture-demand.test.ts +76 -0
  111. package/server/jpeg-capture-demand.ts +98 -0
  112. package/server/jpeg-frame.ts +29 -0
  113. package/server/keyboard-input.test.ts +61 -0
  114. package/server/keyboard-input.ts +104 -0
  115. package/server/live-input-policy.test.ts +958 -0
  116. package/server/live-input-runtime.test.ts +732 -0
  117. package/server/live-input-supervisor.test.ts +147 -0
  118. package/server/mcp-entry.ts +44 -2
  119. package/server/native-video-backpressure.test.ts +228 -0
  120. package/server/native-video-buffer.test.ts +141 -0
  121. package/server/native-video-buffer.ts +114 -0
  122. package/server/native-video-capture.test.ts +599 -0
  123. package/server/native-video-capture.ts +497 -0
  124. package/server/native-video-extension-source.test.ts +201 -0
  125. package/server/native-video-extension-source.ts +364 -0
  126. package/server/native-video-extension.test.ts +41 -0
  127. package/server/native-video-extension.ts +83 -0
  128. package/server/native-video-packet.ts +9 -0
  129. package/server/native-video-settings.test.ts +245 -0
  130. package/server/navigation-budget.ts +8 -0
  131. package/server/navigation-commit.ts +125 -0
  132. package/server/navigation-policy.test.ts +153 -0
  133. package/server/process-identity.ts +58 -0
  134. package/server/reattach-held-input.test.ts +143 -0
  135. package/server/runtime-frame-reuse.test.ts +295 -0
  136. package/server/runtime-jpeg-demand.test.ts +200 -0
  137. package/server/runtime-jpeg-shutdown.test.ts +210 -0
  138. package/server/runtime-jpeg-source-order.test.ts +201 -0
  139. package/server/runtime-launch-environment.test.ts +111 -0
  140. package/server/runtime-navigation.test.ts +327 -0
  141. package/server/runtime-owner-display.test.ts +152 -0
  142. package/server/runtime-owner.ts +305 -31
  143. package/server/runtime-page-state.test.ts +249 -0
  144. package/server/runtime-protocol.ts +31 -1
  145. package/server/runtime-target-retention.test.ts +181 -0
  146. package/server/runtime-video-opt-in.test.ts +82 -0
  147. package/server/runtime-video.test.ts +444 -0
  148. package/server/screencast-source-time.ts +46 -0
  149. package/server/stale-wheel-admission.test.ts +136 -0
  150. package/server/supervisor-client.ts +28 -4
  151. package/server/supervisor-heartbeat.test.ts +80 -0
  152. package/server/supervisor.ts +553 -55
  153. package/server/video-source-recovery.test.ts +20 -0
  154. package/server/video-source-recovery.ts +64 -0
  155. package/server/viewer-capture-lifetime.ts +51 -0
  156. package/server/virtual-display.test.ts +184 -0
  157. package/server/virtual-display.ts +236 -0
  158. package/shared/browser-display-preferences.ts +46 -0
  159. package/shared/browser-video.ts +80 -0
  160. package/shared/browser.ts +286 -65
  161. package/shared/capture-density.ts +16 -0
  162. package/shared/capture-settings.ts +17 -0
  163. package/shared/device-presets.ts +90 -0
  164. package/shared/resolution-menu.test.ts +120 -0
  165. package/shared/resolution-menu.ts +72 -0
  166. package/shared/video-settings.test.ts +38 -0
  167. package/shared/video-settings.ts +34 -0
  168. package/shared/viewport-limits.ts +5 -0
@@ -1,26 +1,86 @@
1
1
  import { createHash, randomUUID } from "node:crypto";
2
2
  import { homedir } from "node:os";
3
3
  import { join } from "node:path";
4
- import { AgentBrowserRuntime, type BrowserViewport } from "./agent-browser-runtime";
4
+ import { browserGestureKeySchema, MAX_BROWSER_TABS } from "../shared/browser";
5
+ import { videoBitrateSchema, videoFrameRateSchema } from "../shared/video-settings";
6
+ import {
7
+ AgentBrowserRuntime,
8
+ type AgentBrowserRuntimeOptions,
9
+ type BrowserViewport,
10
+ } from "./agent-browser-runtime";
5
11
  import { resolveBrowserRuntimeRoot } from "./runtime-path";
6
12
  import type { JsonValue } from "./runtime-protocol";
7
13
  import type { RuntimeOwner } from "./supervisor";
14
+ import {
15
+ createPrivateVirtualDisplay,
16
+ PrivateDisplayError,
17
+ type PrivateVirtualDisplay,
18
+ } from "./virtual-display";
8
19
 
9
20
  const DEFAULT_BROWSER_URL = "https://example.com/";
10
21
 
11
22
  interface OwnedRuntime {
12
23
  runtimeId: string;
13
24
  runtime: AgentBrowserRuntime;
25
+ rootTargetId: string;
26
+ runtimeOptions: AgentBrowserRuntimeOptions;
27
+ tabs: Map<string, Promise<AgentBrowserRuntime>>;
28
+ display: PrivateVirtualDisplay | null;
29
+ /** Bounded, secret-free reason an opted-in private display is not in use. */
30
+ displayNotice: string | null;
14
31
  }
15
32
  interface RuntimeOwnerOptions {
16
33
  initialUrl?: string;
17
34
  headed?: boolean;
35
+ /**
36
+ * Trusted opt-in for a private Linux Xvfb display. When omitted, each runtime creation reads
37
+ * PASEO_SHARED_BROWSER_XVFB from this process's environment and enables it only for "1".
38
+ * Explicit false overrides the environment.
39
+ */
40
+ virtualDisplay?: boolean;
41
+ /** Encoded video is opt-in; defaults to PASEO_SHARED_BROWSER_VIDEO=1 in the daemon environment. */
42
+ nativeVideo?: boolean;
18
43
  }
19
44
 
20
45
  function paseoHome(): string {
21
46
  return process.env.PASEO_HOME ?? join(homedir(), ".paseo");
22
47
  }
23
48
 
49
+ /** Retain omission for legacy/native held-key inference; reject malformed explicit masks. */
50
+ function optionalMouseModifiers(data: Record<string, JsonValue>): [] | [number] {
51
+ return data.modifiers === undefined
52
+ ? []
53
+ : [browserGestureKeySchema.shape.modifiers.parse(data.modifiers)];
54
+ }
55
+
56
+ /** Give each page its own CDP attachment and media state within one Chromium profile. */
57
+ async function tabRuntime(owned: OwnedRuntime, targetId: string): Promise<AgentBrowserRuntime> {
58
+ const existing = owned.tabs.get(targetId);
59
+ if (existing) return existing;
60
+
61
+ const attaching = (async () => {
62
+ const targets = await owned.runtime.targets();
63
+ if (!targets.some((target) => target.targetId === targetId)) {
64
+ throw new Error("Browser tab is no longer available");
65
+ }
66
+ const runtime = new AgentBrowserRuntime(owned.runtimeOptions);
67
+ try {
68
+ await runtime.attachTarget(targetId);
69
+ return runtime;
70
+ } catch (error) {
71
+ await runtime.shutdown().catch(() => undefined);
72
+ throw error;
73
+ }
74
+ })();
75
+ owned.tabs.set(targetId, attaching);
76
+ try {
77
+ return await attaching;
78
+ } catch (error) {
79
+ if (owned.tabs.get(targetId) === attaching) owned.tabs.delete(targetId);
80
+ throw error;
81
+ }
82
+ }
83
+
24
84
  export async function createRuntimeOwner(
25
85
  options: RuntimeOwnerOptions = {},
26
86
  ): Promise<RuntimeOwner<OwnedRuntime>> {
@@ -48,97 +108,311 @@ export async function createRuntimeOwner(
48
108
  return {
49
109
  async create(workspaceId) {
50
110
  const hash = createHash("sha256").update(workspaceId).digest("hex");
51
- const runtime = new AgentBrowserRuntime({
52
- binaryPath,
53
- executablePath,
54
- profilePath: join(root, "profiles", hash),
55
- ipcDirectory,
56
- session: `ws-${hash.slice(0, 16)}`,
57
- initialUrl: options.initialUrl ?? DEFAULT_BROWSER_URL,
58
- headed: options.headed ?? false,
59
- });
60
- await runtime.launch();
61
- return { runtimeId: randomUUID(), runtime };
111
+ // Explicit headed mode uses the caller's real display. An opted-in hidden Linux mode
112
+ // gets a private virtual mouse so native pointer media survive emulation.
113
+ let display: PrivateVirtualDisplay | null = null;
114
+ let displayNotice: string | null = null;
115
+ const wantsDisplay = options.virtualDisplay ?? process.env.PASEO_SHARED_BROWSER_XVFB === "1";
116
+ if (!options.headed && wantsDisplay) {
117
+ try {
118
+ display = await createPrivateVirtualDisplay();
119
+ if (!display) displayNotice = "Private display unavailable: Xvfb was not found.";
120
+ } catch (error) {
121
+ // The helper has finished cleanup and no Chromium was constructed yet.
122
+ displayNotice = `Private display unavailable: ${
123
+ error instanceof PrivateDisplayError ? error.message : "unexpected startup failure"
124
+ }.`;
125
+ }
126
+ if (displayNotice) {
127
+ // The detached supervisor discards stdio; the notice also reaches the viewer state.
128
+ console.warn(`[shared-browser] ${displayNotice} Using headless mode.`);
129
+ }
130
+ }
131
+ let runtime: AgentBrowserRuntime | null = null;
132
+ try {
133
+ const runtimeOptions: AgentBrowserRuntimeOptions = {
134
+ binaryPath,
135
+ executablePath,
136
+ profilePath: join(root, "profiles", hash),
137
+ ipcDirectory,
138
+ session: `ws-${hash.slice(0, 16)}`,
139
+ initialUrl: options.initialUrl ?? DEFAULT_BROWSER_URL,
140
+ headed: display ? true : (options.headed ?? false),
141
+ nativeVideo: options.nativeVideo ?? process.env.PASEO_SHARED_BROWSER_VIDEO === "1",
142
+ ...(display ? { launchEnvironment: display.launchEnvironment } : {}),
143
+ };
144
+ runtime = new AgentBrowserRuntime(runtimeOptions);
145
+ await runtime.launch();
146
+ display?.assertAvailable();
147
+ const identity = await runtime.identity();
148
+ return {
149
+ runtimeId: randomUUID(),
150
+ runtime,
151
+ rootTargetId: identity.targetId,
152
+ runtimeOptions,
153
+ tabs: new Map(),
154
+ display,
155
+ displayNotice,
156
+ };
157
+ } catch (error) {
158
+ let cleanupFailure: unknown = null;
159
+ try {
160
+ await runtime?.shutdown();
161
+ } catch (failure) {
162
+ cleanupFailure = failure;
163
+ } finally {
164
+ await display?.stop();
165
+ }
166
+ // An unconfirmed cleanup must not hide why creation failed.
167
+ if (cleanupFailure)
168
+ throw new AggregateError(
169
+ [error, cleanupFailure],
170
+ `Runtime creation failed (${error instanceof Error ? error.message : String(error)}) and its cleanup was not confirmed`,
171
+ );
172
+ throw error;
173
+ }
62
174
  },
63
175
  async request(owned, operation, input) {
176
+ owned.display?.assertAvailable();
64
177
  const data =
65
178
  input && typeof input === "object" && !Array.isArray(input)
66
179
  ? (input as Record<string, JsonValue>)
67
180
  : {};
181
+ if (operation === "tabs.list") {
182
+ return (await owned.runtime.targets()).map(({ targetId, title, url }) => ({
183
+ targetId,
184
+ title,
185
+ url,
186
+ }));
187
+ }
188
+ if (operation === "tabs.create") {
189
+ if ((await owned.runtime.targets()).length >= MAX_BROWSER_TABS)
190
+ throw new Error("Browser tab limit reached");
191
+ return { targetId: await owned.runtime.createTarget() };
192
+ }
193
+ if (operation === "tabs.close") {
194
+ const targetId = String(data.targetId);
195
+ const targets = await owned.runtime.targets();
196
+ if (!targets.some((target) => target.targetId === targetId)) {
197
+ throw new Error("Browser tab is no longer available");
198
+ }
199
+ if (targets.length <= 1) {
200
+ throw new Error("The last browser tab cannot be closed");
201
+ }
202
+ const controller = owned.tabs.get(targetId);
203
+ if (controller) await (await controller).shutdown();
204
+ if (owned.rootTargetId === targetId) {
205
+ const successor = targets.find((target) => target.targetId !== targetId)!;
206
+ await owned.runtime.selectTarget(successor.targetId, false);
207
+ owned.rootTargetId = successor.targetId;
208
+ }
209
+ await owned.runtime.closeTarget(targetId);
210
+ owned.tabs.delete(targetId);
211
+ return null;
212
+ }
213
+ const targetId = typeof data.targetId === "string" ? data.targetId : null;
214
+ // Keep the launch page on its original CDP session. A second session on
215
+ // the same target can reset emulation and interrupt its video source.
216
+ const runtime =
217
+ targetId && (targetId !== owned.rootTargetId || owned.tabs.has(targetId))
218
+ ? await tabRuntime(owned, targetId)
219
+ : owned.runtime;
220
+ const gestureId = typeof data.gestureId === "string" ? data.gestureId : undefined;
68
221
  switch (operation) {
69
- case "identity":
70
- return owned.runtime.identity() as unknown as JsonValue;
222
+ case "identity": {
223
+ const identity = await runtime.identity();
224
+ return {
225
+ ...identity,
226
+ ...(owned.displayNotice ? { displayNotice: owned.displayNotice } : {}),
227
+ } as unknown as JsonValue;
228
+ }
71
229
  case "state":
72
- return owned.runtime.state() as unknown as JsonValue;
230
+ if (data.timeoutMs === undefined) return runtime.state() as unknown as JsonValue;
231
+ if (typeof data.timeoutMs !== "number")
232
+ throw new Error("Invalid browser metadata timeout");
233
+ return runtime.state({ timeoutMs: data.timeoutMs }) as unknown as JsonValue;
73
234
  case "navigate":
74
- await owned.runtime.navigate(String(data.url));
235
+ await runtime.navigate(String(data.url));
75
236
  return null;
76
237
  case "back":
77
- await owned.runtime.back();
238
+ await runtime.back();
78
239
  return null;
79
240
  case "forward":
80
- await owned.runtime.forward();
241
+ await runtime.forward();
81
242
  return null;
82
243
  case "reload":
83
- await owned.runtime.reload();
244
+ await runtime.reload();
84
245
  return null;
85
246
  case "emulate":
86
- await owned.runtime.emulate(data as unknown as BrowserViewport);
247
+ await runtime.emulate(data as unknown as BrowserViewport);
248
+ return null;
249
+ case "capture.density":
250
+ await runtime.setCaptureDensity(Number(data.density), Number(data.deviceScaleFactor));
87
251
  return null;
88
252
  case "screencast.start":
89
- await owned.runtime.startScreencast(Number(data.quality));
253
+ await runtime.startScreencast(Number(data.quality));
90
254
  return null;
91
255
  case "screencast.stop":
92
- await owned.runtime.stopScreencast();
256
+ await runtime.stopScreencast();
257
+ return null;
258
+ case "video.read":
259
+ return (await runtime.readVideo({
260
+ quality: data.quality === "low" || data.quality === "medium" ? data.quality : "high",
261
+ streamId: typeof data.streamId === "string" ? data.streamId : null,
262
+ requestKeyFrame: data.requestKeyFrame === true,
263
+ ...(data.bitrate === undefined
264
+ ? {}
265
+ : { bitrate: videoBitrateSchema.parse(data.bitrate) }),
266
+ ...(data.fps === undefined ? {} : { fps: videoFrameRateSchema.parse(data.fps) }),
267
+ afterSequence: Number(data.afterSequence ?? 0),
268
+ waitMs: Number(data.waitMs ?? 250),
269
+ })) as unknown as JsonValue;
270
+ case "video.stop":
271
+ await runtime.stopVideo();
272
+ return null;
273
+ case "video.invalidate":
274
+ runtime.invalidateQueuedVideoFrames();
93
275
  return null;
94
276
  case "frame":
95
- return (await owned.runtime.frame(
277
+ return (await runtime.frame(
96
278
  Number(data.maxBytes),
97
279
  Number(data.quality),
98
280
  Number(data.waitMs),
99
281
  )) as unknown as JsonValue;
100
282
  case "mouse.move":
101
- await owned.runtime.mouseMove(Number(data.x), Number(data.y));
283
+ await runtime.mouseMove(
284
+ Number(data.x),
285
+ Number(data.y),
286
+ gestureId,
287
+ ...optionalMouseModifiers(data),
288
+ );
102
289
  return null;
103
290
  case "mouse.down":
104
- await owned.runtime.mouseDown(
291
+ await runtime.mouseDown(
105
292
  Number(data.x),
106
293
  Number(data.y),
107
294
  String(data.button) as "left" | "middle" | "right",
108
295
  Number(data.clickCount),
296
+ gestureId,
297
+ ...optionalMouseModifiers(data),
109
298
  );
110
299
  return null;
111
300
  case "mouse.up":
112
- await owned.runtime.mouseUp(
301
+ await runtime.mouseUp(
113
302
  Number(data.x),
114
303
  Number(data.y),
115
304
  String(data.button) as "left" | "middle" | "right",
116
305
  Number(data.clickCount),
306
+ gestureId,
307
+ ...optionalMouseModifiers(data),
117
308
  );
118
309
  return null;
119
310
  case "mouse.wheel":
120
- await owned.runtime.wheel(
311
+ await runtime.wheel(
121
312
  Number(data.x),
122
313
  Number(data.y),
123
314
  Number(data.deltaX),
124
315
  Number(data.deltaY),
316
+ gestureId,
317
+ ...optionalMouseModifiers(data),
125
318
  );
126
319
  return null;
320
+ case "mouse.leave":
321
+ if (!gestureId) throw new Error("Gesture identity is required");
322
+ await runtime.mouseLeave(gestureId);
323
+ return null;
324
+ case "input.begin":
325
+ if (!gestureId) throw new Error("Gesture identity is required");
326
+ if (typeof data.expectedInputGeneration !== "string") {
327
+ throw new Error("Expected native input generation is required");
328
+ }
329
+ await runtime.beginLiveInput(gestureId, data.expectedInputGeneration);
330
+ return null;
331
+ case "input.check":
332
+ if (!gestureId) throw new Error("Gesture identity is required");
333
+ await runtime.assertLiveInput(gestureId);
334
+ return null;
335
+ case "input.end":
336
+ if (!gestureId) throw new Error("Gesture identity is required");
337
+ await runtime.endLiveInput(gestureId);
338
+ return null;
339
+ case "cursor":
340
+ if (!gestureId) throw new Error("Gesture identity is required");
341
+ return runtime.cursorAt(Number(data.x), Number(data.y), gestureId);
342
+ case "touch": {
343
+ if (!gestureId || !Array.isArray(data.points) || data.points.length > 5)
344
+ throw new Error("Invalid touch input");
345
+ const points = data.points.map((value) => {
346
+ if (!value || typeof value !== "object" || Array.isArray(value))
347
+ throw new Error("Invalid touch point");
348
+ const id = value.id;
349
+ if (
350
+ typeof id !== "number" ||
351
+ typeof value.x !== "number" ||
352
+ typeof value.y !== "number"
353
+ )
354
+ throw new Error("Invalid touch point numbers");
355
+ if (!Number.isSafeInteger(id) || id < 0 || id > 2_147_483_647)
356
+ throw new Error("Invalid touch identifier");
357
+ return { x: Number(value.x), y: Number(value.y), id };
358
+ });
359
+ if (new Set(points.map((point) => point.id)).size !== points.length)
360
+ throw new Error("Duplicate touch identifiers");
361
+ const type = data.type;
362
+ if (type !== "start" && type !== "move" && type !== "end" && type !== "cancel")
363
+ throw new Error("Invalid touch phase");
364
+ if ((type === "end" || type === "cancel") !== (points.length === 0))
365
+ throw new Error("Invalid touch phase points");
366
+ const nativeType =
367
+ type === "start"
368
+ ? "touchStart"
369
+ : type === "move"
370
+ ? "touchMove"
371
+ : type === "end"
372
+ ? "touchEnd"
373
+ : "touchCancel";
374
+ await runtime.touch(nativeType, points, gestureId);
375
+ return null;
376
+ }
377
+ case "input.key":
378
+ if (!gestureId) throw new Error("Gesture identity is required");
379
+ await runtime.dispatchKey(browserGestureKeySchema.parse(data.event), gestureId);
380
+ return null;
381
+ case "input.text":
382
+ if (
383
+ !gestureId ||
384
+ typeof data.text !== "string" ||
385
+ data.text.length < 1 ||
386
+ data.text.length > 16_000
387
+ ) {
388
+ throw new Error("Invalid committed text input");
389
+ }
390
+ await runtime.insertText(data.text, gestureId);
391
+ return null;
127
392
  case "text.insert":
128
- await owned.runtime.insertText(String(data.text));
393
+ await runtime.insertText(String(data.text), gestureId);
129
394
  return null;
130
395
  case "key.down":
131
- await owned.runtime.keyDown(String(data.key));
396
+ await runtime.keyDown(String(data.key), String(data.key), gestureId);
132
397
  return null;
133
398
  case "key.up":
134
- await owned.runtime.keyUp(String(data.key));
399
+ await runtime.keyUp(String(data.key), String(data.key), gestureId);
135
400
  return null;
136
401
  default:
137
402
  throw new Error(`Unknown browser runtime operation: ${operation}`);
138
403
  }
139
404
  },
140
405
  async stop(owned) {
141
- await owned.runtime.shutdown();
406
+ try {
407
+ await Promise.allSettled(
408
+ [...owned.tabs.values()].map(async (controller) =>
409
+ (await controller).shutdown().catch(() => undefined),
410
+ ),
411
+ );
412
+ await owned.runtime.shutdown();
413
+ } finally {
414
+ await owned.display?.stop();
415
+ }
142
416
  },
143
417
  };
144
418
  }
@@ -0,0 +1,249 @@
1
+ /** Fresh metadata reads overlap CDP latency but never accept mixed native identities. */
2
+ import { describe, expect, it, vi } from "vitest";
3
+ import { AgentBrowserRuntime } from "./agent-browser-runtime";
4
+
5
+ function fixture() {
6
+ const runtime = new AgentBrowserRuntime({
7
+ binaryPath: "/tmp/unlaunched-browser",
8
+ executablePath: "/tmp/unlaunched-chromium",
9
+ profilePath: "/tmp/uncreated-state-profile",
10
+ ipcDirectory: "/tmp/uncreated-state-ipc",
11
+ session: "state-test",
12
+ });
13
+ const calls: string[] = [];
14
+ const history = Promise.withResolvers<{
15
+ currentIndex: number;
16
+ entries: { url: string; title: string }[];
17
+ }>();
18
+ const metadata = Promise.withResolvers<{
19
+ result: { value?: { url?: string; title?: string } };
20
+ }>();
21
+ const retryHistory = Promise.withResolvers<Awaited<typeof history.promise>>();
22
+ const retryMetadata = Promise.withResolvers<Awaited<typeof metadata.promise>>();
23
+ const timeouts: number[] = [];
24
+ let onRead: (method: string, attempt: number) => void = () => {};
25
+ const page = {
26
+ send: (method: string, _input: unknown, options: { timeoutMs: number }) => {
27
+ const attempt = calls.filter((call) => call === method).length;
28
+ calls.push(method);
29
+ timeouts.push(options.timeoutMs);
30
+ onRead(method, attempt);
31
+ if (method === "Page.getNavigationHistory") {
32
+ return attempt === 0 ? history.promise : retryHistory.promise;
33
+ }
34
+ return attempt === 0 ? metadata.promise : retryMetadata.promise;
35
+ },
36
+ };
37
+ const control = runtime as unknown as {
38
+ page: typeof page;
39
+ attachmentGeneration: number;
40
+ documentGeneration: number;
41
+ stopping: boolean;
42
+ requirePage(): Promise<typeof page>;
43
+ };
44
+ control.page = page;
45
+ control.requirePage = async () => page;
46
+ return {
47
+ runtime,
48
+ control,
49
+ page,
50
+ calls,
51
+ history,
52
+ metadata,
53
+ retryHistory,
54
+ retryMetadata,
55
+ timeouts,
56
+ intercept: (callback: typeof onRead) => {
57
+ onRead = callback;
58
+ },
59
+ };
60
+ }
61
+ const entries = [
62
+ { url: "https://example.test/previous", title: "Previous" },
63
+ { url: "https://example.test/current", title: "History title" },
64
+ ];
65
+
66
+ describe("fresh parallel native page metadata", () => {
67
+ it("discards both old reads and returns one fresh pair after a same-page document change", async () => {
68
+ const f = fixture();
69
+ const pending = f.runtime.state();
70
+ await Promise.resolve();
71
+ f.control.documentGeneration++;
72
+ f.retryHistory.resolve({
73
+ currentIndex: 0,
74
+ entries: [{ url: "https://example.test/new", title: "New history" }],
75
+ });
76
+ f.retryMetadata.resolve({
77
+ result: { value: { url: "https://example.test/new#ready", title: "New DOM" } },
78
+ });
79
+ f.history.resolve({ currentIndex: 1, entries });
80
+ f.metadata.resolve({
81
+ result: { value: { url: "https://example.test/old", title: "Old DOM" } },
82
+ });
83
+ expect(await pending).toEqual({
84
+ url: "https://example.test/new#ready",
85
+ title: "New DOM",
86
+ canGoBack: false,
87
+ canGoForward: false,
88
+ inputGeneration: "0:1",
89
+ });
90
+ expect(f.calls).toEqual([
91
+ "Page.getNavigationHistory",
92
+ "Runtime.evaluate",
93
+ "Page.getNavigationHistory",
94
+ "Runtime.evaluate",
95
+ ]);
96
+ });
97
+
98
+ it("starts both independent reads before either resolves and retains native history facts", async () => {
99
+ const f = fixture();
100
+ const pending = f.runtime.state();
101
+ await Promise.resolve();
102
+ expect(f.calls).toEqual(["Page.getNavigationHistory", "Runtime.evaluate"]);
103
+ f.metadata.resolve({
104
+ result: { value: { url: "https://example.test/current#hash", title: "Fresh title" } },
105
+ });
106
+ f.history.resolve({ currentIndex: 1, entries });
107
+ expect(await pending).toMatchObject({
108
+ url: "https://example.test/current#hash",
109
+ title: "Fresh title",
110
+ canGoBack: true,
111
+ canGoForward: false,
112
+ });
113
+ });
114
+ it("preserves existing native-history fallback when DOM values are absent", async () => {
115
+ const f = fixture();
116
+ const pending = f.runtime.state();
117
+ f.metadata.resolve({ result: {} });
118
+ f.history.resolve({ currentIndex: 0, entries });
119
+ expect(await pending).toMatchObject({
120
+ url: entries[0]!.url,
121
+ title: "Previous",
122
+ canGoBack: false,
123
+ canGoForward: true,
124
+ });
125
+ });
126
+ it("refuses attachment, page replacement or shutdown during either pending observation", async () => {
127
+ for (const kind of ["attachment", "page", "shutdown"] as const) {
128
+ const f = fixture();
129
+ const pending = f.runtime.state();
130
+ await Promise.resolve();
131
+ if (kind === "shutdown") f.control.stopping = true;
132
+ if (kind === "attachment") f.control.attachmentGeneration++;
133
+ if (kind === "page")
134
+ f.control.page = { send: () => Promise.resolve({}) } as unknown as typeof f.page;
135
+ f.history.resolve({ currentIndex: 1, entries });
136
+ f.metadata.resolve({ result: { value: { title: "Obsolete document" } } });
137
+ await expect(pending).rejects.toThrow(
138
+ kind === "shutdown" ? "shutting down" : "metadata changed",
139
+ );
140
+ expect(f.calls).toHaveLength(2);
141
+ }
142
+ });
143
+ it("refuses repeated document changes after exactly two read pairs", async () => {
144
+ const f = fixture();
145
+ f.intercept((method, attempt) => {
146
+ if (method === "Runtime.evaluate" && attempt === 1) f.control.documentGeneration++;
147
+ });
148
+ const pending = f.runtime.state();
149
+ await Promise.resolve();
150
+ f.control.documentGeneration++;
151
+ f.history.resolve({ currentIndex: 1, entries });
152
+ f.metadata.resolve({ result: {} });
153
+ f.retryHistory.resolve({ currentIndex: 0, entries });
154
+ f.retryMetadata.resolve({ result: {} });
155
+ await expect(pending).rejects.toThrow("metadata changed");
156
+ expect(f.calls).toHaveLength(4);
157
+ });
158
+
159
+ it("shares the original timeout budget across both pairs without restarting attachment", async () => {
160
+ const f = fixture();
161
+ let clock = 1_000;
162
+ const monotonic = vi.spyOn(performance, "now").mockImplementation(() => clock);
163
+ try {
164
+ const pending = f.runtime.state({ timeoutMs: 1_000 });
165
+ await Promise.resolve();
166
+ clock += 700;
167
+ f.control.documentGeneration++;
168
+ f.history.resolve({ currentIndex: 1, entries });
169
+ f.metadata.resolve({ result: {} });
170
+ f.retryHistory.resolve({ currentIndex: 0, entries });
171
+ f.retryMetadata.resolve({ result: {} });
172
+ await pending;
173
+ expect(f.timeouts).toHaveLength(4);
174
+ expect(f.timeouts.slice(0, 2).every((timeout) => timeout <= 1_000)).toBe(true);
175
+ expect(f.timeouts.slice(2).every((timeout) => timeout <= 300)).toBe(true);
176
+ } finally {
177
+ monotonic.mockRestore();
178
+ }
179
+ });
180
+
181
+ it("does not begin a second pair after the original budget expires", async () => {
182
+ const f = fixture();
183
+ let clock = 1_000;
184
+ const monotonic = vi.spyOn(performance, "now").mockImplementation(() => clock);
185
+ try {
186
+ const pending = f.runtime.state({ timeoutMs: 1_000 });
187
+ await Promise.resolve();
188
+ clock += 1_000;
189
+ f.control.documentGeneration++;
190
+ f.history.resolve({ currentIndex: 1, entries });
191
+ f.metadata.resolve({ result: {} });
192
+ await expect(pending).rejects.toThrow("metadata observation timed out");
193
+ expect(f.calls).toHaveLength(2);
194
+ } finally {
195
+ monotonic.mockRestore();
196
+ }
197
+ });
198
+
199
+ for (const phase of ["first", "second"] as const) {
200
+ it(`refuses successful held ${phase} replies at or after the original deadline`, async () => {
201
+ const f = fixture();
202
+ let clock = 1_000;
203
+ const monotonic = vi.spyOn(performance, "now").mockImplementation(() => clock);
204
+ try {
205
+ const secondStarted = Promise.withResolvers<void>();
206
+ f.intercept((method, attempt) => {
207
+ if (method === "Runtime.evaluate" && attempt === 1) secondStarted.resolve();
208
+ });
209
+ const pending = f.runtime.state({ timeoutMs: 1_000 });
210
+ await Promise.resolve();
211
+ if (phase === "second") {
212
+ clock += 700;
213
+ f.control.documentGeneration++;
214
+ f.history.resolve({ currentIndex: 1, entries });
215
+ f.metadata.resolve({ result: {} });
216
+ await secondStarted.promise;
217
+ expect(f.calls).toHaveLength(4);
218
+ clock += 301;
219
+ f.retryHistory.resolve({ currentIndex: 0, entries });
220
+ f.retryMetadata.resolve({ result: { value: { title: "Late fresh metadata" } } });
221
+ } else {
222
+ clock += 1_000;
223
+ f.history.resolve({ currentIndex: 1, entries });
224
+ f.metadata.resolve({ result: { value: { title: "Late metadata" } } });
225
+ }
226
+ await expect(pending).rejects.toThrow("metadata observation timed out");
227
+ expect(f.calls).toHaveLength(phase === "second" ? 4 : 2);
228
+ } finally {
229
+ monotonic.mockRestore();
230
+ }
231
+ });
232
+ }
233
+
234
+ it("propagates failed native reads rather than reusing earlier metadata", async () => {
235
+ for (const failed of ["history", "metadata"]) {
236
+ const f = fixture();
237
+ const pending = f.runtime.state();
238
+ const refusal = expect(pending).rejects.toThrow("Read unavailable");
239
+ await Promise.resolve();
240
+ f.control.documentGeneration++;
241
+ if (failed === "history") f.history.reject(new Error("Read unavailable"));
242
+ else f.history.resolve({ currentIndex: 1, entries });
243
+ if (failed === "metadata") f.metadata.reject(new Error("Read unavailable"));
244
+ else f.metadata.resolve({ result: {} });
245
+ await refusal;
246
+ expect(f.calls).toHaveLength(2);
247
+ }
248
+ });
249
+ });