@omercnet/paseo-shared-browser 1.2.2 → 2.0.0-next.249.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/OVERVIEW.md +2 -2
  2. package/README.md +316 -27
  3. package/REMOTE_BROWSER_RESEARCH.md +395 -0
  4. package/STREAMING_DESIGN.md +196 -0
  5. package/client/browser-auto-capture-density.test.ts +102 -0
  6. package/client/browser-auto-capture-density.ts +30 -0
  7. package/client/browser-canvas-input.test.ts +198 -0
  8. package/client/browser-canvas-input.ts +202 -0
  9. package/client/browser-canvas-keyboard-web.test.ts +113 -0
  10. package/client/browser-canvas-keyboard-web.ts +230 -0
  11. package/client/browser-canvas-keyboard.test.ts +123 -0
  12. package/client/browser-canvas-keyboard.ts +126 -0
  13. package/client/browser-canvas-layout.test.ts +109 -0
  14. package/client/browser-canvas-layout.ts +51 -0
  15. package/client/browser-canvas-viewport-web.test.ts +77 -0
  16. package/client/browser-canvas-viewport-web.ts +63 -0
  17. package/client/browser-canvas-viewport.tsx +152 -0
  18. package/client/browser-capture-cadence.test.ts +9 -0
  19. package/client/browser-capture-cadence.ts +6 -0
  20. package/client/browser-capture-density-controls.tsx +75 -0
  21. package/client/browser-chrome.tsx +234 -0
  22. package/client/browser-client-presence.ts +184 -0
  23. package/client/browser-compose-controls.test.tsx +275 -0
  24. package/client/browser-compose-controls.tsx +174 -0
  25. package/client/browser-control-button.tsx +109 -0
  26. package/client/browser-control-dialog.tsx +71 -0
  27. package/client/browser-control-request.test.ts +27 -0
  28. package/client/browser-control-request.ts +57 -0
  29. package/client/browser-control-tooltip-web.ts +10 -0
  30. package/client/browser-dialog-body.tsx +22 -0
  31. package/client/browser-emulation-mode.test.ts +75 -0
  32. package/client/browser-emulation-mode.ts +98 -0
  33. package/client/browser-fill-viewport.test.ts +45 -0
  34. package/client/browser-fill-viewport.ts +28 -0
  35. package/client/browser-frame-buffer.test.ts +157 -0
  36. package/client/browser-frame-buffer.ts +148 -0
  37. package/client/browser-frame-image.tsx +62 -0
  38. package/client/browser-image-load.test.ts +23 -0
  39. package/client/browser-image-load.ts +26 -0
  40. package/client/browser-input-queue.test.ts +1254 -0
  41. package/client/browser-input-queue.ts +519 -0
  42. package/client/browser-menu-placement.test.ts +90 -0
  43. package/client/browser-menu-placement.ts +78 -0
  44. package/client/browser-panel-styles.ts +384 -0
  45. package/client/browser-quality-controls.test.ts +26 -0
  46. package/client/browser-quality-controls.tsx +139 -0
  47. package/client/browser-resolution-picker.tsx +189 -0
  48. package/client/browser-tabs.tsx +117 -0
  49. package/client/browser-toolbar-menu-web.test.ts +185 -0
  50. package/client/browser-toolbar-menu-web.ts +159 -0
  51. package/client/browser-toolbar-menu.tsx +255 -0
  52. package/client/browser-toolbar-submenu.tsx +93 -0
  53. package/client/browser-video-bytes.test.ts +44 -0
  54. package/client/browser-video-currency.ts +45 -0
  55. package/client/browser-video-decoder.test.ts +473 -0
  56. package/client/browser-video-decoder.ts +311 -0
  57. package/client/browser-video-surface.tsx +21 -0
  58. package/client/browser-video-visibility.test.ts +45 -0
  59. package/client/browser-viewer-recovery.test.ts +111 -0
  60. package/client/browser-viewer-recovery.ts +78 -0
  61. package/client/browser.tsx +1650 -1579
  62. package/client/favorite-star.tsx +23 -0
  63. package/client/frame-lifecycle.ts +62 -59
  64. package/client/sidebar.tsx +1 -2
  65. package/client/use-browser-auto-capture-density.test.ts +104 -0
  66. package/client/use-browser-auto-capture-density.ts +43 -0
  67. package/client/use-browser-canvas-input.ts +375 -0
  68. package/client/use-browser-capture-density.ts +87 -0
  69. package/client/use-browser-emulation-mode.ts +56 -0
  70. package/client/use-browser-fill-viewport.test.ts +108 -0
  71. package/client/use-browser-fill-viewport.ts +62 -0
  72. package/client/use-browser-frame-buffer.ts +66 -0
  73. package/client/use-browser-image-capture.ts +139 -0
  74. package/client/use-browser-scoped-mutation.test.ts +118 -0
  75. package/client/use-browser-scoped-mutation.ts +59 -0
  76. package/client/use-browser-video.test.ts +1060 -0
  77. package/client/use-browser-video.ts +481 -0
  78. package/client/use-browser-viewer-recovery.ts +26 -0
  79. package/client/use-resolution-favorites.ts +90 -0
  80. package/client/web.test.ts +125 -0
  81. package/client/web.ts +405 -0
  82. package/index.client.tsx +0 -10
  83. package/index.server.ts +32 -4
  84. package/npm-shrinkwrap.json +9 -9
  85. package/package.json +10 -5
  86. package/paseo-plugin.json +1 -1
  87. package/scripts/prepare-runtime.mjs +1 -1
  88. package/server/agent-browser-runtime.ts +1601 -314
  89. package/server/browser-cursor.test.ts +64 -0
  90. package/server/browser-cursor.ts +69 -0
  91. package/server/browser-frame-context.ts +22 -0
  92. package/server/browser-frame-epoch.test.ts +95 -0
  93. package/server/browser-frame-rejection.ts +17 -0
  94. package/server/browser-gesture.ts +120 -0
  95. package/server/browser-operation-diagnostics.test.ts +79 -0
  96. package/server/browser-operation-diagnostics.ts +103 -0
  97. package/server/browser-policy.ts +1316 -151
  98. package/server/browser-video-input-authority.test.ts +211 -0
  99. package/server/browser-video-policy.test.ts +564 -0
  100. package/server/browser-video-socket.test.ts +368 -0
  101. package/server/browser.ts +151 -16
  102. package/server/capture-transport-policy.test.ts +105 -0
  103. package/server/capture-transport-policy.ts +154 -0
  104. package/server/capture-transport-runtime.test.ts +213 -0
  105. package/server/daemon-launch-chain.test.ts +70 -0
  106. package/server/daemon-shutdown.test.ts +289 -0
  107. package/server/discrete-input-freshness.test.ts +364 -0
  108. package/server/discrete-input-native-lifetime.test.ts +150 -0
  109. package/server/display-loss-recovery.test.ts +294 -0
  110. package/server/emulation-restoration.test.ts +262 -0
  111. package/server/input-cleanup.ts +23 -0
  112. package/server/input-generation.test.ts +31 -0
  113. package/server/input-generation.ts +36 -0
  114. package/server/jpeg-capture-demand.test.ts +76 -0
  115. package/server/jpeg-capture-demand.ts +98 -0
  116. package/server/jpeg-frame.ts +29 -0
  117. package/server/keyboard-input.test.ts +61 -0
  118. package/server/keyboard-input.ts +104 -0
  119. package/server/live-input-policy.test.ts +958 -0
  120. package/server/live-input-runtime.test.ts +732 -0
  121. package/server/live-input-supervisor.test.ts +147 -0
  122. package/server/mcp-entry.ts +44 -2
  123. package/server/native-video-backpressure.test.ts +228 -0
  124. package/server/native-video-buffer.test.ts +141 -0
  125. package/server/native-video-buffer.ts +114 -0
  126. package/server/native-video-capture.test.ts +599 -0
  127. package/server/native-video-capture.ts +497 -0
  128. package/server/native-video-extension-source.test.ts +201 -0
  129. package/server/native-video-extension-source.ts +364 -0
  130. package/server/native-video-extension.test.ts +41 -0
  131. package/server/native-video-extension.ts +83 -0
  132. package/server/native-video-packet.ts +9 -0
  133. package/server/native-video-settings.test.ts +245 -0
  134. package/server/navigation-budget.ts +8 -0
  135. package/server/navigation-commit.ts +125 -0
  136. package/server/navigation-policy.test.ts +153 -0
  137. package/server/process-identity.ts +58 -0
  138. package/server/reattach-held-input.test.ts +143 -0
  139. package/server/runtime-frame-reuse.test.ts +295 -0
  140. package/server/runtime-jpeg-demand.test.ts +200 -0
  141. package/server/runtime-jpeg-shutdown.test.ts +210 -0
  142. package/server/runtime-jpeg-source-order.test.ts +201 -0
  143. package/server/runtime-launch-environment.test.ts +111 -0
  144. package/server/runtime-navigation.test.ts +327 -0
  145. package/server/runtime-owner-display.test.ts +152 -0
  146. package/server/runtime-owner.ts +340 -31
  147. package/server/runtime-page-state.test.ts +249 -0
  148. package/server/runtime-protocol.ts +32 -2
  149. package/server/runtime-target-retention.test.ts +181 -0
  150. package/server/runtime-video-opt-in.test.ts +82 -0
  151. package/server/runtime-video.test.ts +444 -0
  152. package/server/screencast-source-time.ts +46 -0
  153. package/server/stale-wheel-admission.test.ts +136 -0
  154. package/server/supervisor-client.ts +28 -4
  155. package/server/supervisor-heartbeat.test.ts +80 -0
  156. package/server/supervisor.ts +553 -55
  157. package/server/tab-close-confirmation.test.ts +97 -0
  158. package/server/video-source-recovery.test.ts +20 -0
  159. package/server/video-source-recovery.ts +64 -0
  160. package/server/viewer-capture-lifetime.ts +51 -0
  161. package/server/virtual-display.test.ts +184 -0
  162. package/server/virtual-display.ts +236 -0
  163. package/shared/browser-display-preferences.ts +46 -0
  164. package/shared/browser-video.ts +83 -0
  165. package/shared/browser.ts +286 -65
  166. package/shared/capture-density.ts +16 -0
  167. package/shared/capture-settings.ts +17 -0
  168. package/shared/device-presets.ts +90 -0
  169. package/shared/resolution-menu.test.ts +120 -0
  170. package/shared/resolution-menu.ts +72 -0
  171. package/shared/video-settings.test.ts +38 -0
  172. package/shared/video-settings.ts +34 -0
  173. package/shared/viewport-limits.ts +5 -0
@@ -0,0 +1,262 @@
1
+ import { EventEmitter } from "node:events";
2
+ import { describe, expect, it } from "vitest";
3
+ import { AgentBrowserRuntime, type DeviceEmulation } from "./agent-browser-runtime";
4
+ import { type CdpConnection, CdpSession } from "./cdp";
5
+
6
+ /** Protocol fixture exercises real attachment/configuration methods without a browser process. */
7
+ class ConnectionFixture extends EventEmitter {
8
+ isOpen = true;
9
+ nextSession = 0;
10
+ calls: { method: string; params: Record<string, unknown>; sessionId: string | undefined }[] = [];
11
+ intercept?: (method: string, sessionId?: string) => Promise<void>;
12
+
13
+ async send<T>(
14
+ method: string,
15
+ params: Record<string, unknown> = {},
16
+ options: { sessionId?: string } = {},
17
+ ): Promise<T> {
18
+ this.calls.push({ method, params, sessionId: options.sessionId });
19
+ await this.intercept?.(method, options.sessionId);
20
+ if (method === "Target.getTargets") {
21
+ return {
22
+ targetInfos: ["a", "b"].map((targetId) => ({
23
+ targetId,
24
+ type: "page",
25
+ url: "about:blank",
26
+ title: targetId,
27
+ })),
28
+ } as T;
29
+ }
30
+ if (method === "Target.attachToTarget") return { sessionId: `new-${++this.nextSession}` } as T;
31
+ if (method === "Page.getNavigationHistory") return { currentIndex: 0, entries: [] } as T;
32
+ return {} as T;
33
+ }
34
+
35
+ close(): void {
36
+ this.isOpen = false;
37
+ }
38
+ }
39
+
40
+ interface RuntimeControl {
41
+ connection: CdpConnection;
42
+ page: CdpSession | null;
43
+ targetId: string | null;
44
+ viewport: DeviceEmulation;
45
+ emulationAppliedPage: CdpSession | null;
46
+ screencastActive: boolean;
47
+ screencastFrame: unknown;
48
+ attachmentGeneration: number;
49
+ assertVersion(): Promise<void>;
50
+ connectCdp(targetId?: string): Promise<void>;
51
+ requirePage(): Promise<CdpSession>;
52
+ reattachPageForScreencast(previous: CdpSession): Promise<CdpSession>;
53
+ onScreencastFrame(page: CdpSession, event: unknown): void;
54
+ }
55
+
56
+ const pixel: DeviceEmulation = {
57
+ width: 412,
58
+ height: 839,
59
+ deviceScaleFactor: 2.625,
60
+ captureScale: 2,
61
+ mobile: true,
62
+ touch: true,
63
+ screenWidth: 430,
64
+ screenHeight: 900,
65
+ userAgent: "fixture-mobile-agent",
66
+ platform: "Android",
67
+ };
68
+
69
+ function fixture() {
70
+ const runtime = new AgentBrowserRuntime({
71
+ binaryPath: "/tmp/unlaunched-browser",
72
+ executablePath: "/tmp/unlaunched-chromium",
73
+ profilePath: "/tmp/uncreated-profile",
74
+ ipcDirectory: "/tmp/uncreated-ipc",
75
+ session: "emulation-test",
76
+ });
77
+ const connection = new ConnectionFixture();
78
+ const typedConnection = connection as unknown as CdpConnection;
79
+ const previous = new CdpSession(typedConnection, "previous", "a");
80
+ const control = runtime as unknown as RuntimeControl;
81
+ Object.assign(control, {
82
+ connection: typedConnection,
83
+ page: previous,
84
+ targetId: "a",
85
+ viewport: { ...pixel },
86
+ emulationAppliedPage: previous,
87
+ });
88
+ return { runtime, connection, previous, control };
89
+ }
90
+
91
+ function assertDeviceRestored(connection: ConnectionFixture, sessionId: string) {
92
+ const calls = connection.calls.filter((call) => call.sessionId === sessionId);
93
+ expect(
94
+ calls.find((call) => call.method === "Emulation.setDeviceMetricsOverride")?.params,
95
+ ).toEqual({
96
+ width: 412,
97
+ height: 839,
98
+ deviceScaleFactor: 2.625,
99
+ mobile: true,
100
+ screenWidth: 430,
101
+ screenHeight: 900,
102
+ screenOrientation: { type: "portraitPrimary", angle: 0 },
103
+ });
104
+ expect(
105
+ calls.find((call) => call.method === "Emulation.setTouchEmulationEnabled")?.params,
106
+ ).toEqual({ enabled: true, maxTouchPoints: 5 });
107
+ // Target-scoped: lets a page behind another tab acknowledge input without being raised.
108
+ expect(
109
+ calls.find((call) => call.method === "Emulation.setFocusEmulationEnabled")?.params,
110
+ ).toEqual({ enabled: true });
111
+ expect(calls.find((call) => call.method === "Emulation.setUserAgentOverride")?.params).toEqual({
112
+ userAgent: "fixture-mobile-agent",
113
+ platform: "Android",
114
+ });
115
+ }
116
+
117
+ describe("session-owned device emulation restoration", () => {
118
+ it("restores full retained settings before publishing a newly selected target", async () => {
119
+ const { runtime, connection, control } = fixture();
120
+ const publications: (CdpSession | null)[] = [];
121
+ connection.intercept = async (method) => {
122
+ if (method.startsWith("Emulation.")) publications.push(control.page);
123
+ };
124
+ await runtime.selectTarget("b");
125
+ expect(publications).toEqual([null, null, null, null]);
126
+ expect(control.page?.targetId).toBe("b");
127
+ expect(control.emulationAppliedPage).toBe(control.page);
128
+ expect(control.viewport.captureScale).toBe(2);
129
+ assertDeviceRestored(connection, control.page!.sessionId);
130
+ });
131
+
132
+ it("reconnect restores the device on the replacement transport rather than the old connection", async () => {
133
+ const { runtime, connection, control } = fixture();
134
+ const replacement = new ConnectionFixture();
135
+ control.assertVersion = async () => {};
136
+ control.connectCdp = async () => {
137
+ control.connection = replacement as unknown as CdpConnection;
138
+ await runtime.selectTarget("a");
139
+ };
140
+ await runtime.reconnect();
141
+ expect(connection.isOpen).toBe(false);
142
+ expect(control.page?.connection).toBe(control.connection);
143
+ assertDeviceRestored(replacement, control.page!.sessionId);
144
+ });
145
+
146
+ it("screencast retry detaches the old controller before applying replacement metrics", async () => {
147
+ const { connection, control, previous } = fixture();
148
+ const restored = await control.reattachPageForScreencast(previous);
149
+ const detach = connection.calls.findIndex((call) => call.method === "Target.detachFromTarget");
150
+ const metrics = connection.calls.findIndex(
151
+ (call) => call.method === "Emulation.setDeviceMetricsOverride",
152
+ );
153
+ expect(detach).toBeGreaterThanOrEqual(0);
154
+ expect(metrics).toBeGreaterThan(detach);
155
+ expect(control.page).toBe(restored);
156
+ assertDeviceRestored(connection, restored.sessionId);
157
+ });
158
+
159
+ it("refuses frame admission after partial restoration fails", async () => {
160
+ const { runtime, connection, control } = fixture();
161
+ connection.intercept = async (method) => {
162
+ if (method === "Emulation.setTouchEmulationEnabled")
163
+ throw new Error("touch restore rejected");
164
+ };
165
+ await expect(runtime.selectTarget("b")).rejects.toThrow("touch restore rejected");
166
+ expect(control.page).toBeNull();
167
+ expect(control.emulationAppliedPage).toBeNull();
168
+ expect(
169
+ connection.calls.some(
170
+ (call) => call.method === "Target.detachFromTarget" && call.params.sessionId === "new-1",
171
+ ),
172
+ ).toBe(true);
173
+ control.assertVersion = async () => {
174
+ throw new Error("transport unavailable");
175
+ };
176
+ await expect(runtime.frame(1024, 95, 0)).rejects.toThrow("transport unavailable");
177
+ expect(
178
+ connection.calls.some(
179
+ (call) =>
180
+ call.method === "Page.captureScreenshot" || call.method === "Page.startScreencast",
181
+ ),
182
+ ).toBe(false);
183
+ });
184
+
185
+ it("rejects a disconnect while restoring rather than publishing the detached session", async () => {
186
+ const { runtime, connection, control } = fixture();
187
+ connection.intercept = async (method) => {
188
+ if (method === "Emulation.setUserAgentOverride") connection.close();
189
+ };
190
+ await expect(runtime.selectTarget("b")).rejects.toThrow("attachment changed");
191
+ expect(control.page).toBeNull();
192
+ expect(control.emulationAppliedPage).toBeNull();
193
+ });
194
+
195
+ it("repairs the winning attachment after a stale restoring controller is detached", async () => {
196
+ const { runtime, connection, control } = fixture();
197
+ const gate = Promise.withResolvers<void>();
198
+ const reached = Promise.withResolvers<void>();
199
+ connection.intercept = async (method, sessionId) => {
200
+ if (method === "Emulation.setUserAgentOverride" && sessionId === "new-1") {
201
+ reached.resolve();
202
+ await gate.promise;
203
+ }
204
+ };
205
+ const older = runtime.selectTarget("a");
206
+ await reached.promise;
207
+ await runtime.selectTarget("b");
208
+ const winner = control.page;
209
+ expect(control.emulationAppliedPage).toBe(winner);
210
+ gate.resolve();
211
+ await expect(older).rejects.toThrow("attachment changed");
212
+ expect(control.page).toBe(winner);
213
+ expect(control.emulationAppliedPage).toBeNull();
214
+ control.screencastActive = true;
215
+ control.onScreencastFrame(winner!, { data: "ignored", metadata: {}, sessionId: 4 });
216
+ expect(control.screencastFrame).toBeNull();
217
+ expect(await control.requirePage()).toBe(winner);
218
+ expect(control.emulationAppliedPage).toBe(winner);
219
+ assertDeviceRestored(connection, winner!.sessionId);
220
+ });
221
+
222
+ it("refuses retry publication if detaching the prior emulation controller fails", async () => {
223
+ const { connection, control, previous } = fixture();
224
+ connection.intercept = async (method) => {
225
+ if (method === "Target.detachFromTarget") throw new Error("detach outcome unknown");
226
+ };
227
+ await expect(control.reattachPageForScreencast(previous)).rejects.toThrow(
228
+ "detach outcome unknown",
229
+ );
230
+ expect(control.page).toBeNull();
231
+ expect(control.emulationAppliedPage).toBeNull();
232
+ expect(
233
+ connection.calls.some(
234
+ (call) => call.method.startsWith("Emulation.") || call.method === "Page.startScreencast",
235
+ ),
236
+ ).toBe(false);
237
+ });
238
+
239
+ it("invalidates winner readiness after detaching a superseded retry controller", async () => {
240
+ const { runtime, connection, control, previous } = fixture();
241
+ const reached = Promise.withResolvers<void>();
242
+ const gate = Promise.withResolvers<void>();
243
+ let first = true;
244
+ connection.intercept = async (method) => {
245
+ if (method === "Target.attachToTarget" && first) {
246
+ first = false;
247
+ reached.resolve();
248
+ await gate.promise;
249
+ }
250
+ };
251
+ const retry = control.reattachPageForScreencast(previous);
252
+ await reached.promise;
253
+ await runtime.selectTarget("b");
254
+ const winner = control.page;
255
+ gate.resolve();
256
+ expect(await retry).toBe(winner);
257
+ expect(control.page).toBe(winner);
258
+ expect(control.emulationAppliedPage).toBeNull();
259
+ expect(await control.requirePage()).toBe(winner);
260
+ expect(control.emulationAppliedPage).toBe(winner);
261
+ });
262
+ });
@@ -0,0 +1,23 @@
1
+ /** Input cleanup preserves the first failure, including an uncertain native
2
+ * publication. Cleanup always runs once; neither phase is ever retried. */
3
+
4
+ /** Run an input phase and then its release. A cleanup error is exposed only if
5
+ * the input phase succeeded, so a secondary failure cannot hide UNKNOWN_OUTCOME. */
6
+ export async function runWithInputCleanup<Value>(
7
+ operation: () => Promise<Value>,
8
+ cleanup: () => Promise<unknown>,
9
+ ): Promise<Value> {
10
+ let result: { value: Value } | { error: unknown };
11
+ try {
12
+ result = { value: await operation() };
13
+ } catch (error) {
14
+ result = { error };
15
+ }
16
+ try {
17
+ await cleanup();
18
+ } catch (error) {
19
+ if ("value" in result) throw error;
20
+ }
21
+ if ("error" in result) throw result.error;
22
+ return result.value;
23
+ }
@@ -0,0 +1,31 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { formatRuntimeInputGeneration, sameRuntimeInputAttachment } from "./input-generation";
3
+
4
+ describe("private native input generation codec", () => {
5
+ it("admits document changes only within the same exact native attachment", () => {
6
+ expect(
7
+ sameRuntimeInputAttachment(
8
+ formatRuntimeInputGeneration(3, 4),
9
+ formatRuntimeInputGeneration(3, 5),
10
+ ),
11
+ ).toBe(true);
12
+ expect(sameRuntimeInputAttachment("3:4", "4:4")).toBe(false);
13
+ });
14
+
15
+ it("refuses missing, opaque, truncated and unbounded generation formats", () => {
16
+ for (const invalid of [
17
+ null,
18
+ "opaque",
19
+ "3",
20
+ "3:",
21
+ "03:4",
22
+ "3:-1",
23
+ "3:9007199254740992",
24
+ "3:4:5",
25
+ ]) {
26
+ expect(sameRuntimeInputAttachment("3:4", invalid)).toBe(false);
27
+ expect(sameRuntimeInputAttachment(invalid, "3:4")).toBe(false);
28
+ }
29
+ expect(() => formatRuntimeInputGeneration(-1, 0)).toThrow("Invalid");
30
+ });
31
+ });
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Private runtime metadata codec for attachment and document counters. This is
3
+ * produced by agent-browser-runtime and consumed only by browser-policy. Unknown
4
+ * older formats cannot prove same-attachment completion after input publication.
5
+ */
6
+
7
+ /** Serialize native counters without treating document navigation as target replacement. */
8
+ export function formatRuntimeInputGeneration(attachment: number, document: number): string {
9
+ if (
10
+ !Number.isSafeInteger(attachment) ||
11
+ attachment < 0 ||
12
+ !Number.isSafeInteger(document) ||
13
+ document < 0
14
+ ) {
15
+ throw new Error("Invalid native input generation");
16
+ }
17
+ return `${attachment}:${document}`;
18
+ }
19
+
20
+ /** Compare only independently serialized attachment identity; malformed or absent metadata refuses completion. */
21
+ export function sameRuntimeInputAttachment(
22
+ original: string | null,
23
+ current: string | null,
24
+ ): boolean {
25
+ const before = readNativeCounters(original);
26
+ const after = readNativeCounters(current);
27
+ return before !== null && after !== null && before.attachment === after.attachment;
28
+ }
29
+
30
+ /** Require the complete native codec, including a bounded valid document counter. */
31
+ function readNativeCounters(value: string | null): { attachment: number; document: number } | null {
32
+ if (!value || value.length > 33 || !/^(0|[1-9]\d*):(0|[1-9]\d*)$/.test(value)) return null;
33
+ const [attachment, document] = value.split(":").map(Number);
34
+ if (!Number.isSafeInteger(attachment) || !Number.isSafeInteger(document)) return null;
35
+ return { attachment: attachment!, document: document! };
36
+ }
@@ -0,0 +1,76 @@
1
+ import { afterEach, beforeEach, expect, it, vi } from "vitest";
2
+ import { createJpegCaptureDemand, JPEG_CAPTURE_IDLE_MS } from "./jpeg-capture-demand";
3
+
4
+ beforeEach(() => vi.useFakeTimers());
5
+ afterEach(() => vi.useRealTimers());
6
+ function setup() {
7
+ const retire = vi.fn(async () => {});
8
+ const onError = vi.fn();
9
+ const demand = createJpegCaptureDemand({ now: () => Date.now(), retire, onError });
10
+ return { demand, retire, onError };
11
+ }
12
+ it("no viewer/video-only activity allocates a timer; completed reads renew beyond startup polling", async () => {
13
+ const { demand, retire } = setup();
14
+ const transition = demand.pin(false);
15
+ transition();
16
+ expect(vi.getTimerCount()).toBe(0);
17
+ const release = demand.pin();
18
+ release();
19
+ for (let i = 0; i < 3; i++) {
20
+ await vi.advanceTimersByTimeAsync(1500);
21
+ const releaseNext = demand.pin();
22
+ releaseNext();
23
+ }
24
+ expect(retire).not.toHaveBeenCalled();
25
+ await vi.advanceTimersByTimeAsync(JPEG_CAPTURE_IDLE_MS);
26
+ expect(retire).toHaveBeenCalledTimes(1);
27
+ expect(vi.getTimerCount()).toBe(0);
28
+ });
29
+ it("slow read/start remains pinned across the entire await, and renewal queues after retirement", async () => {
30
+ const { demand, retire } = setup();
31
+ const release = demand.pin();
32
+ await vi.advanceTimersByTimeAsync(10000);
33
+ expect(retire).not.toHaveBeenCalled();
34
+ release();
35
+ const stop = Promise.withResolvers<void>();
36
+ retire.mockImplementationOnce(() => stop.promise);
37
+ await vi.advanceTimersByTimeAsync(JPEG_CAPTURE_IDLE_MS);
38
+ const nextRelease = demand.pin();
39
+ const started = vi.fn(async () => {});
40
+ const start = demand.serialize(started);
41
+ await Promise.resolve();
42
+ expect(started).not.toHaveBeenCalled();
43
+ stop.resolve();
44
+ await start;
45
+ expect(started).toHaveBeenCalledTimes(1);
46
+ nextRelease();
47
+ });
48
+ it("non-image transitions protect but never renew old demand", async () => {
49
+ const { demand, retire } = setup();
50
+ demand.pin()();
51
+ await vi.advanceTimersByTimeAsync(2000);
52
+ const transition = demand.pin(false);
53
+ await vi.advanceTimersByTimeAsync(10000);
54
+ expect(retire).not.toHaveBeenCalled();
55
+ transition();
56
+ await vi.advanceTimersByTimeAsync(1);
57
+ expect(retire).toHaveBeenCalledTimes(1);
58
+ });
59
+ it("explicit cancellation, failed retire and shutdown do not rearm or replay", async () => {
60
+ const { demand, retire, onError } = setup();
61
+ const release = demand.pin();
62
+ demand.cancel();
63
+ release();
64
+ await vi.advanceTimersByTimeAsync(10000);
65
+ expect(retire).not.toHaveBeenCalled();
66
+ retire.mockRejectedValueOnce(new Error("uncertain stop"));
67
+ demand.pin()();
68
+ await vi.advanceTimersByTimeAsync(10000);
69
+ expect(retire).toHaveBeenCalledTimes(1);
70
+ expect(onError).toHaveBeenCalledTimes(1);
71
+ demand.pin()();
72
+ demand.close();
73
+ await vi.advanceTimersByTimeAsync(10000);
74
+ expect(retire).toHaveBeenCalledTimes(1);
75
+ expect(() => demand.pin()).toThrow("closed");
76
+ });
@@ -0,0 +1,98 @@
1
+ /** Runtime-owned JPEG activity lifetime. Viewer/control/video heartbeats do not
2
+ * renew it. Start/stop transitions share one FIFO, while active reads and native
3
+ * target transitions pin retirement before their asynchronous work begins.
4
+ */
5
+ export const JPEG_CAPTURE_IDLE_MS = 3_000;
6
+
7
+ /** Own one unreferenced timer, not a periodic loop. The timeout is twice the
8
+ * existing 1.5s startup image poll (ready polls are 250ms, active input 100ms).
9
+ * Stop failure is reported once; no page mutation or start is silently replayed.
10
+ */
11
+ export function createJpegCaptureDemand(options: {
12
+ now(): number;
13
+ retire(): Promise<void>;
14
+ onError(error: unknown): void;
15
+ }) {
16
+ let timer: ReturnType<typeof setTimeout> | null = null;
17
+ let lastDemand: number | null = null;
18
+ let pins = 0;
19
+ let revision = 0;
20
+ let closed = false;
21
+ let tail = Promise.resolve();
22
+
23
+ function clearTimer() {
24
+ if (timer) clearTimeout(timer);
25
+ timer = null;
26
+ }
27
+
28
+ /** Serialize native stream start/stop without retaining a failed queue tail. */
29
+ function serialize<T>(operation: () => Promise<T>): Promise<T> {
30
+ const result = tail.then(async () => {
31
+ if (closed) throw new Error("JPEG capture lifetime is closed");
32
+ return await operation();
33
+ });
34
+ tail = result.then(
35
+ () => undefined,
36
+ () => undefined,
37
+ );
38
+ return result;
39
+ }
40
+
41
+ function arm() {
42
+ clearTimer();
43
+ if (closed || pins > 0 || lastDemand === null) return;
44
+ const scheduledRevision = revision;
45
+ timer = setTimeout(
46
+ () => {
47
+ timer = null;
48
+ void serialize(async () => {
49
+ if (pins > 0 || revision !== scheduledRevision || lastDemand === null) return;
50
+ if (options.now() - lastDemand < JPEG_CAPTURE_IDLE_MS) {
51
+ arm();
52
+ return;
53
+ }
54
+ lastDemand = null;
55
+ await options.retire();
56
+ }).catch((error) => {
57
+ if (!closed) options.onError(error);
58
+ });
59
+ },
60
+ Math.max(1, lastDemand + JPEG_CAPTURE_IDLE_MS - options.now()),
61
+ );
62
+ timer.unref?.();
63
+ }
64
+
65
+ /** Pin before the first await. Native transitions may pass false to protect
66
+ * existing reads without creating/renewing image demand on a video-only page.
67
+ * Completion renews a real read, allowing slow startup/capture to finish safely.
68
+ */
69
+ function pin(renew = true): () => void {
70
+ if (closed) throw new Error("JPEG capture lifetime is closed");
71
+ clearTimer();
72
+ pins += 1;
73
+ if (renew) lastDemand = options.now();
74
+ const pinnedRevision = revision;
75
+ let released = false;
76
+ return () => {
77
+ if (released) return;
78
+ released = true;
79
+ pins -= 1;
80
+ if (renew && !closed && pinnedRevision === revision) lastDemand = options.now();
81
+ arm();
82
+ };
83
+ }
84
+
85
+ /** Explicit stop discards demand; an older pending read cannot rearm its timer. */
86
+ function cancel() {
87
+ revision += 1;
88
+ lastDemand = null;
89
+ clearTimer();
90
+ }
91
+
92
+ /** Shutdown prevents queued native work and future acquisition. */
93
+ function close() {
94
+ closed = true;
95
+ cancel();
96
+ }
97
+ return { pin, serialize, cancel, close };
98
+ }
@@ -0,0 +1,29 @@
1
+ /** Read actual JPEG dimensions so CDP viewport metadata cannot authorize input on cropped pixels. */
2
+ export function readJpegFrameDimensions(
3
+ dataBase64: string,
4
+ ): { width: number; height: number } | null {
5
+ const bytes = Buffer.from(dataBase64, "base64");
6
+ if (bytes.length < 4 || bytes.readUInt16BE(0) !== 0xffd8) return null;
7
+
8
+ let offset = 2;
9
+ while (offset < bytes.length) {
10
+ if (bytes[offset++] !== 0xff) return null;
11
+ while (bytes[offset] === 0xff) offset += 1;
12
+ const marker = bytes[offset++];
13
+ if (marker === undefined || marker === 0xda || marker === 0xd9) return null;
14
+ if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd8)) continue;
15
+ if (offset + 2 > bytes.length) return null;
16
+
17
+ const length = bytes.readUInt16BE(offset);
18
+ if (length < 2 || offset + length > bytes.length) return null;
19
+ // All JPEG start-of-frame markers except the Huffman/arithmetic table markers.
20
+ if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) {
21
+ if (length < 8) return null;
22
+ const height = bytes.readUInt16BE(offset + 3);
23
+ const width = bytes.readUInt16BE(offset + 5);
24
+ return width > 0 && height > 0 ? { width, height } : null;
25
+ }
26
+ offset += length;
27
+ }
28
+ return null;
29
+ }
@@ -0,0 +1,61 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { browserGestureEventSchema, browserGestureKeySchema } from "../shared/browser";
3
+ import { heldKeyModifiers, nativeKeyEvent } from "./keyboard-input";
4
+
5
+ const key = (key: string, code: string, modifiers = 0) =>
6
+ browserGestureKeySchema.parse({ kind: "key", type: "down", key, code, modifiers });
7
+
8
+ describe("native keyboard envelopes", () => {
9
+ it("preserves Unicode and space while Enter and Tab retain native meanings", () => {
10
+ expect(nativeKeyEvent(key("šŸ˜€", "KeyA"))).toMatchObject({ type: "keyDown", text: "šŸ˜€" });
11
+ expect(nativeKeyEvent(key(" ", "Space"))).toMatchObject({
12
+ text: " ",
13
+ windowsVirtualKeyCode: 32,
14
+ });
15
+ expect(nativeKeyEvent(key("Enter", "Enter"))).toMatchObject({
16
+ text: "\r",
17
+ windowsVirtualKeyCode: 13,
18
+ });
19
+ expect(nativeKeyEvent(key("Tab", "Tab"))).toMatchObject({
20
+ type: "rawKeyDown",
21
+ windowsVirtualKeyCode: 9,
22
+ });
23
+ expect(nativeKeyEvent(key("Tab", "Tab"))).not.toHaveProperty("text");
24
+ expect(nativeKeyEvent(key("ArrowLeft", "ArrowLeft"))).toMatchObject({
25
+ windowsVirtualKeyCode: 37,
26
+ });
27
+ expect(nativeKeyEvent(key("A", "KeyA", 8))).toMatchObject({ text: "A", modifiers: 8 });
28
+ });
29
+
30
+ it("forwards shortcuts and repeat without synthesizing printable characters", () => {
31
+ expect(nativeKeyEvent(key("a", "KeyA", 2))).toMatchObject({
32
+ type: "rawKeyDown",
33
+ windowsVirtualKeyCode: 65,
34
+ modifiers: 2,
35
+ });
36
+ expect(nativeKeyEvent(key("a", "KeyA", 2))).not.toHaveProperty("text");
37
+ expect(nativeKeyEvent({ ...key("a", "KeyA"), repeat: true })).toHaveProperty(
38
+ "autoRepeat",
39
+ true,
40
+ );
41
+ expect(heldKeyModifiers([{ key: "Control" }, { key: "Shift" }])).toBe(10);
42
+ });
43
+
44
+ it("rejects shortcut text, keyup text/repeat and oversized committed text", () => {
45
+ expect(browserGestureKeySchema.safeParse({ ...key("a", "KeyA", 2), text: "a" }).success).toBe(
46
+ false,
47
+ );
48
+ expect(
49
+ browserGestureKeySchema.safeParse({ ...key("a", "KeyA"), type: "up", text: "a" }).success,
50
+ ).toBe(false);
51
+ expect(
52
+ browserGestureKeySchema.safeParse({ ...key("a", "KeyA"), type: "up", repeat: true }).success,
53
+ ).toBe(false);
54
+ expect(browserGestureEventSchema.safeParse({ kind: "text", text: "漢字\nšŸ˜€" }).success).toBe(
55
+ true,
56
+ );
57
+ expect(
58
+ browserGestureEventSchema.safeParse({ kind: "text", text: "x".repeat(16001) }).success,
59
+ ).toBe(false);
60
+ });
61
+ });