@omercnet/paseo-shared-browser 1.2.2 → 2.0.0-next.249.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/OVERVIEW.md +2 -2
  2. package/README.md +316 -27
  3. package/REMOTE_BROWSER_RESEARCH.md +395 -0
  4. package/STREAMING_DESIGN.md +196 -0
  5. package/client/browser-auto-capture-density.test.ts +102 -0
  6. package/client/browser-auto-capture-density.ts +30 -0
  7. package/client/browser-canvas-input.test.ts +198 -0
  8. package/client/browser-canvas-input.ts +202 -0
  9. package/client/browser-canvas-keyboard-web.test.ts +113 -0
  10. package/client/browser-canvas-keyboard-web.ts +230 -0
  11. package/client/browser-canvas-keyboard.test.ts +123 -0
  12. package/client/browser-canvas-keyboard.ts +126 -0
  13. package/client/browser-canvas-layout.test.ts +109 -0
  14. package/client/browser-canvas-layout.ts +51 -0
  15. package/client/browser-canvas-viewport-web.test.ts +77 -0
  16. package/client/browser-canvas-viewport-web.ts +63 -0
  17. package/client/browser-canvas-viewport.tsx +152 -0
  18. package/client/browser-capture-cadence.test.ts +9 -0
  19. package/client/browser-capture-cadence.ts +6 -0
  20. package/client/browser-capture-density-controls.tsx +75 -0
  21. package/client/browser-chrome.tsx +234 -0
  22. package/client/browser-client-presence.ts +184 -0
  23. package/client/browser-compose-controls.test.tsx +275 -0
  24. package/client/browser-compose-controls.tsx +174 -0
  25. package/client/browser-control-button.tsx +109 -0
  26. package/client/browser-control-dialog.tsx +71 -0
  27. package/client/browser-control-request.test.ts +27 -0
  28. package/client/browser-control-request.ts +57 -0
  29. package/client/browser-control-tooltip-web.ts +10 -0
  30. package/client/browser-dialog-body.tsx +22 -0
  31. package/client/browser-emulation-mode.test.ts +75 -0
  32. package/client/browser-emulation-mode.ts +98 -0
  33. package/client/browser-fill-viewport.test.ts +45 -0
  34. package/client/browser-fill-viewport.ts +28 -0
  35. package/client/browser-frame-buffer.test.ts +157 -0
  36. package/client/browser-frame-buffer.ts +148 -0
  37. package/client/browser-frame-image.tsx +62 -0
  38. package/client/browser-image-load.test.ts +23 -0
  39. package/client/browser-image-load.ts +26 -0
  40. package/client/browser-input-queue.test.ts +1254 -0
  41. package/client/browser-input-queue.ts +519 -0
  42. package/client/browser-menu-placement.test.ts +90 -0
  43. package/client/browser-menu-placement.ts +78 -0
  44. package/client/browser-panel-styles.ts +384 -0
  45. package/client/browser-quality-controls.test.ts +26 -0
  46. package/client/browser-quality-controls.tsx +139 -0
  47. package/client/browser-resolution-picker.tsx +189 -0
  48. package/client/browser-tabs.tsx +117 -0
  49. package/client/browser-toolbar-menu-web.test.ts +185 -0
  50. package/client/browser-toolbar-menu-web.ts +159 -0
  51. package/client/browser-toolbar-menu.tsx +255 -0
  52. package/client/browser-toolbar-submenu.tsx +93 -0
  53. package/client/browser-video-bytes.test.ts +44 -0
  54. package/client/browser-video-currency.ts +45 -0
  55. package/client/browser-video-decoder.test.ts +473 -0
  56. package/client/browser-video-decoder.ts +311 -0
  57. package/client/browser-video-surface.tsx +21 -0
  58. package/client/browser-video-visibility.test.ts +45 -0
  59. package/client/browser-viewer-recovery.test.ts +111 -0
  60. package/client/browser-viewer-recovery.ts +78 -0
  61. package/client/browser.tsx +1650 -1579
  62. package/client/favorite-star.tsx +23 -0
  63. package/client/frame-lifecycle.ts +62 -59
  64. package/client/sidebar.tsx +1 -2
  65. package/client/use-browser-auto-capture-density.test.ts +104 -0
  66. package/client/use-browser-auto-capture-density.ts +43 -0
  67. package/client/use-browser-canvas-input.ts +375 -0
  68. package/client/use-browser-capture-density.ts +87 -0
  69. package/client/use-browser-emulation-mode.ts +56 -0
  70. package/client/use-browser-fill-viewport.test.ts +108 -0
  71. package/client/use-browser-fill-viewport.ts +62 -0
  72. package/client/use-browser-frame-buffer.ts +66 -0
  73. package/client/use-browser-image-capture.ts +139 -0
  74. package/client/use-browser-scoped-mutation.test.ts +118 -0
  75. package/client/use-browser-scoped-mutation.ts +59 -0
  76. package/client/use-browser-video.test.ts +1060 -0
  77. package/client/use-browser-video.ts +481 -0
  78. package/client/use-browser-viewer-recovery.ts +26 -0
  79. package/client/use-resolution-favorites.ts +90 -0
  80. package/client/web.test.ts +125 -0
  81. package/client/web.ts +405 -0
  82. package/index.client.tsx +0 -10
  83. package/index.server.ts +32 -4
  84. package/npm-shrinkwrap.json +9 -9
  85. package/package.json +10 -5
  86. package/paseo-plugin.json +1 -1
  87. package/scripts/prepare-runtime.mjs +1 -1
  88. package/server/agent-browser-runtime.ts +1601 -314
  89. package/server/browser-cursor.test.ts +64 -0
  90. package/server/browser-cursor.ts +69 -0
  91. package/server/browser-frame-context.ts +22 -0
  92. package/server/browser-frame-epoch.test.ts +95 -0
  93. package/server/browser-frame-rejection.ts +17 -0
  94. package/server/browser-gesture.ts +120 -0
  95. package/server/browser-operation-diagnostics.test.ts +79 -0
  96. package/server/browser-operation-diagnostics.ts +103 -0
  97. package/server/browser-policy.ts +1316 -151
  98. package/server/browser-video-input-authority.test.ts +211 -0
  99. package/server/browser-video-policy.test.ts +564 -0
  100. package/server/browser-video-socket.test.ts +368 -0
  101. package/server/browser.ts +151 -16
  102. package/server/capture-transport-policy.test.ts +105 -0
  103. package/server/capture-transport-policy.ts +154 -0
  104. package/server/capture-transport-runtime.test.ts +213 -0
  105. package/server/daemon-launch-chain.test.ts +70 -0
  106. package/server/daemon-shutdown.test.ts +289 -0
  107. package/server/discrete-input-freshness.test.ts +364 -0
  108. package/server/discrete-input-native-lifetime.test.ts +150 -0
  109. package/server/display-loss-recovery.test.ts +294 -0
  110. package/server/emulation-restoration.test.ts +262 -0
  111. package/server/input-cleanup.ts +23 -0
  112. package/server/input-generation.test.ts +31 -0
  113. package/server/input-generation.ts +36 -0
  114. package/server/jpeg-capture-demand.test.ts +76 -0
  115. package/server/jpeg-capture-demand.ts +98 -0
  116. package/server/jpeg-frame.ts +29 -0
  117. package/server/keyboard-input.test.ts +61 -0
  118. package/server/keyboard-input.ts +104 -0
  119. package/server/live-input-policy.test.ts +958 -0
  120. package/server/live-input-runtime.test.ts +732 -0
  121. package/server/live-input-supervisor.test.ts +147 -0
  122. package/server/mcp-entry.ts +44 -2
  123. package/server/native-video-backpressure.test.ts +228 -0
  124. package/server/native-video-buffer.test.ts +141 -0
  125. package/server/native-video-buffer.ts +114 -0
  126. package/server/native-video-capture.test.ts +599 -0
  127. package/server/native-video-capture.ts +497 -0
  128. package/server/native-video-extension-source.test.ts +201 -0
  129. package/server/native-video-extension-source.ts +364 -0
  130. package/server/native-video-extension.test.ts +41 -0
  131. package/server/native-video-extension.ts +83 -0
  132. package/server/native-video-packet.ts +9 -0
  133. package/server/native-video-settings.test.ts +245 -0
  134. package/server/navigation-budget.ts +8 -0
  135. package/server/navigation-commit.ts +125 -0
  136. package/server/navigation-policy.test.ts +153 -0
  137. package/server/process-identity.ts +58 -0
  138. package/server/reattach-held-input.test.ts +143 -0
  139. package/server/runtime-frame-reuse.test.ts +295 -0
  140. package/server/runtime-jpeg-demand.test.ts +200 -0
  141. package/server/runtime-jpeg-shutdown.test.ts +210 -0
  142. package/server/runtime-jpeg-source-order.test.ts +201 -0
  143. package/server/runtime-launch-environment.test.ts +111 -0
  144. package/server/runtime-navigation.test.ts +327 -0
  145. package/server/runtime-owner-display.test.ts +152 -0
  146. package/server/runtime-owner.ts +340 -31
  147. package/server/runtime-page-state.test.ts +249 -0
  148. package/server/runtime-protocol.ts +32 -2
  149. package/server/runtime-target-retention.test.ts +181 -0
  150. package/server/runtime-video-opt-in.test.ts +82 -0
  151. package/server/runtime-video.test.ts +444 -0
  152. package/server/screencast-source-time.ts +46 -0
  153. package/server/stale-wheel-admission.test.ts +136 -0
  154. package/server/supervisor-client.ts +28 -4
  155. package/server/supervisor-heartbeat.test.ts +80 -0
  156. package/server/supervisor.ts +553 -55
  157. package/server/tab-close-confirmation.test.ts +97 -0
  158. package/server/video-source-recovery.test.ts +20 -0
  159. package/server/video-source-recovery.ts +64 -0
  160. package/server/viewer-capture-lifetime.ts +51 -0
  161. package/server/virtual-display.test.ts +184 -0
  162. package/server/virtual-display.ts +236 -0
  163. package/shared/browser-display-preferences.ts +46 -0
  164. package/shared/browser-video.ts +83 -0
  165. package/shared/browser.ts +286 -65
  166. package/shared/capture-density.ts +16 -0
  167. package/shared/capture-settings.ts +17 -0
  168. package/shared/device-presets.ts +90 -0
  169. package/shared/resolution-menu.test.ts +120 -0
  170. package/shared/resolution-menu.ts +72 -0
  171. package/shared/video-settings.test.ts +38 -0
  172. package/shared/video-settings.ts +34 -0
  173. package/shared/viewport-limits.ts +5 -0
@@ -0,0 +1,143 @@
1
+ import { EventEmitter } from "node:events";
2
+ import { expect, it } from "vitest";
3
+ import { AgentBrowserRuntime, type DeviceEmulation } from "./agent-browser-runtime";
4
+ import { type CdpConnection, CdpSession } from "./cdp";
5
+
6
+ /** Protocol fixture exercises real attachment/configuration methods without a browser process. */
7
+ class ConnectionFixture extends EventEmitter {
8
+ isOpen = true;
9
+ nextSession = 0;
10
+ calls: { method: string; params: Record<string, unknown>; sessionId: string | undefined }[] = [];
11
+ intercept?: (method: string, sessionId?: string) => Promise<void>;
12
+
13
+ async send<T>(
14
+ method: string,
15
+ params: Record<string, unknown> = {},
16
+ options: { sessionId?: string } = {},
17
+ ): Promise<T> {
18
+ this.calls.push({ method, params, sessionId: options.sessionId });
19
+ await this.intercept?.(method, options.sessionId);
20
+ if (method === "Target.getTargets") {
21
+ return {
22
+ targetInfos: ["a", "b"].map((targetId) => ({
23
+ targetId,
24
+ type: "page",
25
+ url: "about:blank",
26
+ title: targetId,
27
+ })),
28
+ } as T;
29
+ }
30
+ if (method === "Target.attachToTarget") return { sessionId: `new-${++this.nextSession}` } as T;
31
+ if (method === "Page.getNavigationHistory") return { currentIndex: 0, entries: [] } as T;
32
+ return {} as T;
33
+ }
34
+
35
+ close(): void {
36
+ this.isOpen = false;
37
+ }
38
+ }
39
+
40
+ interface RuntimeControl {
41
+ connection: CdpConnection;
42
+ page: CdpSession | null;
43
+ targetId: string | null;
44
+ viewport: DeviceEmulation;
45
+ emulationAppliedPage: CdpSession | null;
46
+ screencastActive: boolean;
47
+ screencastFrame: unknown;
48
+ attachmentGeneration: number;
49
+ assertVersion(): Promise<void>;
50
+ connectCdp(targetId?: string): Promise<void>;
51
+ requirePage(): Promise<CdpSession>;
52
+ reattachPageForScreencast(previous: CdpSession): Promise<CdpSession>;
53
+ onScreencastFrame(page: CdpSession, event: unknown): void;
54
+ }
55
+
56
+ const pixel: DeviceEmulation = {
57
+ width: 412,
58
+ height: 839,
59
+ deviceScaleFactor: 2.625,
60
+ mobile: true,
61
+ touch: true,
62
+ screenWidth: 430,
63
+ screenHeight: 900,
64
+ userAgent: "fixture-mobile-agent",
65
+ platform: "Android",
66
+ };
67
+
68
+ const generation = (runtime: AgentBrowserRuntime) => {
69
+ const { attachmentGeneration, documentGeneration } = runtime as unknown as {
70
+ attachmentGeneration: number;
71
+ documentGeneration: number;
72
+ };
73
+ return `${attachmentGeneration}:${documentGeneration}`;
74
+ };
75
+
76
+ function fixture() {
77
+ const runtime = new AgentBrowserRuntime({
78
+ binaryPath: "/tmp/unlaunched-browser",
79
+ executablePath: "/tmp/unlaunched-chromium",
80
+ profilePath: "/tmp/uncreated-profile",
81
+ ipcDirectory: "/tmp/uncreated-ipc",
82
+ session: "emulation-test",
83
+ });
84
+ const connection = new ConnectionFixture();
85
+ const typedConnection = connection as unknown as CdpConnection;
86
+ const previous = new CdpSession(typedConnection, "previous", "a");
87
+ const control = runtime as unknown as RuntimeControl;
88
+ Object.assign(control, {
89
+ connection: typedConnection,
90
+ page: previous,
91
+ targetId: "a",
92
+ viewport: { ...pixel },
93
+ emulationAppliedPage: previous,
94
+ });
95
+ return { runtime, connection, previous, control };
96
+ }
97
+
98
+ it("never releases input held on an older attachment onto a later one during screencast retry", async () => {
99
+ const { runtime, connection, control } = fixture();
100
+ await runtime.beginLiveInput("old-channel", generation(runtime));
101
+ await runtime.mouseDown(10, 20, "left", 1, "old-channel");
102
+ // Real reconnect + target publication; only transport setup is replaced.
103
+ const reconnected = new ConnectionFixture();
104
+ control.assertVersion = async () => {};
105
+ control.connectCdp = async () => {
106
+ control.connection = reconnected as unknown as CdpConnection;
107
+ await runtime.selectTarget("b");
108
+ };
109
+ await runtime.reconnect();
110
+ expect(connection.isOpen).toBe(false);
111
+ await control.reattachPageForScreencast(control.page!);
112
+ expect(reconnected.calls.filter((call) => call.params.type === "mouseReleased")).toEqual([]);
113
+ await runtime.endLiveInput("old-channel");
114
+ });
115
+
116
+ it("still releases held input on its own attachment before reattach detaches it", async () => {
117
+ const { runtime, connection, control, previous } = fixture();
118
+ await runtime.beginLiveInput("channel", generation(runtime));
119
+ await runtime.mouseDown(10, 20, "left", 1, "channel");
120
+ await control.reattachPageForScreencast(previous);
121
+ const order = connection.calls.map((call) => call.params.type ?? call.method);
122
+ expect(order.indexOf("mouseReleased")).toBeGreaterThan(-1);
123
+ expect(order.indexOf("mouseReleased")).toBeLessThan(order.indexOf("Target.detachFromTarget"));
124
+ expect(connection.calls.find((call) => call.params.type === "mouseReleased")).toMatchObject({
125
+ sessionId: "previous",
126
+ params: { x: 10, y: 20 },
127
+ });
128
+ });
129
+
130
+ it("does not retarget ungestured held input from a replaced attachment onto its replacement", async () => {
131
+ const { runtime, connection, control } = fixture();
132
+ await runtime.mouseDown(10, 20, "left", 1);
133
+ const reconnected = new ConnectionFixture();
134
+ control.assertVersion = async () => {};
135
+ control.connectCdp = async () => {
136
+ control.connection = reconnected as unknown as CdpConnection;
137
+ await runtime.selectTarget("b");
138
+ };
139
+ await runtime.reconnect();
140
+ expect(connection.isOpen).toBe(false);
141
+ await control.reattachPageForScreencast(control.page!);
142
+ expect(reconnected.calls.filter((call) => call.params.type === "mouseReleased")).toEqual([]);
143
+ });
@@ -0,0 +1,295 @@
1
+ /** Exact fresh runtime receipts avoid resending cached JPEGs without reviving invalidated input. */
2
+ import { describe, expect, it } from "vitest";
3
+ import type { BrowserFrame, BrowserState } from "../shared/browser";
4
+ import { DEFAULT_CAPTURE_QUALITY } from "../shared/capture-settings";
5
+ import { type BrowserRuntimeClient, SessionManager } from "./browser-policy";
6
+ import type { JsonValue } from "./runtime-protocol";
7
+
8
+ async function fixture() {
9
+ let now = 1_000;
10
+ let token = 0;
11
+ let frames = 0;
12
+ let inputGeneration = "0:0";
13
+ const raw: Record<string, JsonValue> = {
14
+ dataBase64: Buffer.alloc(200_000, 120).toString("base64"),
15
+ byteLength: 200_000,
16
+ width: 1280,
17
+ height: 800,
18
+ transport: "screenshot",
19
+ capturedAt: new Date(now).toISOString(),
20
+ };
21
+ const client: BrowserRuntimeClient = {
22
+ connect: async () => ({ epoch: 1 }),
23
+ ensureWorkspace: async (workspaceId) => ({
24
+ workspaceId,
25
+ runtimeId: "runtime-one",
26
+ createdAt: 1,
27
+ }),
28
+ requestWorkspace: async (_workspace, operation, input) => {
29
+ if (operation === "identity") return { userAgent: "Fixture" };
30
+ if (operation === "state")
31
+ return {
32
+ url: "https://fixture.invalid/",
33
+ title: "Fixture",
34
+ canGoBack: false,
35
+ canGoForward: false,
36
+ inputGeneration,
37
+ };
38
+ if (operation === "frame") {
39
+ frames++;
40
+ return { ...raw };
41
+ }
42
+ if (operation === "emulate" && input && typeof input === "object" && !Array.isArray(input)) {
43
+ raw.width = input.width!;
44
+ raw.height = input.height!;
45
+ }
46
+ if (operation === "cursor") return null;
47
+ return null;
48
+ },
49
+ archiveWorkspace: async () => {},
50
+ closeWorkspace: async () => {},
51
+ disconnect: () => {},
52
+ };
53
+ const manager = new SessionManager({
54
+ client,
55
+ now: () => now,
56
+ validateWorkspace: async () => true,
57
+ issueToken: () => String(++token).padStart(32, "0"),
58
+ });
59
+ await manager.connect();
60
+ const viewer = await manager.attach("owned-perf-fixture", "Viewer");
61
+ const control = await manager.acquireControl(viewer.viewerToken);
62
+ const expected = (state: BrowserState) => ({
63
+ sessionId: state.sessionId,
64
+ runtimeId: state.runtimeId!,
65
+ bridgeEpoch: state.bridgeEpoch!,
66
+ navigationGeneration: state.navigationGeneration,
67
+ viewportGeneration: state.viewportGeneration,
68
+ });
69
+ return {
70
+ manager,
71
+ raw,
72
+ viewer,
73
+ control,
74
+ expected,
75
+ get frames() {
76
+ return frames;
77
+ },
78
+ advance(ms = 250) {
79
+ now += ms;
80
+ },
81
+ documentChange() {
82
+ inputGeneration = "0:1";
83
+ },
84
+ session: () =>
85
+ (
86
+ manager as unknown as { sessions: Map<string, { runtimeId: string; bridgeEpoch: number }> }
87
+ ).sessions.get("owned-perf-fixture")!,
88
+ };
89
+ }
90
+ const target = (frame: BrowserFrame) => ({
91
+ frameId: frame.frameId,
92
+ navigationGeneration: frame.navigationGeneration,
93
+ viewportGeneration: frame.viewportGeneration,
94
+ });
95
+
96
+ describe("exact runtime receipt reuse across policy cache expiry", () => {
97
+ it("returns metadata-only repeated polls while preserving current-frame input authority", async () => {
98
+ const f = await fixture();
99
+ try {
100
+ const first = await f.manager.capture(f.viewer.viewerToken);
101
+ for (let index = 0; index < 3; index++) {
102
+ f.advance();
103
+ const repeated = await f.manager.capture(
104
+ f.viewer.viewerToken,
105
+ DEFAULT_CAPTURE_QUALITY,
106
+ first.frame!.frameId,
107
+ );
108
+ expect(repeated.frame).toBeNull();
109
+ }
110
+ expect(f.frames).toBe(4);
111
+ const result = await f.manager.beginGesture({
112
+ viewerToken: f.viewer.viewerToken,
113
+ controlToken: f.control.controlToken,
114
+ expected: f.expected(first.state),
115
+ target: target(first.frame!),
116
+ pointerKind: "mouse",
117
+ });
118
+ expect(result).toHaveProperty("gestureId");
119
+ } finally {
120
+ f.manager.disconnect();
121
+ }
122
+ });
123
+
124
+ it("assigns a new token for any changed runtime receipt field, including equal pixels at a new capture time", async () => {
125
+ for (const [field, value] of [
126
+ ["capturedAt", "2026-10-03T12:00:00.000Z"],
127
+ ["transport", "cdp-screencast"],
128
+ ["dataBase64", Buffer.alloc(200_000, 121).toString("base64")],
129
+ ["byteLength", 1],
130
+ ] as const) {
131
+ const f = await fixture();
132
+ try {
133
+ const first = await f.manager.capture(f.viewer.viewerToken);
134
+ f.advance();
135
+ f.raw[field] = value;
136
+ if (field === "byteLength") f.raw.dataBase64 = "eA==";
137
+ const next = await f.manager.capture(
138
+ f.viewer.viewerToken,
139
+ DEFAULT_CAPTURE_QUALITY,
140
+ first.frame!.frameId,
141
+ );
142
+ expect(next.frame?.frameId).not.toBe(first.frame!.frameId);
143
+ } finally {
144
+ f.manager.disconnect();
145
+ }
146
+ }
147
+ });
148
+
149
+ it("keeps per-quality tokens separate even for identical runtime receipts", async () => {
150
+ const f = await fixture();
151
+ try {
152
+ const first = await f.manager.capture(f.viewer.viewerToken, "high");
153
+ f.advance();
154
+ const next = await f.manager.capture(f.viewer.viewerToken, "medium", first.frame!.frameId);
155
+ expect(next.frame?.frameId).not.toBe(first.frame!.frameId);
156
+ } finally {
157
+ f.manager.disconnect();
158
+ }
159
+ });
160
+
161
+ it("cannot revive an identical runtime receipt after input invalidation", async () => {
162
+ const f = await fixture();
163
+ try {
164
+ const first = await f.manager.capture(f.viewer.viewerToken);
165
+ await f.manager.sendInput({
166
+ viewerToken: f.viewer.viewerToken,
167
+ controlToken: f.control.controlToken,
168
+ expected: f.expected(first.state),
169
+ target: target(first.frame!),
170
+ event: { kind: "type", text: "x" },
171
+ });
172
+ f.advance();
173
+ const next = await f.manager.capture(
174
+ f.viewer.viewerToken,
175
+ DEFAULT_CAPTURE_QUALITY,
176
+ first.frame!.frameId,
177
+ );
178
+ expect(next.frame?.frameId).not.toBe(first.frame!.frameId);
179
+ await expect(
180
+ f.manager.beginGesture({
181
+ viewerToken: f.viewer.viewerToken,
182
+ controlToken: f.control.controlToken,
183
+ expected: f.expected(next.state),
184
+ target: target(first.frame!),
185
+ pointerKind: "mouse",
186
+ }),
187
+ ).resolves.toMatchObject({ admission: "stale-frame" });
188
+ } finally {
189
+ f.manager.disconnect();
190
+ }
191
+ });
192
+
193
+ it("changes token for exact resized geometry and refuses malformed old dimensions", async () => {
194
+ const f = await fixture();
195
+ try {
196
+ const first = await f.manager.capture(f.viewer.viewerToken);
197
+ await f.manager.resize({
198
+ viewerToken: f.viewer.viewerToken,
199
+ controlToken: f.control.controlToken,
200
+ expected: f.expected(first.state),
201
+ viewport: { width: 1920, height: 1200 },
202
+ });
203
+ const next = await f.manager.capture(
204
+ f.viewer.viewerToken,
205
+ DEFAULT_CAPTURE_QUALITY,
206
+ first.frame!.frameId,
207
+ );
208
+ expect(next.frame).toMatchObject({ width: 1920, height: 1200 });
209
+ expect(next.frame!.frameId).not.toBe(first.frame!.frameId);
210
+ f.advance();
211
+ f.raw.width = 1280;
212
+ await expect(f.manager.capture(f.viewer.viewerToken)).rejects.toThrow("stale viewport");
213
+ } finally {
214
+ f.manager.disconnect();
215
+ }
216
+ });
217
+
218
+ it("does not reuse receipts under changed runtime identity or bridge epoch", async () => {
219
+ for (const field of ["runtimeId", "bridgeEpoch"] as const) {
220
+ const f = await fixture();
221
+ try {
222
+ const first = await f.manager.capture(f.viewer.viewerToken);
223
+ f.advance();
224
+ if (field === "runtimeId") f.session().runtimeId = "runtime-two";
225
+ else f.session().bridgeEpoch = 2;
226
+ const next = await f.manager.capture(
227
+ f.viewer.viewerToken,
228
+ DEFAULT_CAPTURE_QUALITY,
229
+ first.frame!.frameId,
230
+ );
231
+ expect(next.frame!.frameId).not.toBe(first.frame!.frameId);
232
+ } finally {
233
+ f.manager.disconnect();
234
+ }
235
+ }
236
+ });
237
+
238
+ it("cannot deduplicate legacy unmarked receipts or metadata after a document change", async () => {
239
+ for (const field of ["capturedAt", "transport"] as const) {
240
+ const f = await fixture();
241
+ try {
242
+ delete f.raw[field];
243
+ const first = await f.manager.capture(f.viewer.viewerToken);
244
+ f.advance();
245
+ const next = await f.manager.capture(
246
+ f.viewer.viewerToken,
247
+ DEFAULT_CAPTURE_QUALITY,
248
+ first.frame!.frameId,
249
+ );
250
+ expect(next.frame!.frameId).not.toBe(first.frame!.frameId);
251
+ } finally {
252
+ f.manager.disconnect();
253
+ }
254
+ }
255
+ const f = await fixture();
256
+ try {
257
+ const first = await f.manager.capture(f.viewer.viewerToken);
258
+ f.documentChange();
259
+ await f.manager.status(f.viewer.viewerToken);
260
+ f.advance();
261
+ const next = await f.manager.capture(
262
+ f.viewer.viewerToken,
263
+ DEFAULT_CAPTURE_QUALITY,
264
+ first.frame!.frameId,
265
+ );
266
+ expect(next.frame!.frameId).not.toBe(first.frame!.frameId);
267
+ expect(next.state.navigationGeneration).toBeGreaterThan(first.state.navigationGeneration);
268
+ } finally {
269
+ f.manager.disconnect();
270
+ }
271
+ });
272
+
273
+ it("avoids repeated large payloads while every expired policy read still asks the runtime for fresh pixels", async () => {
274
+ const f = await fixture();
275
+ try {
276
+ const first = await f.manager.capture(f.viewer.viewerToken);
277
+ let before = JSON.stringify(first).length;
278
+ let after = before;
279
+ for (let index = 0; index < 3; index++) {
280
+ f.advance();
281
+ const reply = await f.manager.capture(
282
+ f.viewer.viewerToken,
283
+ DEFAULT_CAPTURE_QUALITY,
284
+ first.frame!.frameId,
285
+ );
286
+ after += JSON.stringify(reply).length;
287
+ before += JSON.stringify({ state: reply.state, frame: first.frame }).length;
288
+ }
289
+ expect(after).toBeLessThan(before * 0.26);
290
+ expect(f.frames).toBe(4);
291
+ } finally {
292
+ f.manager.disconnect();
293
+ }
294
+ });
295
+ });
@@ -0,0 +1,200 @@
1
+ /** Actual runtime entry points with owned fake CDP, no Chromium/process fixture. */
2
+ import { afterEach, beforeEach, expect, it, vi } from "vitest";
3
+ import { DEFAULT_JPEG_QUALITY } from "../shared/capture-settings";
4
+ import { AgentBrowserRuntime, type RuntimeFrame } from "./agent-browser-runtime";
5
+ import { JPEG_CAPTURE_IDLE_MS } from "./jpeg-capture-demand";
6
+
7
+ function jpeg(tag: number) {
8
+ return Buffer.from([
9
+ 255,
10
+ 216,
11
+ 255,
12
+ 224,
13
+ 0,
14
+ 3,
15
+ tag,
16
+ 255,
17
+ 192,
18
+ 0,
19
+ 11,
20
+ 8,
21
+ 3,
22
+ 32,
23
+ 5,
24
+ 0,
25
+ 1,
26
+ 1,
27
+ 17,
28
+ 0,
29
+ 255,
30
+ 217,
31
+ ]).toString("base64");
32
+ }
33
+ beforeEach(() => {
34
+ vi.useFakeTimers();
35
+ vi.spyOn(performance, "now").mockImplementation(() => Date.now());
36
+ });
37
+ afterEach(() => {
38
+ vi.clearAllTimers();
39
+ vi.useRealTimers();
40
+ vi.restoreAllMocks();
41
+ });
42
+ function setup() {
43
+ const runtime = new AgentBrowserRuntime({
44
+ nativeVideo: true,
45
+ binaryPath: "/tmp/unlaunched",
46
+ executablePath: "/tmp/unlaunched-chrome",
47
+ profilePath: "/tmp/uncreated-p",
48
+ ipcDirectory: "/tmp/uncreated-i",
49
+ session: "fixture",
50
+ });
51
+ const page = { send: vi.fn(async (_method: string, _params?: unknown) => ({})) };
52
+ let tag = 0;
53
+ const internal = runtime as unknown as {
54
+ page: typeof page;
55
+ requirePage(): Promise<typeof page>;
56
+ startScreencastSession(source: typeof page): Promise<void>;
57
+ onScreencastFrame(source: typeof page, event: unknown): void;
58
+ captureScreenshot(budget: number, quality: number, preserve?: boolean): Promise<RuntimeFrame>;
59
+ screencastActive: boolean;
60
+ invoke(args: string[]): Promise<unknown>;
61
+ };
62
+ Object.assign(runtime, {
63
+ page,
64
+ emulationAppliedPage: page,
65
+ viewport: { width: 1280, height: 800, deviceScaleFactor: 1, mobile: false, touch: false },
66
+ });
67
+ internal.requirePage = vi.fn(async () => page);
68
+ const emit = () =>
69
+ internal.onScreencastFrame(page, {
70
+ sessionId: 7,
71
+ data: jpeg(++tag),
72
+ metadata: { timestamp: (Date.now() + tag / 1000) / 1000 },
73
+ });
74
+ internal.startScreencastSession = vi.fn(async () => {
75
+ emit();
76
+ });
77
+ internal.captureScreenshot = vi.fn(
78
+ async (): Promise<RuntimeFrame> => ({
79
+ width: 1280,
80
+ height: 800,
81
+ transport: "screenshot",
82
+ dataBase64: jpeg(++tag),
83
+ byteLength: 24,
84
+ capturedAt: new Date().toISOString(),
85
+ }),
86
+ );
87
+ return { runtime, internal, page, emit };
88
+ }
89
+ /** A restored stream's first swap can fall inside the conservative source
90
+ * margin. Settle zero-wait polling so the honest screenshot fallback can finish. */
91
+ async function readFrame(runtime: AgentBrowserRuntime, quality: number = DEFAULT_JPEG_QUALITY) {
92
+ const pending = runtime.frame(100, quality, 0);
93
+ await vi.advanceTimersByTimeAsync(0);
94
+ return pending;
95
+ }
96
+
97
+ it("lazy image/agent read starts shared JPEG once, idle video-only interval stops it, fresh read restarts", async () => {
98
+ const { runtime, internal, page, emit } = setup();
99
+ expect(internal.startScreencastSession).not.toHaveBeenCalled();
100
+ const first = await readFrame(runtime);
101
+ await vi.advanceTimersByTimeAsync(1500);
102
+ emit();
103
+ await readFrame(runtime);
104
+ expect(internal.startScreencastSession).toHaveBeenCalledTimes(1);
105
+ await vi.advanceTimersByTimeAsync(JPEG_CAPTURE_IDLE_MS);
106
+ expect(page.send).toHaveBeenCalledWith("Page.stopScreencast", {}, { mutation: true });
107
+ expect(internal.screencastActive).toBe(false);
108
+ const renewed = await readFrame(runtime);
109
+ expect(internal.startScreencastSession).toHaveBeenCalledTimes(2);
110
+ expect(renewed.dataBase64).not.toBe(first.dataBase64);
111
+ await runtime.stopScreencast();
112
+ });
113
+ it("nonmatching viewer quality stays screenshot-only and cannot keep JPEG95 running", async () => {
114
+ const { runtime, internal } = setup();
115
+ for (const quality of [70, 90, 100]) await readFrame(runtime, quality);
116
+ expect(internal.startScreencastSession).not.toHaveBeenCalled();
117
+ expect(internal.captureScreenshot).toHaveBeenCalledTimes(3);
118
+ await vi.advanceTimersByTimeAsync(JPEG_CAPTURE_IDLE_MS);
119
+ expect(internal.screencastActive).toBe(false);
120
+ });
121
+ it("switching to screenshot-only quality does not renew the former shared stream", async () => {
122
+ const { runtime, internal } = setup();
123
+ await readFrame(runtime);
124
+ for (let i = 0; i < 7; i++) {
125
+ await vi.advanceTimersByTimeAsync(500);
126
+ await readFrame(runtime, 100);
127
+ }
128
+ expect(internal.screencastActive).toBe(false);
129
+ expect(internal.startScreencastSession).toHaveBeenCalledTimes(1);
130
+ });
131
+
132
+ it("stream startup and screenshot completion both pin retirement beyond the idle deadline", async () => {
133
+ const { runtime, internal, page, emit } = setup();
134
+ const startup = Promise.withResolvers<void>();
135
+ internal.startScreencastSession = vi.fn(async () => {
136
+ await startup.promise;
137
+ emit();
138
+ });
139
+ const read = runtime.frame(100, DEFAULT_JPEG_QUALITY, 0);
140
+ await vi.advanceTimersByTimeAsync(10000);
141
+ expect(page.send).not.toHaveBeenCalledWith("Page.stopScreencast", {}, { mutation: true });
142
+ startup.resolve();
143
+ await read;
144
+ const screenshot = Promise.withResolvers<RuntimeFrame>();
145
+ internal.captureScreenshot = vi.fn(() => screenshot.promise);
146
+ const fallback = runtime.frame(100, 100, 0);
147
+ await vi.advanceTimersByTimeAsync(10000);
148
+ expect(internal.screencastActive).toBe(true);
149
+ screenshot.resolve({
150
+ width: 1280,
151
+ height: 800,
152
+ transport: "screenshot",
153
+ dataBase64: jpeg(9),
154
+ byteLength: 24,
155
+ capturedAt: new Date().toISOString(),
156
+ });
157
+ await fallback;
158
+ await vi.advanceTimersByTimeAsync(JPEG_CAPTURE_IDLE_MS);
159
+ expect(internal.screencastActive).toBe(false);
160
+ });
161
+ it("renewed read waits for exact old stop acknowledgment before restarting", async () => {
162
+ const { runtime, internal, page } = setup();
163
+ await readFrame(runtime);
164
+ const stop = Promise.withResolvers<object>();
165
+ page.send.mockImplementation(async (method) =>
166
+ method === "Page.stopScreencast" ? await stop.promise : {},
167
+ );
168
+ await vi.advanceTimersByTimeAsync(JPEG_CAPTURE_IDLE_MS);
169
+ const renewed = runtime.frame(100, DEFAULT_JPEG_QUALITY, 0);
170
+ await vi.advanceTimersByTimeAsync(100);
171
+ expect(internal.startScreencastSession).toHaveBeenCalledTimes(1);
172
+ stop.resolve({});
173
+ await vi.advanceTimersByTimeAsync(0);
174
+ await renewed;
175
+ expect(internal.startScreencastSession).toHaveBeenCalledTimes(2);
176
+ await runtime.stopScreencast();
177
+ });
178
+ it("failed optional stream falls back without setup retry every image poll; idle renew permits new setup", async () => {
179
+ const { runtime, internal } = setup();
180
+ internal.startScreencastSession = vi.fn(async () => {
181
+ throw new Error("unsupported stream");
182
+ });
183
+ await readFrame(runtime);
184
+ await readFrame(runtime);
185
+ expect(internal.startScreencastSession).toHaveBeenCalledTimes(1);
186
+ expect(internal.captureScreenshot).toHaveBeenCalledTimes(2);
187
+ await vi.advanceTimersByTimeAsync(JPEG_CAPTURE_IDLE_MS);
188
+ await readFrame(runtime);
189
+ expect(internal.startScreencastSession).toHaveBeenCalledTimes(2);
190
+ await runtime.stopScreencast();
191
+ });
192
+ it("shutdown cancels runtime-owned JPEG timer even without explicit viewer detach", async () => {
193
+ const { runtime, internal, page } = setup();
194
+ await readFrame(runtime);
195
+ internal.invoke = vi.fn(async () => ({}));
196
+ await runtime.shutdown();
197
+ await vi.advanceTimersByTimeAsync(10000);
198
+ expect(page.send).not.toHaveBeenCalledWith("Page.stopScreencast", {}, { mutation: true });
199
+ expect(vi.getTimerCount()).toBe(0);
200
+ });