@omercnet/paseo-shared-browser 1.2.2-next.232.1 → 1.2.2-next.239.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. package/README.md +312 -25
  2. package/REMOTE_BROWSER_RESEARCH.md +395 -0
  3. package/STREAMING_DESIGN.md +191 -0
  4. package/client/browser-auto-capture-density.test.ts +102 -0
  5. package/client/browser-auto-capture-density.ts +30 -0
  6. package/client/browser-canvas-input.test.ts +198 -0
  7. package/client/browser-canvas-input.ts +202 -0
  8. package/client/browser-canvas-keyboard-web.test.ts +113 -0
  9. package/client/browser-canvas-keyboard-web.ts +230 -0
  10. package/client/browser-canvas-keyboard.test.ts +123 -0
  11. package/client/browser-canvas-keyboard.ts +126 -0
  12. package/client/browser-canvas-layout.test.ts +109 -0
  13. package/client/browser-canvas-layout.ts +51 -0
  14. package/client/browser-canvas-viewport-web.test.ts +77 -0
  15. package/client/browser-canvas-viewport-web.ts +63 -0
  16. package/client/browser-canvas-viewport.tsx +152 -0
  17. package/client/browser-capture-cadence.test.ts +9 -0
  18. package/client/browser-capture-cadence.ts +6 -0
  19. package/client/browser-capture-density-controls.tsx +75 -0
  20. package/client/browser-chrome.tsx +234 -0
  21. package/client/browser-client-presence.ts +184 -0
  22. package/client/browser-compose-controls.test.tsx +275 -0
  23. package/client/browser-compose-controls.tsx +174 -0
  24. package/client/browser-control-button.tsx +109 -0
  25. package/client/browser-control-dialog.tsx +71 -0
  26. package/client/browser-control-request.test.ts +27 -0
  27. package/client/browser-control-request.ts +57 -0
  28. package/client/browser-control-tooltip-web.ts +10 -0
  29. package/client/browser-dialog-body.tsx +22 -0
  30. package/client/browser-emulation-mode.test.ts +75 -0
  31. package/client/browser-emulation-mode.ts +98 -0
  32. package/client/browser-fill-viewport.test.ts +45 -0
  33. package/client/browser-fill-viewport.ts +28 -0
  34. package/client/browser-frame-buffer.test.ts +157 -0
  35. package/client/browser-frame-buffer.ts +148 -0
  36. package/client/browser-frame-image.tsx +62 -0
  37. package/client/browser-image-load.test.ts +23 -0
  38. package/client/browser-image-load.ts +26 -0
  39. package/client/browser-input-queue.test.ts +1201 -0
  40. package/client/browser-input-queue.ts +505 -0
  41. package/client/browser-menu-placement.test.ts +90 -0
  42. package/client/browser-menu-placement.ts +78 -0
  43. package/client/browser-panel-styles.ts +384 -0
  44. package/client/browser-quality-controls.test.ts +26 -0
  45. package/client/browser-quality-controls.tsx +139 -0
  46. package/client/browser-resolution-picker.tsx +189 -0
  47. package/client/browser-tabs.tsx +117 -0
  48. package/client/browser-toolbar-menu-web.test.ts +185 -0
  49. package/client/browser-toolbar-menu-web.ts +159 -0
  50. package/client/browser-toolbar-menu.tsx +255 -0
  51. package/client/browser-toolbar-submenu.tsx +93 -0
  52. package/client/browser-video-bytes.test.ts +44 -0
  53. package/client/browser-video-decoder.test.ts +473 -0
  54. package/client/browser-video-decoder.ts +311 -0
  55. package/client/browser-video-surface.tsx +21 -0
  56. package/client/browser-video-visibility.test.ts +45 -0
  57. package/client/browser-viewer-recovery.test.ts +111 -0
  58. package/client/browser-viewer-recovery.ts +78 -0
  59. package/client/browser.tsx +1642 -1569
  60. package/client/favorite-star.tsx +23 -0
  61. package/client/frame-lifecycle.ts +62 -59
  62. package/client/use-browser-auto-capture-density.test.ts +104 -0
  63. package/client/use-browser-auto-capture-density.ts +43 -0
  64. package/client/use-browser-canvas-input.ts +370 -0
  65. package/client/use-browser-capture-density.ts +87 -0
  66. package/client/use-browser-emulation-mode.ts +56 -0
  67. package/client/use-browser-fill-viewport.test.ts +108 -0
  68. package/client/use-browser-fill-viewport.ts +62 -0
  69. package/client/use-browser-frame-buffer.ts +66 -0
  70. package/client/use-browser-image-capture.ts +139 -0
  71. package/client/use-browser-scoped-mutation.test.ts +118 -0
  72. package/client/use-browser-scoped-mutation.ts +59 -0
  73. package/client/use-browser-video.test.ts +991 -0
  74. package/client/use-browser-video.ts +474 -0
  75. package/client/use-browser-viewer-recovery.ts +26 -0
  76. package/client/use-resolution-favorites.ts +90 -0
  77. package/client/web.test.ts +125 -0
  78. package/client/web.ts +405 -0
  79. package/index.client.tsx +0 -10
  80. package/index.server.ts +32 -0
  81. package/npm-shrinkwrap.json +7 -7
  82. package/package.json +9 -5
  83. package/paseo-plugin.json +3 -1
  84. package/scripts/prepare-runtime.mjs +1 -1
  85. package/server/agent-browser-runtime.ts +1598 -314
  86. package/server/browser-cursor.test.ts +64 -0
  87. package/server/browser-cursor.ts +69 -0
  88. package/server/browser-frame-context.ts +22 -0
  89. package/server/browser-frame-epoch.test.ts +95 -0
  90. package/server/browser-frame-rejection.ts +17 -0
  91. package/server/browser-gesture.ts +120 -0
  92. package/server/browser-operation-diagnostics.test.ts +79 -0
  93. package/server/browser-operation-diagnostics.ts +103 -0
  94. package/server/browser-policy.ts +1313 -151
  95. package/server/browser-video-input-authority.test.ts +208 -0
  96. package/server/browser-video-policy.test.ts +564 -0
  97. package/server/browser-video-socket.test.ts +368 -0
  98. package/server/browser.ts +151 -16
  99. package/server/capture-transport-policy.test.ts +105 -0
  100. package/server/capture-transport-policy.ts +154 -0
  101. package/server/capture-transport-runtime.test.ts +213 -0
  102. package/server/daemon-launch-chain.test.ts +70 -0
  103. package/server/daemon-shutdown.test.ts +289 -0
  104. package/server/discrete-input-freshness.test.ts +364 -0
  105. package/server/discrete-input-native-lifetime.test.ts +150 -0
  106. package/server/display-loss-recovery.test.ts +294 -0
  107. package/server/emulation-restoration.test.ts +262 -0
  108. package/server/input-cleanup.ts +23 -0
  109. package/server/input-generation.test.ts +31 -0
  110. package/server/input-generation.ts +36 -0
  111. package/server/jpeg-capture-demand.test.ts +76 -0
  112. package/server/jpeg-capture-demand.ts +98 -0
  113. package/server/jpeg-frame.ts +29 -0
  114. package/server/keyboard-input.test.ts +61 -0
  115. package/server/keyboard-input.ts +104 -0
  116. package/server/live-input-policy.test.ts +958 -0
  117. package/server/live-input-runtime.test.ts +732 -0
  118. package/server/live-input-supervisor.test.ts +147 -0
  119. package/server/mcp-entry.ts +44 -2
  120. package/server/native-video-backpressure.test.ts +228 -0
  121. package/server/native-video-buffer.test.ts +141 -0
  122. package/server/native-video-buffer.ts +114 -0
  123. package/server/native-video-capture.test.ts +599 -0
  124. package/server/native-video-capture.ts +497 -0
  125. package/server/native-video-extension-source.test.ts +201 -0
  126. package/server/native-video-extension-source.ts +364 -0
  127. package/server/native-video-extension.test.ts +41 -0
  128. package/server/native-video-extension.ts +83 -0
  129. package/server/native-video-packet.ts +9 -0
  130. package/server/native-video-settings.test.ts +245 -0
  131. package/server/navigation-budget.ts +8 -0
  132. package/server/navigation-commit.ts +125 -0
  133. package/server/navigation-policy.test.ts +153 -0
  134. package/server/process-identity.ts +58 -0
  135. package/server/reattach-held-input.test.ts +143 -0
  136. package/server/runtime-frame-reuse.test.ts +295 -0
  137. package/server/runtime-jpeg-demand.test.ts +200 -0
  138. package/server/runtime-jpeg-shutdown.test.ts +210 -0
  139. package/server/runtime-jpeg-source-order.test.ts +201 -0
  140. package/server/runtime-launch-environment.test.ts +111 -0
  141. package/server/runtime-navigation.test.ts +327 -0
  142. package/server/runtime-owner-display.test.ts +152 -0
  143. package/server/runtime-owner.ts +305 -31
  144. package/server/runtime-page-state.test.ts +249 -0
  145. package/server/runtime-protocol.ts +31 -1
  146. package/server/runtime-target-retention.test.ts +181 -0
  147. package/server/runtime-video-opt-in.test.ts +82 -0
  148. package/server/runtime-video.test.ts +444 -0
  149. package/server/screencast-source-time.ts +46 -0
  150. package/server/stale-wheel-admission.test.ts +136 -0
  151. package/server/supervisor-client.ts +28 -4
  152. package/server/supervisor-heartbeat.test.ts +80 -0
  153. package/server/supervisor.ts +553 -55
  154. package/server/video-source-recovery.test.ts +20 -0
  155. package/server/video-source-recovery.ts +64 -0
  156. package/server/viewer-capture-lifetime.ts +51 -0
  157. package/server/virtual-display.test.ts +184 -0
  158. package/server/virtual-display.ts +236 -0
  159. package/shared/browser-display-preferences.ts +46 -0
  160. package/shared/browser-video.ts +80 -0
  161. package/shared/browser.ts +286 -65
  162. package/shared/capture-density.ts +16 -0
  163. package/shared/capture-settings.ts +17 -0
  164. package/shared/device-presets.ts +90 -0
  165. package/shared/resolution-menu.test.ts +120 -0
  166. package/shared/resolution-menu.ts +72 -0
  167. package/shared/video-settings.test.ts +38 -0
  168. package/shared/video-settings.ts +34 -0
  169. package/shared/viewport-limits.ts +5 -0
@@ -36,7 +36,11 @@ export type AgentBrowserOperation =
36
36
  | "release-control"
37
37
  | "navigate"
38
38
  | "input"
39
- | "viewport";
39
+ | "viewport"
40
+ | "tabs.list"
41
+ | "tabs.create"
42
+ | "tabs.select"
43
+ | "tabs.close";
40
44
 
41
45
  export type RuntimeRequest =
42
46
  | (AdminRequestBase & { method: "bridge.claim"; takeover?: boolean })
@@ -58,6 +62,12 @@ export type RuntimeRequest =
58
62
  epoch: number;
59
63
  workspaceId: string;
60
64
  })
65
+ | (AdminRequestBase & {
66
+ method: "workspace.close";
67
+ epoch: number;
68
+ workspaceId: string;
69
+ runtimeId: string;
70
+ })
61
71
  | (AdminRequestBase & {
62
72
  method: "browser.request";
63
73
  epoch: number;
@@ -107,6 +117,7 @@ export type RuntimeErrorCode =
107
117
  | "WORKSPACE_NOT_FOUND"
108
118
  | "RUNTIME_BUSY"
109
119
  | "UNKNOWN_OUTCOME"
120
+ | "RUNTIME_LOST"
110
121
  | "RUNTIME_FAILURE";
111
122
 
112
123
  export class RuntimeProtocolError extends Error {
@@ -119,6 +130,14 @@ export class RuntimeProtocolError extends Error {
119
130
  }
120
131
  }
121
132
 
133
+ /** The owned runtime can never serve again; the next explicit attach must replace it. */
134
+ export class RuntimeLostError extends Error {
135
+ constructor(message: string) {
136
+ super(message);
137
+ this.name = "RuntimeLostError";
138
+ }
139
+ }
140
+
122
141
  export function isRuntimeResponse(value: unknown): value is RuntimeResponse {
123
142
  if (!isRecord(value) || typeof value.id !== "string" || typeof value.ok !== "boolean")
124
143
  return false;
@@ -183,6 +202,17 @@ export function parseRuntimeRequest(value: unknown): RuntimeRequest {
183
202
  epoch,
184
203
  workspaceId: requireString(value, "workspaceId"),
185
204
  };
205
+ if (method === "workspace.close")
206
+ return {
207
+ id,
208
+ version: RUNTIME_PROTOCOL_VERSION,
209
+ method,
210
+ token,
211
+ bridgeId,
212
+ epoch,
213
+ workspaceId: requireString(value, "workspaceId"),
214
+ runtimeId: requireString(value, "runtimeId"),
215
+ };
186
216
  if (method === "workspace.request")
187
217
  return {
188
218
  id,
@@ -0,0 +1,181 @@
1
+ /** Exercise real target attachment/reconnect methods without native sockets or profiles. */
2
+ import { EventEmitter } from "node:events";
3
+ import { afterEach, describe, expect, it, vi } from "vitest";
4
+ import { AgentBrowserRuntime } from "./agent-browser-runtime";
5
+ import { CdpConnection, type CdpSession } from "./cdp";
6
+
7
+ class ConnectionFixture extends EventEmitter {
8
+ isOpen = true;
9
+ nextSession = 0;
10
+ readonly calls: { method: string; params: Record<string, unknown> }[] = [];
11
+ constructor(readonly targetIds: string[]) {
12
+ super();
13
+ }
14
+
15
+ async send<T>(method: string, params: Record<string, unknown> = {}): Promise<T> {
16
+ this.calls.push({ method, params });
17
+ if (method === "Target.getTargets")
18
+ return {
19
+ targetInfos: this.targetIds.map((targetId) => ({
20
+ targetId,
21
+ type: "page",
22
+ url: "about:blank",
23
+ title: targetId,
24
+ })),
25
+ } as T;
26
+ if (method === "Target.attachToTarget")
27
+ return { sessionId: `fixture-${++this.nextSession}` } as T;
28
+ if (method === "Page.getNavigationHistory") return { currentIndex: 0, entries: [] } as T;
29
+ if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "root" } } } as T;
30
+ return {} as T;
31
+ }
32
+
33
+ close(): void {
34
+ this.isOpen = false;
35
+ this.emit("disconnect");
36
+ }
37
+ }
38
+
39
+ interface RuntimeControl {
40
+ invoke(args: string[]): Promise<unknown>;
41
+ assertVersion(): Promise<void>;
42
+ connectCdp(targetId?: string | null): Promise<void>;
43
+ reattachPageForScreencast(previous: CdpSession): Promise<CdpSession>;
44
+ connection: CdpConnection;
45
+ targetId: string | null;
46
+ page: CdpSession | null;
47
+ }
48
+
49
+ /** Stub only binary discovery and CDP transport, preserving runtime selection policy. */
50
+ function fixture(targetLists: string[][]) {
51
+ const connections = targetLists.map((ids) => new ConnectionFixture(ids));
52
+ let next = 0;
53
+ vi.spyOn(CdpConnection, "connect").mockImplementation(async () => {
54
+ const connection = connections[next++];
55
+ if (!connection) throw new Error("Unexpected reconnect");
56
+ return connection as unknown as CdpConnection;
57
+ });
58
+ const runtime = new AgentBrowserRuntime({
59
+ binaryPath: "/tmp/unlaunched-browser",
60
+ executablePath: "/tmp/unlaunched-chromium",
61
+ profilePath: "/tmp/uncreated-profile",
62
+ ipcDirectory: "/tmp/uncreated-ipc",
63
+ session: "target-retention-test",
64
+ });
65
+ const control = runtime as unknown as RuntimeControl;
66
+ control.invoke = vi.fn(async () => ({ cdpUrl: "ws://127.0.0.1:1/fixture" }));
67
+ control.assertVersion = async () => {};
68
+ return { runtime, control, connections };
69
+ }
70
+
71
+ afterEach(() => vi.restoreAllMocks());
72
+ describe("exact selected target retention", () => {
73
+ it("permits initial selection when no page has ever been selected", async () => {
74
+ const { control } = fixture([["first", "second"]]);
75
+ await control.connectCdp();
76
+ expect(control.targetId).toBe("first");
77
+ });
78
+
79
+ it("retains the selected surviving page across disconnect instead of adopting the first tab", async () => {
80
+ const { runtime, control, connections } = fixture([
81
+ ["other", "original"],
82
+ ["other", "original"],
83
+ ]);
84
+ await control.connectCdp("original");
85
+ connections[0]!.close();
86
+ expect(control.page).toBeNull();
87
+ expect(control.targetId).toBe("original");
88
+ await runtime.reconnect();
89
+ expect(control.page?.targetId).toBe("original");
90
+ expect(
91
+ connections[1]!.calls
92
+ .filter((call) => call.method === "Target.activateTarget")
93
+ .map((call) => call.params.targetId),
94
+ ).toEqual(["original"]);
95
+ expect(control.invoke).toHaveBeenCalledWith([
96
+ "--session",
97
+ "target-retention-test",
98
+ "--json",
99
+ "get",
100
+ "cdp-url",
101
+ ]);
102
+ expect(
103
+ connections
104
+ .flatMap((connection) => connection.calls)
105
+ .some((call) => call.method === "Page.navigate"),
106
+ ).toBe(false);
107
+ });
108
+
109
+ it("refuses a missing selected page on every reconnect without adopting another tab", async () => {
110
+ const { runtime, control, connections } = fixture([["original"], ["other"], ["other"]]);
111
+ await control.connectCdp("original");
112
+ connections[0]!.close();
113
+ await expect(runtime.reconnect()).rejects.toThrow(
114
+ "selected browser page is no longer available",
115
+ );
116
+ await expect(runtime.reconnect()).rejects.toThrow(
117
+ "selected browser page is no longer available",
118
+ );
119
+ expect(control.targetId).toBe("original");
120
+ expect(control.page).toBeNull();
121
+ expect(
122
+ connections
123
+ .slice(1)
124
+ .flatMap((connection) => connection.calls)
125
+ .some(
126
+ (call) =>
127
+ call.method === "Target.attachToTarget" || call.method === "Target.activateTarget",
128
+ ),
129
+ ).toBe(false);
130
+ });
131
+
132
+ it("requires an explicit initial target rather than replacing a missing launch target", async () => {
133
+ const { runtime, control, connections } = fixture([["other"], ["other"]]);
134
+ await expect(control.connectCdp("missing")).rejects.toThrow(
135
+ "selected browser page is no longer available",
136
+ );
137
+ expect(control.targetId).toBe("missing");
138
+ await expect(runtime.reconnect()).rejects.toThrow(
139
+ "selected browser page is no longer available",
140
+ );
141
+ expect(control.page).toBeNull();
142
+ expect(
143
+ connections
144
+ .flatMap((connection) => connection.calls)
145
+ .some((call) => call.method === "Target.activateTarget"),
146
+ ).toBe(false);
147
+ });
148
+
149
+ it("retains a newly explicitly selected page across later disconnects", async () => {
150
+ const { runtime, control, connections } = fixture([
151
+ ["first", "chosen"],
152
+ ["first", "chosen"],
153
+ ]);
154
+ await control.connectCdp();
155
+ await runtime.selectTarget("chosen");
156
+ connections[0]!.close();
157
+ await runtime.reconnect();
158
+ expect(control.page?.targetId).toBe("chosen");
159
+ });
160
+
161
+ it("refuses screencast reattachment when its exact former page disappeared", async () => {
162
+ const { control, connections } = fixture([["original", "other"]]);
163
+ await control.connectCdp("original");
164
+ const previous = control.page!;
165
+ connections[0]!.targetIds.splice(0, 1);
166
+ const before = connections[0]!.calls.length;
167
+ await expect(control.reattachPageForScreencast(previous)).rejects.toThrow(
168
+ "selected browser page is no longer available",
169
+ );
170
+ expect(control.page).toBe(previous);
171
+ expect(control.targetId).toBe("original");
172
+ expect(
173
+ connections[0]!.calls
174
+ .slice(before)
175
+ .some(
176
+ (call) =>
177
+ call.method === "Target.activateTarget" || call.method === "Target.attachToTarget",
178
+ ),
179
+ ).toBe(false);
180
+ });
181
+ });
@@ -0,0 +1,82 @@
1
+ /** Encoded video is host-opt-in: image-only sessions never load the capture extension. */
2
+
3
+ import { mkdtemp, readdir, rm } from "node:fs/promises";
4
+ import { tmpdir } from "node:os";
5
+ import { join } from "node:path";
6
+ import { afterEach, describe, expect, it, vi } from "vitest";
7
+ import { AgentBrowserRuntime } from "./agent-browser-runtime";
8
+
9
+ // The owned CLI is simulated at the process boundary, as in daemon-shutdown.test.ts: a
10
+ // POSIX-shebang script is not executable on Windows, so no executable file is required.
11
+ vi.mock("node:child_process", async (importOriginal) => ({
12
+ ...(await importOriginal<typeof import("node:child_process")>()),
13
+ execFile: (
14
+ _file: string,
15
+ _args: string[],
16
+ _options: unknown,
17
+ callback: (error: null, result: { stdout: string }) => void,
18
+ ) => callback(null, { stdout: "0.38.2\n" }),
19
+ }));
20
+
21
+ const temporary: string[] = [];
22
+ afterEach(async () => {
23
+ await Promise.all(temporary.splice(0).map((path) => rm(path, { recursive: true, force: true })));
24
+ });
25
+
26
+ async function fixture(nativeVideo?: boolean) {
27
+ const directory = await mkdtemp(join(tmpdir(), "shared-browser-video-opt-in-"));
28
+ temporary.push(directory);
29
+ const binary = join(directory, "owned-cli");
30
+ const ipcDirectory = join(directory, "ipc");
31
+ const runtime = new AgentBrowserRuntime({
32
+ binaryPath: binary,
33
+ executablePath: join(directory, "unlaunched-chromium"),
34
+ profilePath: join(directory, "profile"),
35
+ ipcDirectory,
36
+ session: "video-opt-in",
37
+ ...(nativeVideo === undefined ? {} : { nativeVideo }),
38
+ });
39
+ const internal = runtime as unknown as {
40
+ invoke(args: string[]): Promise<unknown>;
41
+ video: unknown;
42
+ };
43
+ const launches: string[][] = [];
44
+ internal.invoke = async (args) => {
45
+ launches.push(args);
46
+ throw new Error("Owned fixture stops before a browser exists");
47
+ };
48
+ return { runtime, internal, launches, ipcDirectory };
49
+ }
50
+
51
+ describe("encoded video opt-in", () => {
52
+ it("launches image-only sessions without the extension, its allowlist or any materialized files", async () => {
53
+ for (const option of [undefined, false]) {
54
+ const f = await fixture(option);
55
+ await expect(f.runtime.launch()).rejects.toThrow("stops before a browser");
56
+ const [args] = f.launches;
57
+ expect(args).not.toContain("--extension");
58
+ expect(args?.join(" ")).not.toContain("allowlisted-extension-id");
59
+ expect(await readdir(f.ipcDirectory)).toEqual([]);
60
+ }
61
+ });
62
+
63
+ it("loads and allowlists the capture extension only when the host opted in", async () => {
64
+ const f = await fixture(true);
65
+ await expect(f.runtime.launch()).rejects.toThrow("stops before a browser");
66
+ const [args] = f.launches;
67
+ expect(args).toContain("--extension");
68
+ expect(args?.join(" ")).toContain("--allowlisted-extension-id=");
69
+ });
70
+
71
+ it("reports a typed disabled reply without creating any capture source", async () => {
72
+ const f = await fixture();
73
+ const reply = await f.runtime.readVideo({ quality: "high" });
74
+ expect(reply).toMatchObject({
75
+ status: "unsupported",
76
+ reasonCode: "video-disabled",
77
+ streamId: null,
78
+ packets: [],
79
+ });
80
+ expect(f.internal.video).toBeNull();
81
+ });
82
+ });