@omercnet/paseo-shared-browser 1.2.2-next.232.1 → 1.2.2-next.237.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/README.md +312 -25
  2. package/REMOTE_BROWSER_RESEARCH.md +395 -0
  3. package/STREAMING_DESIGN.md +191 -0
  4. package/client/browser-auto-capture-density.test.ts +102 -0
  5. package/client/browser-auto-capture-density.ts +30 -0
  6. package/client/browser-canvas-input.test.ts +198 -0
  7. package/client/browser-canvas-input.ts +202 -0
  8. package/client/browser-canvas-keyboard-web.test.ts +113 -0
  9. package/client/browser-canvas-keyboard-web.ts +230 -0
  10. package/client/browser-canvas-keyboard.test.ts +123 -0
  11. package/client/browser-canvas-keyboard.ts +126 -0
  12. package/client/browser-canvas-layout.test.ts +109 -0
  13. package/client/browser-canvas-layout.ts +51 -0
  14. package/client/browser-canvas-viewport-web.test.ts +77 -0
  15. package/client/browser-canvas-viewport-web.ts +63 -0
  16. package/client/browser-canvas-viewport.tsx +152 -0
  17. package/client/browser-capture-cadence.test.ts +9 -0
  18. package/client/browser-capture-cadence.ts +6 -0
  19. package/client/browser-capture-density-controls.tsx +75 -0
  20. package/client/browser-chrome.tsx +234 -0
  21. package/client/browser-client-presence.ts +184 -0
  22. package/client/browser-compose-controls.test.tsx +275 -0
  23. package/client/browser-compose-controls.tsx +174 -0
  24. package/client/browser-control-button.tsx +109 -0
  25. package/client/browser-control-dialog.tsx +71 -0
  26. package/client/browser-control-request.test.ts +27 -0
  27. package/client/browser-control-request.ts +57 -0
  28. package/client/browser-control-tooltip-web.ts +10 -0
  29. package/client/browser-dialog-body.tsx +22 -0
  30. package/client/browser-emulation-mode.test.ts +75 -0
  31. package/client/browser-emulation-mode.ts +98 -0
  32. package/client/browser-fill-viewport.test.ts +45 -0
  33. package/client/browser-fill-viewport.ts +28 -0
  34. package/client/browser-frame-buffer.test.ts +157 -0
  35. package/client/browser-frame-buffer.ts +148 -0
  36. package/client/browser-frame-image.tsx +62 -0
  37. package/client/browser-image-load.test.ts +23 -0
  38. package/client/browser-image-load.ts +26 -0
  39. package/client/browser-input-queue.test.ts +1201 -0
  40. package/client/browser-input-queue.ts +505 -0
  41. package/client/browser-menu-placement.test.ts +90 -0
  42. package/client/browser-menu-placement.ts +78 -0
  43. package/client/browser-panel-styles.ts +384 -0
  44. package/client/browser-quality-controls.test.ts +26 -0
  45. package/client/browser-quality-controls.tsx +139 -0
  46. package/client/browser-resolution-picker.tsx +189 -0
  47. package/client/browser-tabs.tsx +117 -0
  48. package/client/browser-toolbar-menu-web.test.ts +185 -0
  49. package/client/browser-toolbar-menu-web.ts +159 -0
  50. package/client/browser-toolbar-menu.tsx +255 -0
  51. package/client/browser-toolbar-submenu.tsx +93 -0
  52. package/client/browser-video-bytes.test.ts +44 -0
  53. package/client/browser-video-decoder.test.ts +473 -0
  54. package/client/browser-video-decoder.ts +311 -0
  55. package/client/browser-video-surface.tsx +21 -0
  56. package/client/browser-video-visibility.test.ts +45 -0
  57. package/client/browser-viewer-recovery.test.ts +111 -0
  58. package/client/browser-viewer-recovery.ts +78 -0
  59. package/client/browser.tsx +1642 -1569
  60. package/client/favorite-star.tsx +23 -0
  61. package/client/frame-lifecycle.ts +62 -59
  62. package/client/use-browser-auto-capture-density.test.ts +104 -0
  63. package/client/use-browser-auto-capture-density.ts +43 -0
  64. package/client/use-browser-canvas-input.ts +370 -0
  65. package/client/use-browser-capture-density.ts +87 -0
  66. package/client/use-browser-emulation-mode.ts +56 -0
  67. package/client/use-browser-fill-viewport.test.ts +108 -0
  68. package/client/use-browser-fill-viewport.ts +62 -0
  69. package/client/use-browser-frame-buffer.ts +66 -0
  70. package/client/use-browser-image-capture.ts +139 -0
  71. package/client/use-browser-scoped-mutation.test.ts +118 -0
  72. package/client/use-browser-scoped-mutation.ts +59 -0
  73. package/client/use-browser-video.test.ts +991 -0
  74. package/client/use-browser-video.ts +474 -0
  75. package/client/use-browser-viewer-recovery.ts +26 -0
  76. package/client/use-resolution-favorites.ts +90 -0
  77. package/client/web.test.ts +125 -0
  78. package/client/web.ts +405 -0
  79. package/index.client.tsx +0 -10
  80. package/index.server.ts +32 -0
  81. package/package.json +9 -5
  82. package/paseo-plugin.json +3 -1
  83. package/scripts/prepare-runtime.mjs +1 -1
  84. package/server/agent-browser-runtime.ts +1598 -314
  85. package/server/browser-cursor.test.ts +64 -0
  86. package/server/browser-cursor.ts +69 -0
  87. package/server/browser-frame-context.ts +22 -0
  88. package/server/browser-frame-epoch.test.ts +95 -0
  89. package/server/browser-frame-rejection.ts +17 -0
  90. package/server/browser-gesture.ts +120 -0
  91. package/server/browser-operation-diagnostics.test.ts +79 -0
  92. package/server/browser-operation-diagnostics.ts +103 -0
  93. package/server/browser-policy.ts +1313 -151
  94. package/server/browser-video-input-authority.test.ts +208 -0
  95. package/server/browser-video-policy.test.ts +564 -0
  96. package/server/browser-video-socket.test.ts +368 -0
  97. package/server/browser.ts +151 -16
  98. package/server/capture-transport-policy.test.ts +105 -0
  99. package/server/capture-transport-policy.ts +154 -0
  100. package/server/capture-transport-runtime.test.ts +213 -0
  101. package/server/daemon-launch-chain.test.ts +70 -0
  102. package/server/daemon-shutdown.test.ts +289 -0
  103. package/server/discrete-input-freshness.test.ts +364 -0
  104. package/server/discrete-input-native-lifetime.test.ts +150 -0
  105. package/server/display-loss-recovery.test.ts +294 -0
  106. package/server/emulation-restoration.test.ts +262 -0
  107. package/server/input-cleanup.ts +23 -0
  108. package/server/input-generation.test.ts +31 -0
  109. package/server/input-generation.ts +36 -0
  110. package/server/jpeg-capture-demand.test.ts +76 -0
  111. package/server/jpeg-capture-demand.ts +98 -0
  112. package/server/jpeg-frame.ts +29 -0
  113. package/server/keyboard-input.test.ts +61 -0
  114. package/server/keyboard-input.ts +104 -0
  115. package/server/live-input-policy.test.ts +958 -0
  116. package/server/live-input-runtime.test.ts +732 -0
  117. package/server/live-input-supervisor.test.ts +147 -0
  118. package/server/mcp-entry.ts +44 -2
  119. package/server/native-video-backpressure.test.ts +228 -0
  120. package/server/native-video-buffer.test.ts +141 -0
  121. package/server/native-video-buffer.ts +114 -0
  122. package/server/native-video-capture.test.ts +599 -0
  123. package/server/native-video-capture.ts +497 -0
  124. package/server/native-video-extension-source.test.ts +201 -0
  125. package/server/native-video-extension-source.ts +364 -0
  126. package/server/native-video-extension.test.ts +41 -0
  127. package/server/native-video-extension.ts +83 -0
  128. package/server/native-video-packet.ts +9 -0
  129. package/server/native-video-settings.test.ts +245 -0
  130. package/server/navigation-budget.ts +8 -0
  131. package/server/navigation-commit.ts +125 -0
  132. package/server/navigation-policy.test.ts +153 -0
  133. package/server/process-identity.ts +58 -0
  134. package/server/reattach-held-input.test.ts +143 -0
  135. package/server/runtime-frame-reuse.test.ts +295 -0
  136. package/server/runtime-jpeg-demand.test.ts +200 -0
  137. package/server/runtime-jpeg-shutdown.test.ts +210 -0
  138. package/server/runtime-jpeg-source-order.test.ts +201 -0
  139. package/server/runtime-launch-environment.test.ts +111 -0
  140. package/server/runtime-navigation.test.ts +327 -0
  141. package/server/runtime-owner-display.test.ts +152 -0
  142. package/server/runtime-owner.ts +305 -31
  143. package/server/runtime-page-state.test.ts +249 -0
  144. package/server/runtime-protocol.ts +31 -1
  145. package/server/runtime-target-retention.test.ts +181 -0
  146. package/server/runtime-video-opt-in.test.ts +82 -0
  147. package/server/runtime-video.test.ts +444 -0
  148. package/server/screencast-source-time.ts +46 -0
  149. package/server/stale-wheel-admission.test.ts +136 -0
  150. package/server/supervisor-client.ts +28 -4
  151. package/server/supervisor-heartbeat.test.ts +80 -0
  152. package/server/supervisor.ts +553 -55
  153. package/server/video-source-recovery.test.ts +20 -0
  154. package/server/video-source-recovery.ts +64 -0
  155. package/server/viewer-capture-lifetime.ts +51 -0
  156. package/server/virtual-display.test.ts +184 -0
  157. package/server/virtual-display.ts +236 -0
  158. package/shared/browser-display-preferences.ts +46 -0
  159. package/shared/browser-video.ts +80 -0
  160. package/shared/browser.ts +286 -65
  161. package/shared/capture-density.ts +16 -0
  162. package/shared/capture-settings.ts +17 -0
  163. package/shared/device-presets.ts +90 -0
  164. package/shared/resolution-menu.test.ts +120 -0
  165. package/shared/resolution-menu.ts +72 -0
  166. package/shared/video-settings.test.ts +38 -0
  167. package/shared/video-settings.ts +34 -0
  168. package/shared/viewport-limits.ts +5 -0
@@ -0,0 +1,125 @@
1
+ import { describe, expect, it, vi } from "vitest";
2
+
3
+ vi.mock("react-native", () => ({ Platform: { OS: "web" } }));
4
+
5
+ import type { BrowserGestureEvent } from "../shared/browser";
6
+ import { createBrowserCanvasInput } from "./browser-canvas-input";
7
+ import { type BrowserCanvasNode, bindBrowserCanvasWeb } from "./web";
8
+
9
+ type Handler = (event: Record<string, unknown>) => void;
10
+
11
+ /** Minimal DOM that records listeners; the real web adapter and real canvas input model run on it. */
12
+ function fixture() {
13
+ const registry = new Map<string, Set<Handler>>();
14
+ const target = (name: string) => ({
15
+ addEventListener(type: string, listener: Handler) {
16
+ const key = `${name}:${type}`;
17
+ registry.set(key, (registry.get(key) ?? new Set()).add(listener));
18
+ },
19
+ removeEventListener(type: string, listener: Handler) {
20
+ registry.get(`${name}:${type}`)?.delete(listener);
21
+ },
22
+ });
23
+ const document = {
24
+ hidden: false,
25
+ ...target("doc"),
26
+ defaultView: target("win"),
27
+ };
28
+ const node = {
29
+ getBoundingClientRect: () => ({ left: 100, top: 100, width: 400, height: 300 }),
30
+ ...target("node"),
31
+ style: { cursor: "", touchAction: "" },
32
+ ownerDocument: document,
33
+ } as unknown as BrowserCanvasNode;
34
+ const events: BrowserGestureEvent[] = [];
35
+ let cancels = 0;
36
+ const input = createBrowserCanvasInput({
37
+ enabled: () => true,
38
+ viewport: () => ({ width: 800, height: 600 }),
39
+ enqueue: (event) => {
40
+ events.push(event);
41
+ return true;
42
+ },
43
+ finish: () => {},
44
+ cancel: () => {
45
+ cancels += 1;
46
+ },
47
+ onPoint: () => {},
48
+ onActivity: () => {},
49
+ });
50
+ const dispose = bindBrowserCanvasWeb(
51
+ node,
52
+ input,
53
+ () => true,
54
+ () => {},
55
+ );
56
+ const fire = (where: string, type: string, event: Record<string, unknown> = {}) => {
57
+ for (const listener of registry.get(`${where}:${type}`) ?? [])
58
+ listener({ preventDefault() {}, stopPropagation() {}, ...event });
59
+ };
60
+ const kinds = () => events.map((event) => event.kind);
61
+ return { document, events, fire, kinds, cancels: () => cancels, dispose };
62
+ }
63
+
64
+ const inside = { clientX: 200, clientY: 200, button: 0, detail: 1 };
65
+ const outside = { clientX: 900, clientY: 900, button: 0, detail: 1 };
66
+ const touch = (x: number, y: number) => ({
67
+ touches: [{ identifier: 3, clientX: x, clientY: y }],
68
+ });
69
+
70
+ describe("web canvas adapter cancellation", () => {
71
+ it("window blur cancels a held drag and drops the late document mouseup", () => {
72
+ const f = fixture();
73
+ f.fire("node", "mousedown", inside);
74
+ f.fire("doc", "mousemove", outside);
75
+ expect(f.kinds()).toEqual(["down", "move"]);
76
+ f.fire("win", "blur");
77
+ expect(f.cancels()).toBe(1);
78
+ f.fire("doc", "mouseup", outside);
79
+ f.fire("doc", "mousemove", outside);
80
+ expect(f.kinds()).toEqual(["down", "move"]);
81
+ });
82
+
83
+ it("page hidden cancels a held button; a visible release afterwards publishes nothing", () => {
84
+ const f = fixture();
85
+ f.fire("node", "mousedown", inside);
86
+ f.document.hidden = true;
87
+ f.fire("doc", "visibilitychange");
88
+ expect(f.cancels()).toBe(1);
89
+ f.document.hidden = false;
90
+ f.fire("node", "mouseup", inside);
91
+ expect(f.kinds()).not.toContain("up");
92
+ });
93
+
94
+ it("a drag released outside the canvas still ends once at its pointer position, then leaves", () => {
95
+ const f = fixture();
96
+ f.fire("node", "mousedown", inside);
97
+ f.fire("doc", "mouseup", outside);
98
+ expect(f.events.filter((event) => event.kind === "up")).toHaveLength(1);
99
+ expect(f.cancels()).toBe(0);
100
+ });
101
+
102
+ it("blur cancels held touch and quarantines its late move/end until liftoff", () => {
103
+ const f = fixture();
104
+ f.fire("node", "touchstart", touch(200, 200));
105
+ expect(f.kinds()).toContain("touch");
106
+ const before = f.events.length;
107
+ f.fire("win", "blur");
108
+ expect(f.cancels()).toBe(1);
109
+ f.fire("node", "touchmove", touch(220, 220));
110
+ f.fire("node", "touchend", { touches: [] });
111
+ expect(f.events.length).toBe(before);
112
+ });
113
+
114
+ it("pointercancel and dispose cancel held input", () => {
115
+ const f = fixture();
116
+ f.fire("node", "mousedown", inside);
117
+ f.fire("node", "pointercancel");
118
+ expect(f.cancels()).toBe(1);
119
+ f.fire("node", "mousedown", inside);
120
+ f.dispose();
121
+ expect(f.cancels()).toBeGreaterThanOrEqual(2);
122
+ f.fire("doc", "mouseup", outside);
123
+ expect(f.events.filter((event) => event.kind === "up")).toHaveLength(0);
124
+ });
125
+ });
package/client/web.ts ADDED
@@ -0,0 +1,405 @@
1
+ /**
2
+ * Web-only input and video adapters for the native View's DOM host node. No DOM globals or
3
+ * handlers leak into the cross-platform panel. Non-passive wheel/touch listeners
4
+ * capture only a controlled image canvas; document mouse listeners preserve drags outside
5
+ * its rectangle and visibility/blur immediately cancels held remote input.
6
+ */
7
+ import { Platform } from "react-native";
8
+ import type { BrowserCursor } from "../shared/browser";
9
+ import type { createBrowserCanvasInput } from "./browser-canvas-input";
10
+
11
+ type CanvasInput = ReturnType<typeof createBrowserCanvasInput>;
12
+ interface Rect {
13
+ left: number;
14
+ top: number;
15
+ width: number;
16
+ height: number;
17
+ }
18
+ interface DomTouch {
19
+ identifier: number;
20
+ clientX: number;
21
+ clientY: number;
22
+ }
23
+ interface DomEvent {
24
+ clientX?: number;
25
+ clientY?: number;
26
+ button?: number;
27
+ detail?: number;
28
+ altKey?: boolean;
29
+ ctrlKey?: boolean;
30
+ metaKey?: boolean;
31
+ shiftKey?: boolean;
32
+ deltaX?: number;
33
+ deltaY?: number;
34
+ deltaMode?: number;
35
+ touches?: ArrayLike<DomTouch>;
36
+ sourceCapabilities?: { firesTouchEvents?: boolean };
37
+ preventDefault(): void;
38
+ stopPropagation(): void;
39
+ }
40
+ export interface BrowserCanvasNode {
41
+ getBoundingClientRect(): Rect;
42
+ addEventListener(
43
+ type: string,
44
+ listener: (event: DomEvent) => void,
45
+ options?: { passive: boolean },
46
+ ): void;
47
+ removeEventListener(type: string, listener: (event: DomEvent) => void): void;
48
+ style: { cursor: string; touchAction: string };
49
+ ownerDocument: {
50
+ hidden: boolean;
51
+ addEventListener(
52
+ type: string,
53
+ listener: (event: DomEvent) => void,
54
+ options?: { passive?: boolean; capture?: boolean },
55
+ ): void;
56
+ removeEventListener(
57
+ type: string,
58
+ listener: (event: DomEvent) => void,
59
+ options?: { capture?: boolean },
60
+ ): void;
61
+ defaultView: {
62
+ addEventListener(type: string, listener: () => void): void;
63
+ removeEventListener(type: string, listener: () => void): void;
64
+ } | null;
65
+ };
66
+ }
67
+
68
+ /** Bind real web envelopes only on web; native touch uses the responder adapter. */
69
+ export function bindBrowserCanvasWeb(
70
+ node: BrowserCanvasNode,
71
+ input: CanvasInput,
72
+ enabled: () => boolean,
73
+ onCursorVisibility: (visible: boolean) => void,
74
+ ) {
75
+ if (Platform.OS !== "web") return () => {};
76
+ const previousTouchAction = node.style.touchAction;
77
+ node.style.touchAction = "none";
78
+ const listeners = new Map<string, (event: DomEvent) => void>();
79
+ const point = (x = 0, y = 0) => {
80
+ const rect = node.getBoundingClientRect();
81
+ if (rect.width <= 0 || rect.height <= 0) return null;
82
+ return {
83
+ x: Math.max(0, Math.min(rect.width, x - rect.left)),
84
+ y: Math.max(0, Math.min(rect.height, y - rect.top)),
85
+ width: rect.width,
86
+ height: rect.height,
87
+ };
88
+ };
89
+ const listen = (type: string, listener: (event: DomEvent) => void) => {
90
+ listeners.set(type, listener);
91
+ node.addEventListener(type, listener, { passive: false });
92
+ };
93
+ const consume = (event: DomEvent) => {
94
+ event.preventDefault();
95
+ event.stopPropagation();
96
+ };
97
+ // Snapshot actual pointer modifiers even when the key was pressed before
98
+ // this canvas acquired focus. Do not fabricate remote keyboard edges.
99
+ const modifiers = (event: DomEvent) =>
100
+ (event.altKey ? 1 : 0) |
101
+ (event.ctrlKey ? 2 : 0) |
102
+ (event.metaKey ? 4 : 0) |
103
+ (event.shiftKey ? 8 : 0);
104
+ let touchActive = false;
105
+ const mouse = (type: "move" | "down" | "up", event: DomEvent) => {
106
+ // Consumed touch events suppress compatibility mouse events in browsers.
107
+ // Chromium also explicitly identifies them; never forward a second tap.
108
+ if (touchActive || event.sourceCapabilities?.firesTouchEvents || !enabled()) {
109
+ return;
110
+ }
111
+ const location = point(event.clientX, event.clientY);
112
+ if (!location) return;
113
+ onCursorVisibility(true);
114
+ const button = event.button === 2 ? "right" : event.button === 1 ? "middle" : "left";
115
+ const count = event.detail === 2 ? 2 : 1;
116
+ if (type === "down") {
117
+ if (input.mouseDown(location, button, count, modifiers(event))) consume(event);
118
+ } else if (type === "up") {
119
+ if (input.mouseUp(location, button, count, modifiers(event))) consume(event);
120
+ } else {
121
+ const held = input.isHoldingMouse();
122
+ input.mouseMove(location, modifiers(event));
123
+ if (held) consume(event);
124
+ }
125
+ };
126
+ // MouseEvent.detail carries physical double-click counts. PointerEvent.detail
127
+ // commonly stays zero. Only mouse events publish input, avoiding duplicate edges.
128
+ listen("mousedown", (event) => mouse("down", event));
129
+ listen("mousemove", (event) => mouse("move", event));
130
+ listen("mouseup", (event) => mouse("up", event));
131
+ listen("mouseleave", () => {
132
+ onCursorVisibility(false);
133
+ input.leave();
134
+ node.style.cursor = "default";
135
+ });
136
+ listen("pointercancel", () => input.cancel());
137
+ const isOutside = (event: DomEvent) => {
138
+ const rect = node.getBoundingClientRect();
139
+ return (
140
+ event.clientX === undefined ||
141
+ event.clientY === undefined ||
142
+ event.clientX < rect.left ||
143
+ event.clientX > rect.left + rect.width ||
144
+ event.clientY < rect.top ||
145
+ event.clientY > rect.top + rect.height
146
+ );
147
+ };
148
+ const outsideMove = (event: DomEvent) => {
149
+ if (input.isHoldingMouse()) {
150
+ mouse("move", event);
151
+ onCursorVisibility(!isOutside(event));
152
+ }
153
+ };
154
+ const outsideUp = (event: DomEvent) => {
155
+ if (input.isHoldingMouse()) {
156
+ mouse("up", event);
157
+ const outside = isOutside(event);
158
+ onCursorVisibility(!outside);
159
+ if (outside) input.leave();
160
+ }
161
+ };
162
+ node.ownerDocument.addEventListener("mousemove", outsideMove);
163
+ node.ownerDocument.addEventListener("mouseup", outsideUp);
164
+ listen("contextmenu", (event) => {
165
+ if (enabled()) consume(event);
166
+ });
167
+ listen("wheel", (event) => {
168
+ if (!enabled()) return;
169
+ const location = point(event.clientX, event.clientY);
170
+ if (!location) return;
171
+ // Wheel deltaMode is local CSS pixels, lines, or pages. Line-mode uses the
172
+ // browser's conventional 16 CSS-pixel step; page mode uses this image's size.
173
+ const multiplierX = event.deltaMode === 2 ? location.width : event.deltaMode === 1 ? 16 : 1;
174
+ const multiplierY = event.deltaMode === 2 ? location.height : event.deltaMode === 1 ? 16 : 1;
175
+ if (
176
+ input.wheel(
177
+ location,
178
+ (event.deltaX ?? 0) * multiplierX,
179
+ (event.deltaY ?? 0) * multiplierY,
180
+ modifiers(event),
181
+ )
182
+ ) {
183
+ consume(event);
184
+ }
185
+ });
186
+ const documentTouchListeners = new Map<string, (event: DomEvent) => void>();
187
+ for (const phase of ["start", "move", "end", "cancel"] as const) {
188
+ // An observe-only overlay has pointerEvents:none. Track only contacts that
189
+ // start over its image (without consuming them), so acquiring control mid-
190
+ // contact cannot turn a later move/second finger into a remote fresh press.
191
+ const trackUncontrolledContact = (event: DomEvent) => {
192
+ const rect = node.getBoundingClientRect();
193
+ const touches = Array.from(event.touches ?? []);
194
+ const overImage = touches.some(
195
+ (touch) =>
196
+ touch.clientX >= rect.left &&
197
+ touch.clientX <= rect.left + rect.width &&
198
+ touch.clientY >= rect.top &&
199
+ touch.clientY <= rect.top + rect.height,
200
+ );
201
+ if (input.isTouchQuarantined() || (!enabled() && overImage)) {
202
+ const points = touches
203
+ .map((touch) => {
204
+ const location = point(touch.clientX, touch.clientY);
205
+ return location ? { ...location, id: touch.identifier } : null;
206
+ })
207
+ .filter((item) => item !== null);
208
+ input.touch(phase, points);
209
+ }
210
+ };
211
+ documentTouchListeners.set(`touch${phase}`, trackUncontrolledContact);
212
+ node.ownerDocument.addEventListener(`touch${phase}`, trackUncontrolledContact, {
213
+ capture: true,
214
+ passive: true,
215
+ });
216
+ listen(`touch${phase}`, (event) => {
217
+ touchActive = phase !== "cancel" && (event.touches?.length ?? 0) > 0;
218
+ const points = Array.from(event.touches ?? []).map((touch) => {
219
+ const location = point(touch.clientX, touch.clientY);
220
+ return location ? { ...location, id: touch.identifier } : null;
221
+ });
222
+ if (points.some((item) => !item)) {
223
+ input.cancel();
224
+ return;
225
+ }
226
+ const accepted = input.touch(
227
+ phase,
228
+ points.filter((item) => item !== null),
229
+ );
230
+ if (accepted && enabled()) {
231
+ consume(event);
232
+ }
233
+ });
234
+ }
235
+ const cancel = () => {
236
+ onCursorVisibility(false);
237
+ touchActive = false;
238
+ input.cancel();
239
+ node.style.cursor = "default";
240
+ };
241
+ const visibility = () => {
242
+ if (node.ownerDocument.hidden) cancel();
243
+ };
244
+ node.ownerDocument.addEventListener("visibilitychange", visibility);
245
+ node.ownerDocument.defaultView?.addEventListener("blur", cancel);
246
+ return () => {
247
+ cancel();
248
+ for (const [type, listener] of listeners) {
249
+ node.removeEventListener(type, listener);
250
+ }
251
+ for (const [type, listener] of documentTouchListeners) {
252
+ node.ownerDocument.removeEventListener(type, listener, { capture: true });
253
+ }
254
+ node.ownerDocument.removeEventListener("visibilitychange", visibility);
255
+ node.ownerDocument.removeEventListener("mousemove", outsideMove);
256
+ node.ownerDocument.removeEventListener("mouseup", outsideUp);
257
+ node.ownerDocument.defaultView?.removeEventListener("blur", cancel);
258
+ node.style.touchAction = previousTouchAction;
259
+ };
260
+ }
261
+
262
+ /** Cursor enums originate from the guarded remote response, never custom URLs. */
263
+ export function setBrowserCanvasCursor(node: unknown, cursor: BrowserCursor | null): void {
264
+ if (Platform.OS === "web" && node && typeof node === "object" && "style" in node) {
265
+ (node as BrowserCanvasNode).style.cursor = cursor ?? "default";
266
+ }
267
+ }
268
+
269
+ /** Public web codec boundary. Native clients never read browser constructors. */
270
+ export function supportsBrowserVideo(): boolean {
271
+ if (Platform.OS !== "web") return false;
272
+ return typeof VideoDecoder !== "undefined" && typeof EncodedVideoChunk !== "undefined";
273
+ }
274
+
275
+ declare const VideoDecoder: {
276
+ new (callbacks: {
277
+ output(frame: import("./browser-video-decoder").DecodedBrowserVideoFrame): void;
278
+ error(error: unknown): void;
279
+ }): import("./browser-video-decoder").BrowserVideoCodec & {
280
+ addEventListener(type: "dequeue", listener: () => void): void;
281
+ };
282
+ };
283
+ declare const EncodedVideoChunk: {
284
+ new (input: { type: "key" | "delta"; timestamp: number; data: Uint8Array }): unknown;
285
+ };
286
+ declare function atob(data: string): string;
287
+
288
+ interface VideoCanvasElement {
289
+ width: number;
290
+ height: number;
291
+ style: { width: string; height: string; display: string };
292
+ getContext(kind: "2d"): {
293
+ drawImage(frame: unknown, x: number, y: number): void;
294
+ } | null;
295
+ remove(): void;
296
+ }
297
+ export interface BrowserVideoCanvasNode {
298
+ getClientRects(): ArrayLike<unknown>;
299
+ appendChild(child: VideoCanvasElement): void;
300
+ ownerDocument: {
301
+ createElement(kind: "canvas"): VideoCanvasElement;
302
+ hidden?: boolean;
303
+ addEventListener(type: string, listener: () => void): void;
304
+ removeEventListener(type: string, listener: () => void): void;
305
+ defaultView: {
306
+ IntersectionObserver: new (
307
+ callback: (entries: { isIntersecting: boolean }[]) => void,
308
+ ) => {
309
+ observe(node: BrowserVideoCanvasNode): void;
310
+ disconnect(): void;
311
+ };
312
+ requestAnimationFrame(callback: () => void): number;
313
+ cancelAnimationFrame(id: number): void;
314
+ addEventListener(type: string, listener: () => void): void;
315
+ removeEventListener(type: string, listener: () => void): void;
316
+ } | null;
317
+ };
318
+ }
319
+
320
+ /** Decode validated packet bytes without a per-character callback or intermediate
321
+ * binary string on modern clients. Older web clients retain an exact byte loop.
322
+ * This web adapter is never evaluated as a native phone video implementation. */
323
+ export function decodeBrowserVideoBytes(data: string): Uint8Array {
324
+ const bytes = Uint8Array as typeof Uint8Array & { fromBase64?(value: string): Uint8Array };
325
+ if (typeof bytes.fromBase64 === "function") return bytes.fromBase64(data);
326
+
327
+ const binary = atob(data);
328
+ const decoded = new Uint8Array(binary.length);
329
+ for (let index = 0; index < binary.length; index += 1) {
330
+ decoded[index] = binary.charCodeAt(index);
331
+ }
332
+ return decoded;
333
+ }
334
+
335
+ /** Create one retained canvas; every decoded VideoFrame is drawn before its authority is admitted. */
336
+ export function createBrowserVideoEnvironment(node: BrowserVideoCanvasNode) {
337
+ if (!supportsBrowserVideo()) return null;
338
+ const canvas = node.ownerDocument.createElement("canvas");
339
+ const context = canvas.getContext("2d");
340
+ const view = node.ownerDocument.defaultView;
341
+ if (!context || !view) return null;
342
+ canvas.style.width = "100%";
343
+ canvas.style.height = "100%";
344
+ canvas.style.display = "block";
345
+ node.appendChild(canvas);
346
+ const environment: import("./browser-video-decoder").BrowserVideoDecodeEnvironment = {
347
+ createDecoder(callbacks) {
348
+ const decoder = new VideoDecoder(callbacks);
349
+ decoder.addEventListener("dequeue", callbacks.dequeue);
350
+ return decoder;
351
+ },
352
+ createChunk: (input) => new EncodedVideoChunk(input),
353
+ decodeBase64(data) {
354
+ return decodeBrowserVideoBytes(data);
355
+ },
356
+ scheduleDraw(draw) {
357
+ const id = view.requestAnimationFrame(draw);
358
+ return () => view.cancelAnimationFrame(id);
359
+ },
360
+ draw(frame) {
361
+ if (canvas.width !== frame.displayWidth) canvas.width = frame.displayWidth;
362
+ if (canvas.height !== frame.displayHeight) canvas.height = frame.displayHeight;
363
+ context.drawImage(frame, 0, 0);
364
+ },
365
+ };
366
+ return { environment, dispose: () => canvas.remove() };
367
+ }
368
+
369
+ /** Observe the actual canvas document, including page-cache suspension. The
370
+ * initial callback is synchronous; cleanup never synthesizes a resume. */
371
+ export function bindBrowserVideoVisibility(
372
+ node: BrowserVideoCanvasNode,
373
+ onActive: (active: boolean) => void,
374
+ ) {
375
+ const document = node.ownerDocument;
376
+ const view = document.defaultView;
377
+ let suspended = false;
378
+ let intersecting = node.getClientRects().length > 0;
379
+ const changed = () => onActive(!document.hidden && !suspended && intersecting);
380
+ const observer = view
381
+ ? new view.IntersectionObserver((entries) => {
382
+ intersecting = entries.some((entry) => entry.isIntersecting);
383
+ changed();
384
+ })
385
+ : null;
386
+ observer?.observe(node);
387
+ const hide = () => {
388
+ suspended = true;
389
+ changed();
390
+ };
391
+ const show = () => {
392
+ suspended = false;
393
+ changed();
394
+ };
395
+ document.addEventListener("visibilitychange", changed);
396
+ view?.addEventListener("pagehide", hide);
397
+ view?.addEventListener("pageshow", show);
398
+ changed();
399
+ return () => {
400
+ observer?.disconnect();
401
+ document.removeEventListener("visibilitychange", changed);
402
+ view?.removeEventListener("pagehide", hide);
403
+ view?.removeEventListener("pageshow", show);
404
+ };
405
+ }
package/index.client.tsx CHANGED
@@ -21,16 +21,6 @@ export default function contribute(client: PluginClientContext) {
21
21
  openPanel("shared-browser");
22
22
  },
23
23
  });
24
- client.addCommandCenterItem({
25
- id: "open-shared-browser-agent",
26
- title: "Open Shared Browser",
27
- icon: "PanelsTopLeft",
28
- keywords: ["browser", "shared", "remote"],
29
- context: "agent",
30
- onSelect({ openPanel }) {
31
- openPanel("shared-browser");
32
- },
33
- });
34
24
 
35
25
  const removeSidebar = contributeSharedBrowserSidebar(client);
36
26
  const cleanup = contributeSharedBrowserClient(client);
package/index.server.ts CHANGED
@@ -8,13 +8,23 @@ import {
8
8
  handleAcquireControl,
9
9
  handleApplyDevicePreset,
10
10
  handleAttachBrowser,
11
+ handleBeginBrowserGesture,
11
12
  handleCaptureBrowser,
13
+ handleCloseBrowser,
14
+ handleCloseBrowserTab,
15
+ handleCreateBrowserTab,
12
16
  handleDetachBrowser,
17
+ handleEndBrowserGesture,
18
+ handleListBrowserTabs,
13
19
  handleListOpenBrowserWorkspaces,
14
20
  handleNavigateBrowser,
21
+ handleReadBrowserVideo,
15
22
  handleReleaseControl,
23
+ handleReopenBrowser,
16
24
  handleResizeBrowser,
17
25
  handleSendBrowserInput,
26
+ handleSetCaptureDensity,
27
+ handleUpdateBrowserGesture,
18
28
  handleWorkspaceArchived,
19
29
  issueAgentTicket,
20
30
  revokeAgentBrowserAccess,
@@ -24,14 +34,25 @@ import {
24
34
  acquireControlRpc,
25
35
  applyDevicePresetRpc,
26
36
  attachBrowserRpc,
37
+ beginBrowserGestureRpc,
27
38
  captureBrowserRpc,
39
+ closeBrowserRpc,
40
+ closeBrowserTabRpc,
41
+ createBrowserTabRpc,
28
42
  detachBrowserRpc,
43
+ endBrowserGestureRpc,
44
+ listBrowserTabsRpc,
29
45
  listOpenBrowserWorkspacesRpc,
30
46
  navigateBrowserRpc,
31
47
  releaseControlRpc,
48
+ reopenBrowserRpc,
32
49
  resizeBrowserRpc,
33
50
  sendBrowserInputRpc,
51
+ setCaptureDensityRpc,
52
+ updateBrowserGestureRpc,
34
53
  } from "./shared/browser";
54
+ import { browserDisplayPreferences } from "./shared/browser-display-preferences";
55
+ import { readBrowserVideoRpc } from "./shared/browser-video";
35
56
 
36
57
  const TICKET_ENV = "PASEO_SHARED_BROWSER_TICKET";
37
58
  const MCP_SERVER_ID = "shared-browser";
@@ -67,12 +88,23 @@ function mcpBundlePath(): string {
67
88
  }
68
89
 
69
90
  export default function contribute(server: PluginServerContext) {
91
+ server.registerSettings(browserDisplayPreferences);
70
92
  server.handle(attachBrowserRpc, handleAttachBrowser);
71
93
  server.handle(detachBrowserRpc, handleDetachBrowser);
94
+ server.handle(closeBrowserRpc, handleCloseBrowser);
95
+ server.handle(closeBrowserTabRpc, handleCloseBrowserTab);
96
+ server.handle(createBrowserTabRpc, handleCreateBrowserTab);
97
+ server.handle(listBrowserTabsRpc, handleListBrowserTabs);
98
+ server.handle(reopenBrowserRpc, handleReopenBrowser);
72
99
  server.handle(captureBrowserRpc, handleCaptureBrowser);
100
+ server.handle(setCaptureDensityRpc, handleSetCaptureDensity);
101
+ server.handle(readBrowserVideoRpc, handleReadBrowserVideo);
73
102
  server.handle(listOpenBrowserWorkspacesRpc, handleListOpenBrowserWorkspaces);
74
103
  server.handle(acquireControlRpc, handleAcquireControl);
75
104
  server.handle(releaseControlRpc, handleReleaseControl);
105
+ server.handle(beginBrowserGestureRpc, handleBeginBrowserGesture);
106
+ server.handle(updateBrowserGestureRpc, handleUpdateBrowserGesture);
107
+ server.handle(endBrowserGestureRpc, handleEndBrowserGesture);
76
108
  server.handle(navigateBrowserRpc, handleNavigateBrowser);
77
109
  server.handle(resizeBrowserRpc, handleResizeBrowser);
78
110
  server.handle(applyDevicePresetRpc, handleApplyDevicePreset);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@omercnet/paseo-shared-browser",
3
- "version": "1.2.2-next.232.1",
3
+ "version": "1.2.2-next.237.1",
4
4
  "description": "One real Chromium session per Paseo workspace, shared live across every connected client.",
5
5
  "keywords": [
6
6
  "browser",
@@ -24,6 +24,8 @@
24
24
  "files": [
25
25
  "LICENSE",
26
26
  "README.md",
27
+ "STREAMING_DESIGN.md",
28
+ "REMOTE_BROWSER_RESEARCH.md",
27
29
  "index.client.tsx",
28
30
  "index.server.ts",
29
31
  "client",
@@ -45,19 +47,21 @@
45
47
  "check:write": "biome check --write .",
46
48
  "typecheck": "tsc --noEmit",
47
49
  "prepare:runtime": "node scripts/prepare-runtime.mjs",
48
- "test": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts",
49
- "test:unit": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts",
50
- "test:smoke": "vitest run tests/browser.smoke.test.ts --testTimeout=30000"
50
+ "test": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/agent-input-safety.test.ts tests/tabs.test.ts tests/tabs-runtime-loss.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts client server shared",
51
+ "test:unit": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/agent-input-safety.test.ts tests/tabs.test.ts tests/tabs-runtime-loss.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts client server shared",
52
+ "test:smoke": "vitest run tests/browser.smoke.test.ts tests/browser.drag-cancel.smoke.test.ts tests/xvfb.smoke.test.ts tests/browser.failed-open.smoke.test.ts tests/browser.background-tab.smoke.test.ts --testTimeout=30000",
53
+ "test:video": "vitest run tests/video.smoke.test.ts --testTimeout=30000"
51
54
  },
52
55
  "dependencies": {
53
56
  "@getpaseo/plugin": "0.11.1",
54
57
  "@modelcontextprotocol/sdk": "1.30.0",
55
- "agent-browser": "0.37.1",
58
+ "agent-browser": "0.38.2",
56
59
  "esbuild": "0.28.2",
57
60
  "zod": "4.4.3"
58
61
  },
59
62
  "devDependencies": {
60
63
  "@biomejs/biome": "^2.5.10",
64
+ "@getpaseo/server": "0.11.1",
61
65
  "@tanstack/react-query": "^5.102.3",
62
66
  "@types/node": "^24.10.1",
63
67
  "@types/react": "^19.2.18",
package/paseo-plugin.json CHANGED
@@ -1,6 +1,8 @@
1
1
  {
2
2
  "id": "shared-browser",
3
3
  "description": "One real Chromium session per Paseo workspace, shared live across every connected client.",
4
- "requirements": { "paseo": "^0.9.0 || ^0.10.0 || ^0.11.0" },
4
+ "requirements": {
5
+ "paseo": "^0.9.0 || ^0.10.0 || ^0.11.0 || 0.11.0-beta.3 || 0.11.0-beta.4"
6
+ },
5
7
  "build": [["node", "scripts/prepare-dependencies.mjs"], ["npm", "run", "prepare:runtime"]]
6
8
  }
@@ -18,7 +18,7 @@ function resolveDependencyRoot(packageName) {
18
18
  }
19
19
 
20
20
  const execFileAsync = promisify(execFile);
21
- const expectedVersion = "0.37.1";
21
+ const expectedVersion = "0.38.2";
22
22
  const hostPlatform = platform();
23
23
  const hostArch = arch();
24
24
  const projectRoot = dirname(dirname(fileURLToPath(import.meta.url)));