@omercnet/paseo-shared-browser 1.2.2 → 2.0.0-next.244.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/OVERVIEW.md +2 -2
  2. package/README.md +316 -27
  3. package/REMOTE_BROWSER_RESEARCH.md +395 -0
  4. package/STREAMING_DESIGN.md +196 -0
  5. package/client/browser-auto-capture-density.test.ts +102 -0
  6. package/client/browser-auto-capture-density.ts +30 -0
  7. package/client/browser-canvas-input.test.ts +198 -0
  8. package/client/browser-canvas-input.ts +202 -0
  9. package/client/browser-canvas-keyboard-web.test.ts +113 -0
  10. package/client/browser-canvas-keyboard-web.ts +230 -0
  11. package/client/browser-canvas-keyboard.test.ts +123 -0
  12. package/client/browser-canvas-keyboard.ts +126 -0
  13. package/client/browser-canvas-layout.test.ts +109 -0
  14. package/client/browser-canvas-layout.ts +51 -0
  15. package/client/browser-canvas-viewport-web.test.ts +77 -0
  16. package/client/browser-canvas-viewport-web.ts +63 -0
  17. package/client/browser-canvas-viewport.tsx +152 -0
  18. package/client/browser-capture-cadence.test.ts +9 -0
  19. package/client/browser-capture-cadence.ts +6 -0
  20. package/client/browser-capture-density-controls.tsx +75 -0
  21. package/client/browser-chrome.tsx +234 -0
  22. package/client/browser-client-presence.ts +184 -0
  23. package/client/browser-compose-controls.test.tsx +275 -0
  24. package/client/browser-compose-controls.tsx +174 -0
  25. package/client/browser-control-button.tsx +109 -0
  26. package/client/browser-control-dialog.tsx +71 -0
  27. package/client/browser-control-request.test.ts +27 -0
  28. package/client/browser-control-request.ts +57 -0
  29. package/client/browser-control-tooltip-web.ts +10 -0
  30. package/client/browser-dialog-body.tsx +22 -0
  31. package/client/browser-emulation-mode.test.ts +75 -0
  32. package/client/browser-emulation-mode.ts +98 -0
  33. package/client/browser-fill-viewport.test.ts +45 -0
  34. package/client/browser-fill-viewport.ts +28 -0
  35. package/client/browser-frame-buffer.test.ts +157 -0
  36. package/client/browser-frame-buffer.ts +148 -0
  37. package/client/browser-frame-image.tsx +62 -0
  38. package/client/browser-image-load.test.ts +23 -0
  39. package/client/browser-image-load.ts +26 -0
  40. package/client/browser-input-queue.test.ts +1254 -0
  41. package/client/browser-input-queue.ts +519 -0
  42. package/client/browser-menu-placement.test.ts +90 -0
  43. package/client/browser-menu-placement.ts +78 -0
  44. package/client/browser-panel-styles.ts +384 -0
  45. package/client/browser-quality-controls.test.ts +26 -0
  46. package/client/browser-quality-controls.tsx +139 -0
  47. package/client/browser-resolution-picker.tsx +189 -0
  48. package/client/browser-tabs.tsx +117 -0
  49. package/client/browser-toolbar-menu-web.test.ts +185 -0
  50. package/client/browser-toolbar-menu-web.ts +159 -0
  51. package/client/browser-toolbar-menu.tsx +255 -0
  52. package/client/browser-toolbar-submenu.tsx +93 -0
  53. package/client/browser-video-bytes.test.ts +44 -0
  54. package/client/browser-video-currency.ts +45 -0
  55. package/client/browser-video-decoder.test.ts +473 -0
  56. package/client/browser-video-decoder.ts +311 -0
  57. package/client/browser-video-surface.tsx +21 -0
  58. package/client/browser-video-visibility.test.ts +45 -0
  59. package/client/browser-viewer-recovery.test.ts +111 -0
  60. package/client/browser-viewer-recovery.ts +78 -0
  61. package/client/browser.tsx +1650 -1579
  62. package/client/favorite-star.tsx +23 -0
  63. package/client/frame-lifecycle.ts +62 -59
  64. package/client/sidebar.tsx +1 -2
  65. package/client/use-browser-auto-capture-density.test.ts +104 -0
  66. package/client/use-browser-auto-capture-density.ts +43 -0
  67. package/client/use-browser-canvas-input.ts +375 -0
  68. package/client/use-browser-capture-density.ts +87 -0
  69. package/client/use-browser-emulation-mode.ts +56 -0
  70. package/client/use-browser-fill-viewport.test.ts +108 -0
  71. package/client/use-browser-fill-viewport.ts +62 -0
  72. package/client/use-browser-frame-buffer.ts +66 -0
  73. package/client/use-browser-image-capture.ts +139 -0
  74. package/client/use-browser-scoped-mutation.test.ts +118 -0
  75. package/client/use-browser-scoped-mutation.ts +59 -0
  76. package/client/use-browser-video.test.ts +1060 -0
  77. package/client/use-browser-video.ts +481 -0
  78. package/client/use-browser-viewer-recovery.ts +26 -0
  79. package/client/use-resolution-favorites.ts +90 -0
  80. package/client/web.test.ts +125 -0
  81. package/client/web.ts +405 -0
  82. package/index.client.tsx +0 -10
  83. package/index.server.ts +32 -4
  84. package/npm-shrinkwrap.json +9 -9
  85. package/package.json +10 -5
  86. package/paseo-plugin.json +1 -1
  87. package/scripts/prepare-runtime.mjs +1 -1
  88. package/server/agent-browser-runtime.ts +1601 -314
  89. package/server/browser-cursor.test.ts +64 -0
  90. package/server/browser-cursor.ts +69 -0
  91. package/server/browser-frame-context.ts +22 -0
  92. package/server/browser-frame-epoch.test.ts +95 -0
  93. package/server/browser-frame-rejection.ts +17 -0
  94. package/server/browser-gesture.ts +120 -0
  95. package/server/browser-operation-diagnostics.test.ts +79 -0
  96. package/server/browser-operation-diagnostics.ts +103 -0
  97. package/server/browser-policy.ts +1316 -151
  98. package/server/browser-video-input-authority.test.ts +211 -0
  99. package/server/browser-video-policy.test.ts +564 -0
  100. package/server/browser-video-socket.test.ts +368 -0
  101. package/server/browser.ts +151 -16
  102. package/server/capture-transport-policy.test.ts +105 -0
  103. package/server/capture-transport-policy.ts +154 -0
  104. package/server/capture-transport-runtime.test.ts +213 -0
  105. package/server/daemon-launch-chain.test.ts +70 -0
  106. package/server/daemon-shutdown.test.ts +289 -0
  107. package/server/discrete-input-freshness.test.ts +364 -0
  108. package/server/discrete-input-native-lifetime.test.ts +150 -0
  109. package/server/display-loss-recovery.test.ts +294 -0
  110. package/server/emulation-restoration.test.ts +262 -0
  111. package/server/input-cleanup.ts +23 -0
  112. package/server/input-generation.test.ts +31 -0
  113. package/server/input-generation.ts +36 -0
  114. package/server/jpeg-capture-demand.test.ts +76 -0
  115. package/server/jpeg-capture-demand.ts +98 -0
  116. package/server/jpeg-frame.ts +29 -0
  117. package/server/keyboard-input.test.ts +61 -0
  118. package/server/keyboard-input.ts +104 -0
  119. package/server/live-input-policy.test.ts +958 -0
  120. package/server/live-input-runtime.test.ts +732 -0
  121. package/server/live-input-supervisor.test.ts +147 -0
  122. package/server/mcp-entry.ts +44 -2
  123. package/server/native-video-backpressure.test.ts +228 -0
  124. package/server/native-video-buffer.test.ts +141 -0
  125. package/server/native-video-buffer.ts +114 -0
  126. package/server/native-video-capture.test.ts +599 -0
  127. package/server/native-video-capture.ts +497 -0
  128. package/server/native-video-extension-source.test.ts +201 -0
  129. package/server/native-video-extension-source.ts +364 -0
  130. package/server/native-video-extension.test.ts +41 -0
  131. package/server/native-video-extension.ts +83 -0
  132. package/server/native-video-packet.ts +9 -0
  133. package/server/native-video-settings.test.ts +245 -0
  134. package/server/navigation-budget.ts +8 -0
  135. package/server/navigation-commit.ts +125 -0
  136. package/server/navigation-policy.test.ts +153 -0
  137. package/server/process-identity.ts +58 -0
  138. package/server/reattach-held-input.test.ts +143 -0
  139. package/server/runtime-frame-reuse.test.ts +295 -0
  140. package/server/runtime-jpeg-demand.test.ts +200 -0
  141. package/server/runtime-jpeg-shutdown.test.ts +210 -0
  142. package/server/runtime-jpeg-source-order.test.ts +201 -0
  143. package/server/runtime-launch-environment.test.ts +111 -0
  144. package/server/runtime-navigation.test.ts +327 -0
  145. package/server/runtime-owner-display.test.ts +152 -0
  146. package/server/runtime-owner.ts +340 -31
  147. package/server/runtime-page-state.test.ts +249 -0
  148. package/server/runtime-protocol.ts +31 -1
  149. package/server/runtime-target-retention.test.ts +181 -0
  150. package/server/runtime-video-opt-in.test.ts +82 -0
  151. package/server/runtime-video.test.ts +444 -0
  152. package/server/screencast-source-time.ts +46 -0
  153. package/server/stale-wheel-admission.test.ts +136 -0
  154. package/server/supervisor-client.ts +28 -4
  155. package/server/supervisor-heartbeat.test.ts +80 -0
  156. package/server/supervisor.ts +553 -55
  157. package/server/tab-close-confirmation.test.ts +97 -0
  158. package/server/video-source-recovery.test.ts +20 -0
  159. package/server/video-source-recovery.ts +64 -0
  160. package/server/viewer-capture-lifetime.ts +51 -0
  161. package/server/virtual-display.test.ts +184 -0
  162. package/server/virtual-display.ts +236 -0
  163. package/shared/browser-display-preferences.ts +46 -0
  164. package/shared/browser-video.ts +83 -0
  165. package/shared/browser.ts +286 -65
  166. package/shared/capture-density.ts +16 -0
  167. package/shared/capture-settings.ts +17 -0
  168. package/shared/device-presets.ts +90 -0
  169. package/shared/resolution-menu.test.ts +120 -0
  170. package/shared/resolution-menu.ts +72 -0
  171. package/shared/video-settings.test.ts +38 -0
  172. package/shared/video-settings.ts +34 -0
  173. package/shared/viewport-limits.ts +5 -0
package/OVERVIEW.md CHANGED
@@ -1,4 +1,4 @@
1
- Shared Browser runs one real Chromium browser per workspace on the daemon host and shows that same live page to every connected Paseo client, so viewers and agents act on one page with one set of cookies and logins. Many viewers can watch, and one human at a time holds control. Open it from the Command Center, the **Shared Browser** composer pill, or, on Paseo 0.11 and newer, the sidebar footer.
1
+ Shared Browser runs one real Chromium browser per workspace on the daemon host and shows that same live page to every connected Paseo client, so viewers and agents act on one page with one set of cookies and logins. Many viewers can watch, and one human at a time holds control. Open it from the Command Center, the **Shared Browser** composer pill, or the sidebar footer.
2
2
 
3
3
  ## How it works
4
4
 
@@ -13,7 +13,7 @@ New agents whose provider accepts external MCP servers receive a local MCP serve
13
13
 
14
14
  ## Setup
15
15
 
16
- - Paseo 0.9, 0.10 or 0.11, and Node.js 24 or newer on the daemon host.
16
+ - Paseo ^0.11.0, and Node.js 24 or newer on the daemon host.
17
17
  - Chromium is downloaded when the plugin is installed and stored in the Paseo data directory. On Linux ARM64 nothing is downloaded, and you install a non-Snap Chromium at `/usr/bin/chromium` first. `PASEO_SHARED_BROWSER_CHROMIUM_EXECUTABLE` points the plugin at a different Chromium.
18
18
 
19
19
  ## Limits
package/README.md CHANGED
@@ -1,14 +1,14 @@
1
1
  # Shared Browser
2
2
 
3
- A Paseo plugin that runs one real Chromium browser per workspace on the daemon host and shares that
4
- exact live session with every connected Paseo client. Version 1.0 replaces the previous browser
3
+ A Paseo plugin that runs one real Chromium browser per workspace on the daemon host and shares its
4
+ pages with connected Paseo clients and workspace agents. Version 1.0 replaces the previous browser
5
5
  runtime in place with a plugin-owned, pinned `agent-browser` runtime; existing installations keep
6
6
  the `shared-browser` plugin ID and upgrade without installing a second plugin.
7
7
 
8
- This is not URL synchronization and not a second browser with copied cookies. Every viewer and
9
- eligible workspace agent acts on the same running page, DOM, navigation state, and login state.
10
- Many viewers can watch; human control remains server-authoritative and takes priority over agent
11
- input.
8
+ Each browser tab is a real page in the same Chromium profile, so tabs share site logins and cookies.
9
+ Each client and agent selects a tab independently. Viewers of the same tab share its live page,
10
+ navigation and control state. Human control remains server-authoritative and takes priority over
11
+ agent input on that tab.
12
12
 
13
13
  ## Demo
14
14
 
@@ -33,12 +33,23 @@ organization names.
33
33
  - The plugin server runs beside the Paseo daemon and starts one `agent-browser` session and Chromium
34
34
  process per open browser workspace. Browser execution, profiles, IPC, and network access are on the
35
35
  daemon host, not on the viewing phone, browser, or desktop app.
36
- - The plugin owns `agent-browser` version `0.37.1`, its IPC directory, and the Chromium executable.
36
+ - A workspace can have up to eight tabs. Each tab has its own page, capture stream, viewport and
37
+ controller. Selecting a tab affects only the selecting viewer or agent. Closing a tab removes it
38
+ for every participant viewing it; the last tab stays open until the browser is closed.
39
+ - Viewers can select a browser action while observing. A dialog offers control, names the current
40
+ controller when a takeover is needed, and continues the selected action after control is granted.
41
+ Closing a tab or browser still requires its own confirmation. Actions that need a fresh frame wait
42
+ for one before sending input.
43
+ - The plugin owns `agent-browser` version `0.38.2`, its IPC directory, and the Chromium executable.
37
44
  It strips inherited `AGENT_BROWSER_*` variables and sets `AGENT_BROWSER_SOCKET_DIR`,
38
45
  `AGENT_BROWSER_IDLE_TIMEOUT_MS=0`, `AGENT_BROWSER_STREAM_PORT=0`, and
39
46
  `AGENT_BROWSER_NO_AUTO_DIALOG=1` itself.
40
- - Frames stream from Chromium through CDP `Page.startScreencast`, with a bounded screenshot fallback.
41
- Remote input supports tap, double-tap, right-click, drag, swipe scrolling, text, and special keys.
47
+ - By default every client receives JPEG frames from CDP `Page.startScreencast` or bounded
48
+ screenshots, and no capture extension is loaded into Chromium. When the daemon is started with
49
+ `PASEO_SHARED_BROWSER_VIDEO=1`, desktop/web clients with usable WebCodecs receive encoded
50
+ workspace-tab video instead; other clients keep JPEG.
51
+ Remote input supports mouse hover, wheel scrolling, continuous dragging, native touch pan/pinch,
52
+ tap, double-tap, right-click, text, and special keys.
42
53
  - Device presets for Desktop Chrome, iPhone 15 Pro, Pixel 7, and iPad Pro 11 change Chromium's
43
54
  viewport, device pixel ratio, touch behavior, platform, and user agent. A phone can view and
44
55
  control the shared browser, but the rendered browser remains Chromium. An iPhone preset is mobile
@@ -53,6 +64,12 @@ bridge can reclaim the existing workspace runtime. If no bridge reconnects, the
53
64
  all orphaned runtimes after a 120-second grace period. A missed heartbeat fences the old bridge after
54
65
  30 seconds; the bridge sends heartbeats every 10 seconds.
55
66
 
67
+ The browser actions menu has **Close shared browser**. After confirmation,
68
+ it stops that workspace's Chromium process and ends every viewer and agent connection to it, even
69
+ when their panels are active. Background reconnects cannot reopen a deliberately closed browser.
70
+ A person can select the prominent **Open shared browser** button in the closed panel to start a new runtime with the
71
+ retained profile. Closing the browser does not remove saved cookies or site data.
72
+
56
73
  Each workspace gets a private profile under
57
74
  `$PASEO_HOME/plugin-data/shared-browser/profiles/<workspace-id-sha256>`. Cookies and site login state
58
75
  therefore survive viewer disconnects, plugin reloads, and Chromium process restarts while that
@@ -93,7 +110,7 @@ Before upgrading the plugin on Windows, close active Shared Browser sessions and
93
110
  daemon; Windows does not allow the installer to replace runtime executables that are still running.
94
111
 
95
112
  Open a workspace, search the Command Center for **Open Shared Browser**, or tap the **Shared
96
- Browser** composer pill while a workspace session is open. On Paseo 0.11 or newer, a **Shared
113
+ Browser** composer pill while a workspace session is open. A **Shared
97
114
  Browser (n)** row in the sidebar footer appears while browser sessions are open; it lists them and
98
115
  jumps to the chosen workspace's browser panel.
99
116
 
@@ -102,21 +119,22 @@ jumps to the chosen workspace's browser panel.
102
119
  The plugin automatically injects its stdio MCP adapter only when a new, non-internal agent is
103
120
  created with a provider that accepts external MCP servers. Agents that already exist, resumed
104
121
  sessions, imported sessions, and Paseo's internal agents are not modified. Paseo's built-in OMP
105
- provider accepts session MCP servers from Paseo 0.11, so new OMP agents receive the adapter there.
106
- On Paseo 0.9 and 0.10 the built-in OMP adapter rejects external MCP servers, so OMP agents are left
107
- unchanged. Pi agents continue to receive the adapter, but they require Pi's optional MCP support to
122
+ provider accepts session MCP servers, so new OMP agents receive the adapter.
123
+ Pi agents also receive the adapter, but require Pi's optional MCP support to
108
124
  launch it.
109
125
 
110
- The injected MCP server exposes exactly these tools: `shared_browser_status`,
111
- `shared_browser_capture`, `shared_browser_acquire_control`, `shared_browser_release_control`,
112
- `shared_browser_navigate`, `shared_browser_input`, and `shared_browser_viewport`. It does not
126
+ The injected MCP server exposes `shared_browser_status`, `shared_browser_capture`,
127
+ `shared_browser_acquire_control`, `shared_browser_release_control`,
128
+ `shared_browser_navigate`, `shared_browser_input`, `shared_browser_viewport`,
129
+ `shared_browser_tabs`, `shared_browser_tab_open`, `shared_browser_tab_select`, and
130
+ `shared_browser_tab_close`. It does not
113
131
  expose arbitrary CDP commands, JavaScript or page evaluation, browser profile access, or filesystem
114
132
  access.
115
133
 
116
134
  Each adapter launch receives an opaque credential bound to its workspace. It cannot use that
117
135
  credential to operate another workspace's browser. Agent input follows human-priority control: an
118
- agent cannot force a takeover while a human viewer holds control. The human must release control or
119
- the lease must expire before agent input can proceed.
136
+ agent cannot force a takeover while a human viewer holds control of the same tab. The human must
137
+ release control or the lease must expire before agent input can proceed on that tab.
120
138
 
121
139
  The provider launches the stdio adapter on the Paseo daemon host, beside the daemon-owned browser
122
140
  runtime. Web, desktop, and mobile clients never own or host the adapter or Chromium, and a client
@@ -137,11 +155,163 @@ User-supplied `AGENT_BROWSER_*` variables are deliberately ignored.
137
155
 
138
156
  ## Controls
139
157
 
140
- - Toolbar: back, forward, reload, address bar, and device emulation.
158
+ - Toolbar: back, forward, reload, address bar, a combined monitor menu, mobile emulation toggle
159
+ and a vertical-dots browser actions menu. The controller can close the shared browser from that
160
+ menu after confirmation. Icon controls expose their action name
161
+ as a hover tooltip on desktop/web and retain native accessibility labels.
162
+ - Custom viewports support 320 to 3840 pixels wide and 480 to 3840 pixels high. Invalid sizes
163
+ show their error inside the device dialog; a successful Apply closes it.
164
+ - Desktop presets also include 1280 × 800 (16:10) and 1280 × 1280 (1:1).
165
+ - Pixel 7 (high resolution) keeps the same 412 × 839 phone layout and input coordinates,
166
+ with a sharper 824 × 1678 capture.
167
+ - Desktop presets include 1920 × 1080 and 2560 × 1440 (16:9), 1920 × 1200 and
168
+ 2560 × 1600 (16:10), and 1920 × 1920 and 2560 × 2560 (1:1).
169
+ - Desktop choices use height names such as 720p, 1080p, 1200p and 1440p, with
170
+ dimensions shown alongside. Square choices name their width explicitly.
171
+ The 1440-wide choices are 1440 × 810, 1440 × 900 and 1440 × 1440.
172
+ - The Resolution and quality menu groups desktop choices by 16:9, 16:10 and 1:1,
173
+ then mobile. Each group sorts by width. Filled stars mark favorites; the monitor
174
+ menu provides favorites in that same grouped order and access to the full
175
+ resolution and quality list.
176
+ - **Fill viewport** resizes the shared browser to the visible panel and follows panel size
177
+ changes while this viewer controls it. It preserves mobile behavior. Auto capture density
178
+ adjusts detail for the screen as the panel grows or shrinks; manual density is retained.
179
+ Following pauses when the panel is hidden, control is released, or its dimensions exceed
180
+ the supported resolution at the current density. Choosing a preset or custom resolution
181
+ stops following; a failed resize also stops it and displays an error.
182
+ - Monitor and browser actions open compact anchored menus rather than dialogs. Menus stay
183
+ within the pane, scroll long lists, and dismiss with an outside press, Escape or native Back.
184
+ - The monitor menu offers **Fit to panel** (default) and **Actual size (100%)**. Actual size
185
+ uses browser layout pixels, not the larger JPEG dimensions of sharper phone captures.
186
+ This choice affects only the current viewer, without resizing the shared browser.
187
+ Desktop local scrollbars reveal overflow while wheel events on a controlled frame go to
188
+ the remote page. Phone viewers can pan while observing; controlling sends swipes to the
189
+ page instead. Fitting images stay centered without empty scroll ranges; a scrollbar on
190
+ one axis does not force one on the other. Returning to Fit resets local offsets without
191
+ reloading the page or image.
192
+ - With `PASEO_SHARED_BROWSER_VIDEO=1` set in the daemon environment, desktop and web clients
193
+ with WebCodecs play genuine encoded tab video (H264 when supported, otherwise VP8). The
194
+ bundled capture helper, which holds `tabCapture` and `debugger` permissions, is then
195
+ loaded into every workspace Chromium profile and captures the exact workspace tab, not a
196
+ screenshot loop or title-selected window. Without the variable the helper is never
197
+ materialized or loaded, and clients show a one-time disabled reply then use JPEG.
198
+ Android and other native apps use JPEG: the plugin SDK this package targets publishes no
199
+ encoded-video surface, so native video is deferred until the host ships one.
200
+ - The monitor menu's resolution and quality list separates JPEG quality (40%, 65%, 85%,
201
+ or 100%), video bitrate (2, 5, 12, or 24 Mbps), and video frame rate (15, 30, or 60 FPS).
202
+ Defaults are JPEG65 (Medium) and, when video is enabled, 12 Mbps at 30 FPS. Old saved quality choices keep their
203
+ corresponding video bitrate when upgraded. These are encoder targets, not measured
204
+ bandwidth or guaranteed frame rates. Large JPEGs may reduce quality to stay within
205
+ the 800 KB frame bound. Preferences persist on the connected Paseo host.
206
+ Capture density defaults to Auto for each viewer panel. While controlling, it selects
207
+ 1x or 2x from the displayed page size and screen pixel density, accounting for Fit's
208
+ letterboxing and Actual size. Manual 1x/2x choices override Auto until Auto is selected
209
+ again, the workspace changes, or the panel is reopened. Density raises real source
210
+ pixel detail without changing page size, input mode, or user agent. The 2x option
211
+ requires both page dimensions to be 1920 or less; larger pages use 1x. Fill viewport
212
+ adjusts automatic density and layout size together. Hidden panels, observers, active
213
+ gestures, and pending operations never trigger automatic density writes. A failed
214
+ adjustment pauses automation until Auto is selected again. Higher density can increase
215
+ processing and bandwidth substantially. Quality changes do not navigate or reload the page.
216
+ - Video packets travel through the existing authenticated Paseo RPC connection, including its
217
+ relay. No extra video port or unauthenticated media server is opened. This is encoded video
218
+ delivered in bounded request/reply batches, not WebRTC or a custom push subscription.
219
+ - Animated native capture supports a source ceiling of 60 frames per second. Ordinary
220
+ frames follow each encoder's requested rate; a recovery keyframe can use the next
221
+ eligible frame without waiting for that interval. Quiet tabs produce genuine frames
222
+ around once per second; repeated packets never acquire a newer source capture timestamp.
223
+ At most three bitrate/frame-rate cohorts share one source. Changing quality never evicts
224
+ another active cohort. If all three are occupied, another profile uses JPEG fallback
225
+ and retries video after 2.5 seconds while visible. Hiding or unmounting cancels
226
+ that viewing-only retry; the soft error clears after a fresh video frame paints.
227
+ Idle encoders release their buffers and codecs after two seconds without an active read;
228
+ when no video consumer remains, the native track and helper also close. A returning video
229
+ viewer can reacquire the source safely. Higher rates/bitrates and multiple encoders cost
230
+ more host CPU and bandwidth; pixel density and the host's native codecs also affect cost.
231
+ - Hidden documents, page-cache suspension, inactive native AppState and panel unmount stop
232
+ this viewer's video reads and release decoding resources. Resuming requests a fresh
233
+ keyframe. One already-issued bounded RPC may finish, but cannot restore old input authority.
234
+ Video remains video when the page is quiet; no idle still-image overlay or static-page
235
+ polling is used. Keyframes restore decoding after a gap but do not make video lossless.
236
+ - Initial human control requires decoded and painted geometry. Once admitted,
237
+ the controller's ordered input channel continues while video catches up.
238
+ A click may reach a page ahead of the displayed image, as in remote desktop.
239
+ Agent commands retain strict fresh-frame targeting. Document, viewport,
240
+ runtime, bridge and controller changes invalidate the human admission. Dropped packets recover at a keyframe rather
241
+ than decoding a broken delta chain. Video reads wait outside the input queues.
242
+ - If no fresh video is painted for 2.5 seconds, the viewer revokes video input authority and
243
+ requests JPEG fallback and a fresh keyframe. The retained video remains visible
244
+ until a current image has decoded, so a delayed fallback cannot flash an older view. Decoder/capture failures also retain
245
+ fallback. The Video, CDP and fallback labels describe the displayed transport.
246
+ - JPEG captures refresh after input. A stalled screencast falls back to a fresh screenshot.
247
+ Healthy painted video suppresses redundant JPEG polling and focus captures. Media
248
+ reads use source generations without repeatedly querying page metadata inside
249
+ the control queue; ordinary status checks still reconcile navigation state.
250
+ Shared JPEG screencasting starts on image demand and stops after three seconds
251
+ without demand. Phone/image viewers and agent captures restart it lazily;
252
+ video reads alone do not keep it running. An image fallback after retirement
253
+ may wait for stream startup or use the existing screenshot fallback.
254
+ Repeated reads of the exact cached JPEG send metadata only; image source age still expires
255
+ within one second. The last decoded image remains visible while its replacement loads.
256
+ - Reconnecting or switching browser tabs restores the selected device viewport, capture density,
257
+ touch and user agent before accepting new frames.
258
+ - Actual video and JPEG dimensions must match the selected capture resolution before a frame is accepted.
259
+ The fallback captures the complete visible viewport, preserves scroll position, and accounts
260
+ for device pixel ratio and capture scale so inputs still use the original layout coordinates.
261
+ - The last decoded frame stays visible while its replacement loads, with native image fading
262
+ disabled. Initial human admission and agent commands use the painted frame;
263
+ admitted human input retains its qualified geometry while pixels catch up.
264
+ Obsolete image callbacks cannot replace the displayed frame.
141
265
  - Status row: session state, viewer count, controller, and lease expiry.
266
+ - Returning after viewer expiry reattaches viewing once automatically while preserving the
267
+ page. Expired control is cleared; take control again to send input. Expected expiry is not
268
+ shown as an action failure; a failed reattachment still offers manual retry.
269
+ - Clicking a link or submitting a form may navigate before its input reply arrives. A completed
270
+ input returns the new viewing state without a false failure or repeating the action.
271
+ Replaced controls, targets and uncertain sends remain rejected.
272
+ - A document change during metadata reads discards both partial observations and
273
+ permits one fresh read on the same attachment within the original timeout.
274
+ Repeated changes, failed reads and replacement attachments remain errors;
275
+ browser input is never replayed to obtain metadata.
276
+ - Agent stale-frame errors identify initial admission or a context change during admitted
277
+ input. Known receipt age is bounded to 60 seconds and uses its original clock; missing
278
+ receipts report missing-or-revoked without inventing an age. These diagnostics contain
279
+ no page content or authority identifiers and do not authorize replay. Initial frame
280
+ admission still expires after five seconds.
142
281
  - Human control: **Take control**, **Release**, and **Take over** for explicit handoff. Agent MCP
143
282
  calls have no forced-takeover operation.
144
- - Mobile: swipe scrolls by default; pointer and keyboard options open as bottom sheets.
283
+ - With control, mouse movement forwards real hover effects and standard browser cursor changes.
284
+ Wheel scrolling stays inside the browser canvas; held drags update before release. Leaving the
285
+ canvas clears remote hover, while dragging beyond its edges still releases the held button.
286
+ - Hidden Linux browsers are headless by default. Opting in to a private virtual display
287
+ (see Optional Linux desktop hover) supplies native desktop pointer/hover capabilities,
288
+ including after switching back from phone mode.
289
+ - Native touch forwards active fingers continuously, including pan and pinch on desktop and mobile
290
+ presets. Physical mouse buttons and double clicks retain their natural actions; touch never
291
+ changes into a synthetic mouse gesture.
292
+ Losing control, changing page or resizing cancels held input; old contacts must lift before a
293
+ fresh touch can target the replacement page. No uncertain input is replayed after a failure.
294
+ - Desktop viewports show native scrollbars. Dragging a scrollbar thumb scrolls the page
295
+ continuously before release. Wheel scrolling and touch panning remain available;
296
+ mobile emulation retains its normal mobile scrollbar behavior.
297
+ - Click the controlled canvas to type directly on desktop and web. Shortcuts, repeat,
298
+ modifier clicks and committed composition text are forwarded; other Paseo inputs stay local.
299
+ - The address-bar Browser menu contains a Send keys submenu and reconnect actions. On desktop,
300
+ keys open beside the menu; on compact screens, Back returns to the parent menu. Extra keys
301
+ are no longer shown below the canvas or in a separate dialog.
302
+ - On native phones and compact web layouts, Compose text in that menu opens an explicit local
303
+ draft for keyboard or IME text, inserted once into the focused page field when you choose
304
+ Done. Native phones have no live software-keyboard relay; compact web also keeps canvas
305
+ typing. Extra keys remain available in the Send keys submenu.
306
+ - Local plain-text paste is forwarded; remote copy/cut are not synchronized to the local clipboard.
307
+ - The address-bar mobile toggle changes shared emulation while preserving the current display
308
+ dimensions and capture density. Choose a resolution explicitly to change the display.
309
+ Your device default applies the first time you take control, never while observing, and
310
+ also preserves the current display. Narrow desktop layouts do not count as phones.
311
+ - The mobile address field keeps a full text line with compact padding. Read-only addresses
312
+ stay legible while observing; taking control is still required to edit or navigate.
313
+ - Mode changes preserve the current page and unsaved fields. Sites that select layouts only
314
+ at page load may need an explicit Reload. There are no click, swipe or scroll mode controls.
145
315
 
146
316
  ## Security boundary
147
317
 
@@ -160,29 +330,148 @@ User-supplied `AGENT_BROWSER_*` variables are deliberately ignored.
160
330
  callbacks expose no authenticated caller identity, so these human-viewer tokens are a workflow
161
331
  safeguard, not an authorization boundary. The stdio MCP adapter separately uses an
162
332
  opaque, workspace-bound credential.
163
- - Downloads, uploads, clipboard synchronization, media permissions, extensions, native passkeys,
164
- and platform authenticators are not exposed by this plugin.
333
+ - The bundled trusted video helper has tabCapture/debugger access to capture the exact workspace
334
+ tab. It exposes no content scripts, site messages, network host permissions or web-accessible
335
+ resources. User-supplied extensions and page media permissions remain unavailable.
336
+ - Downloads, uploads, clipboard synchronization, native passkeys and platform authenticators
337
+ are not exposed by this plugin.
165
338
  - Browser navigation uses the daemon OS user's network access, including local development servers.
166
339
  It is therefore a trusted-agent capability; this plugin deliberately does not apply a blanket
167
340
  loopback or RFC1918 navigation ban.
168
341
 
169
342
  ## Develop
170
343
 
344
+ Follow the repository's `AGENTS.md`. Runtime verification must use an isolated
345
+ Paseo home, never the default daemon, active plugin installation, or signed-in
346
+ profile. Run dependency installation from the monorepo root, then the following
347
+ checks from `paseo-shared-browser`:
348
+
171
349
  ```bash
172
- bun install
350
+ bun install --frozen-lockfile
173
351
  bun run typecheck
174
- bun run lint
175
- bun run format:check
352
+ bun run check
176
353
  bun run test:unit
354
+ ```
355
+
356
+ The `test:tabs` native check requires explicit
357
+ `PASEO_SHARED_BROWSER_AGENT_BROWSER_BINARY` and
358
+ `PASEO_SHARED_BROWSER_CHROMIUM_EXECUTABLE` paths. It creates and cleans up its own
359
+ temporary Paseo home and browser profile.
360
+
361
+ `node scripts/test-image-capture-mounted.mjs --tooling-root /path/to/test-workspace`
362
+ opt-in checks the real React/React Query image scheduling and viewing recovery hooks
363
+ in JSDOM. The tooling workspace must already provide compatible React, react-dom,
364
+ jsdom, React Query and esbuild. The plugin intentionally does not add these DOM test
365
+ dependencies or install missing packages. Omitting `--tooling-root` uses this
366
+ checkout if its dependencies include them. The command owns and removes temporary
367
+ hook bundles; it never launches or accesses a browser/runtime or profile.
368
+ For reproducible mounted checks, create a disposable tooling workspace with
369
+ React and react-dom 19.1.0 (matching this repository's React version), jsdom
370
+ 30.1.1, React Query 5.102.3 and esbuild 0.28.2. jsdom requires Node 24.15 or
371
+ newer on the supported Node 24 line. These are test-only dependencies, not
372
+ plugin runtime dependencies. For example:
373
+
374
+ ```bash
375
+ npm install --prefix /tmp/shared-browser-dom-tools --ignore-scripts --no-save \
376
+ react@19.1.0 react-dom@19.1.0 jsdom@30.1.1 \
377
+ @tanstack/react-query@5.102.3 esbuild@0.28.2
378
+ node scripts/test-image-capture-mounted.mjs --tooling-root /tmp/shared-browser-dom-tools
379
+ node scripts/test-panel-video-mounted.mjs --tooling-root /tmp/shared-browser-dom-tools --fixture settlement
380
+ ```
381
+
382
+ The panel fixture also accepts `handoff`, `rapid`, `continuous` and `admission`.
383
+ Run each variant for changes to media handoff or input policy. The `settlement`
384
+ fixture checks late replies across viewer and controller replacement. Delete
385
+ the tooling directory when done.
386
+
387
+ Use the same command with `--fixture density` for six mounted controller/density
388
+ menu regressions, including observer bounds, same-workspace token replacement,
389
+ unmount and no retry after an uncertain write.
390
+
391
+ Prepare runtime assets with an explicit temporary `PASEO_HOME` before either
392
+ browser gate. Keep that directory outside the default Paseo home. For example,
393
+ on POSIX systems:
394
+
395
+ ```bash
396
+ export PASEO_HOME="$(mktemp -d -t shared-browser-verification-XXXXXX)"
177
397
  bun run prepare:runtime
398
+ bun run test:smoke
399
+ bun run test:video
178
400
  ```
179
401
 
402
+ Remove the temporary directory after verification. On Windows, use a new
403
+ explicit temporary directory in `$env:PASEO_HOME` instead. Tests manage their
404
+ own browser teardown; do not run daemon lifecycle commands against the default
405
+ host to recover a test failure.
406
+
180
407
  `bun run test:smoke` launches the configured real Chromium runtime and exercises two viewers,
181
408
  control handoff, reconnect, stale-frame rejection, viewport changes, device emulation, profile
182
409
  persistence, and archive teardown.
183
410
 
411
+ `bun run test:video` is an optional real video gate. It opts its own runtime into encoded video
412
+ and reads prepared Chromium/CLI assets,
413
+ then creates its own temporary Paseo home, browser profile, Unix socket and local fixture pages.
414
+ It checks the production video decoder and canvas at 1280x800, sharp Pixel and 2560x2560,
415
+ including all four corner markers. It never uses the live browser. This gate requires usable
416
+ WebCodecs/tab capture and the local runtime assets; it is separate from unit tests.
417
+ Linux CI requires Xvfb and runs this video gate without skipping unsupported
418
+ capture. macOS and Windows run the image/browser smoke; their native video
419
+ hardware and remote-network performance require separate qualification. CI also
420
+ runs the actual Paseo compiler on both plugin entries.
421
+
184
422
  Release Please maintains the version, changelog, component tag, and GitHub release from
185
423
  Conventional Commits in the monorepo.
186
424
 
187
- Both the Paseo daemon and app must satisfy `^0.9.0 || ^0.10.0 || ^0.11.0`. The client surface uses React Native primitives
188
- and works in desktop, web, iOS, and Android Paseo clients.
425
+ Both the Paseo daemon and app must satisfy `^0.11.0`. Encoded video is opt-in on the daemon and
426
+ requires WebCodecs in a web client; JPEG is always available. The client surface uses React
427
+ Native primitives and works in desktop, web, iOS, and Android Paseo clients.
428
+
429
+ Every independent press (mouse down or first touch contact) carries the decoded frame
430
+ receipt that was visible when it was made. Receipts last five seconds and are spent by
431
+ acknowledged input, so a stalled or superseded presentation cannot admit a later click;
432
+ held drags, releases and keys continue on their channel. A wheel capture can finish decoding
433
+ after input revoked its frame token. The server
434
+ returns a known non-admission receipt before creating a new channel; the canvas waits
435
+ for another decoded frame before admitting that still-unsent gesture. Recovery allows
436
+ up to three admission attempts within one four-second decoded-frame wait budget.
437
+ Published input, unknown outcomes, expired leases, and replaced contexts are never retried.
438
+
439
+
440
+ ## Maintainer references
441
+
442
+ [Streaming design](STREAMING_DESIGN.md) documents media ownership, resource
443
+ bounds, input admission and recovery. [Remote-control research](REMOTE_BROWSER_RESEARCH.md)
444
+ records primary-source comparisons and the trade-offs behind continuous human
445
+ input. These describe the implementation's guarantees, not a promise of a
446
+ particular frame rate or remote-network latency.
447
+
448
+
449
+ ## Optional Linux desktop hover
450
+
451
+ Hidden browsers are headless by default. To expose desktop pointer and hover
452
+ behavior on Linux, opt in by setting `PASEO_SHARED_BROWSER_XVFB=1` (exactly `1`;
453
+ any other value stays headless) in the environment of the Shared Browser
454
+ supervisor process. The supervisor is a detached process that inherits the
455
+ daemon environment when it is spawned, so changing the variable affects it only
456
+ after that supervisor process restarts. Within a running supervisor the variable
457
+ is read each time a workspace browser runtime is created; existing runtimes keep
458
+ their mode. A trusted embedding can pass `virtualDisplay: true | false` to
459
+ `createRuntimeOwner`; an explicit `false` overrides the variable.
460
+
461
+ When opted in and `/usr/bin/Xvfb` exists, each hidden Linux browser uses its own
462
+ private, authenticated display. No visible window opens and no TCP or pathname
463
+ listener is created. The display is stopped with its browser. Headed mode and
464
+ non-Linux platforms are unchanged. The plugin never installs Xvfb or changes host
465
+ display settings.
466
+
467
+ If the display cannot start, the browser falls back to headless and the viewer
468
+ shows a short notice (for example "Private display unavailable: Xvfb was not
469
+ found."). The notice never contains paths or credentials. If an opted-in display
470
+ dies while running, requests fail and the viewer shows an error; use **Reconnect
471
+ viewer** to replace the browser. Reconnect discards the old runtime and starts a
472
+ new one, so the page reloads from its initial URL, and old viewer, control and
473
+ input attachments are rejected. Nothing is replayed automatically.
474
+
475
+ Known limitation: if the supervisor is killed with SIGKILL it cannot stop its
476
+ child, so an orphaned Xvfb process and its private authority directory under the
477
+ temporary directory can remain until removed manually.