@omercnet/paseo-shared-browser 1.2.2 → 2.0.0-next.249.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/OVERVIEW.md +2 -2
  2. package/README.md +316 -27
  3. package/REMOTE_BROWSER_RESEARCH.md +395 -0
  4. package/STREAMING_DESIGN.md +196 -0
  5. package/client/browser-auto-capture-density.test.ts +102 -0
  6. package/client/browser-auto-capture-density.ts +30 -0
  7. package/client/browser-canvas-input.test.ts +198 -0
  8. package/client/browser-canvas-input.ts +202 -0
  9. package/client/browser-canvas-keyboard-web.test.ts +113 -0
  10. package/client/browser-canvas-keyboard-web.ts +230 -0
  11. package/client/browser-canvas-keyboard.test.ts +123 -0
  12. package/client/browser-canvas-keyboard.ts +126 -0
  13. package/client/browser-canvas-layout.test.ts +109 -0
  14. package/client/browser-canvas-layout.ts +51 -0
  15. package/client/browser-canvas-viewport-web.test.ts +77 -0
  16. package/client/browser-canvas-viewport-web.ts +63 -0
  17. package/client/browser-canvas-viewport.tsx +152 -0
  18. package/client/browser-capture-cadence.test.ts +9 -0
  19. package/client/browser-capture-cadence.ts +6 -0
  20. package/client/browser-capture-density-controls.tsx +75 -0
  21. package/client/browser-chrome.tsx +234 -0
  22. package/client/browser-client-presence.ts +184 -0
  23. package/client/browser-compose-controls.test.tsx +275 -0
  24. package/client/browser-compose-controls.tsx +174 -0
  25. package/client/browser-control-button.tsx +109 -0
  26. package/client/browser-control-dialog.tsx +71 -0
  27. package/client/browser-control-request.test.ts +27 -0
  28. package/client/browser-control-request.ts +57 -0
  29. package/client/browser-control-tooltip-web.ts +10 -0
  30. package/client/browser-dialog-body.tsx +22 -0
  31. package/client/browser-emulation-mode.test.ts +75 -0
  32. package/client/browser-emulation-mode.ts +98 -0
  33. package/client/browser-fill-viewport.test.ts +45 -0
  34. package/client/browser-fill-viewport.ts +28 -0
  35. package/client/browser-frame-buffer.test.ts +157 -0
  36. package/client/browser-frame-buffer.ts +148 -0
  37. package/client/browser-frame-image.tsx +62 -0
  38. package/client/browser-image-load.test.ts +23 -0
  39. package/client/browser-image-load.ts +26 -0
  40. package/client/browser-input-queue.test.ts +1254 -0
  41. package/client/browser-input-queue.ts +519 -0
  42. package/client/browser-menu-placement.test.ts +90 -0
  43. package/client/browser-menu-placement.ts +78 -0
  44. package/client/browser-panel-styles.ts +384 -0
  45. package/client/browser-quality-controls.test.ts +26 -0
  46. package/client/browser-quality-controls.tsx +139 -0
  47. package/client/browser-resolution-picker.tsx +189 -0
  48. package/client/browser-tabs.tsx +117 -0
  49. package/client/browser-toolbar-menu-web.test.ts +185 -0
  50. package/client/browser-toolbar-menu-web.ts +159 -0
  51. package/client/browser-toolbar-menu.tsx +255 -0
  52. package/client/browser-toolbar-submenu.tsx +93 -0
  53. package/client/browser-video-bytes.test.ts +44 -0
  54. package/client/browser-video-currency.ts +45 -0
  55. package/client/browser-video-decoder.test.ts +473 -0
  56. package/client/browser-video-decoder.ts +311 -0
  57. package/client/browser-video-surface.tsx +21 -0
  58. package/client/browser-video-visibility.test.ts +45 -0
  59. package/client/browser-viewer-recovery.test.ts +111 -0
  60. package/client/browser-viewer-recovery.ts +78 -0
  61. package/client/browser.tsx +1650 -1579
  62. package/client/favorite-star.tsx +23 -0
  63. package/client/frame-lifecycle.ts +62 -59
  64. package/client/sidebar.tsx +1 -2
  65. package/client/use-browser-auto-capture-density.test.ts +104 -0
  66. package/client/use-browser-auto-capture-density.ts +43 -0
  67. package/client/use-browser-canvas-input.ts +375 -0
  68. package/client/use-browser-capture-density.ts +87 -0
  69. package/client/use-browser-emulation-mode.ts +56 -0
  70. package/client/use-browser-fill-viewport.test.ts +108 -0
  71. package/client/use-browser-fill-viewport.ts +62 -0
  72. package/client/use-browser-frame-buffer.ts +66 -0
  73. package/client/use-browser-image-capture.ts +139 -0
  74. package/client/use-browser-scoped-mutation.test.ts +118 -0
  75. package/client/use-browser-scoped-mutation.ts +59 -0
  76. package/client/use-browser-video.test.ts +1060 -0
  77. package/client/use-browser-video.ts +481 -0
  78. package/client/use-browser-viewer-recovery.ts +26 -0
  79. package/client/use-resolution-favorites.ts +90 -0
  80. package/client/web.test.ts +125 -0
  81. package/client/web.ts +405 -0
  82. package/index.client.tsx +0 -10
  83. package/index.server.ts +32 -4
  84. package/npm-shrinkwrap.json +9 -9
  85. package/package.json +10 -5
  86. package/paseo-plugin.json +1 -1
  87. package/scripts/prepare-runtime.mjs +1 -1
  88. package/server/agent-browser-runtime.ts +1601 -314
  89. package/server/browser-cursor.test.ts +64 -0
  90. package/server/browser-cursor.ts +69 -0
  91. package/server/browser-frame-context.ts +22 -0
  92. package/server/browser-frame-epoch.test.ts +95 -0
  93. package/server/browser-frame-rejection.ts +17 -0
  94. package/server/browser-gesture.ts +120 -0
  95. package/server/browser-operation-diagnostics.test.ts +79 -0
  96. package/server/browser-operation-diagnostics.ts +103 -0
  97. package/server/browser-policy.ts +1316 -151
  98. package/server/browser-video-input-authority.test.ts +211 -0
  99. package/server/browser-video-policy.test.ts +564 -0
  100. package/server/browser-video-socket.test.ts +368 -0
  101. package/server/browser.ts +151 -16
  102. package/server/capture-transport-policy.test.ts +105 -0
  103. package/server/capture-transport-policy.ts +154 -0
  104. package/server/capture-transport-runtime.test.ts +213 -0
  105. package/server/daemon-launch-chain.test.ts +70 -0
  106. package/server/daemon-shutdown.test.ts +289 -0
  107. package/server/discrete-input-freshness.test.ts +364 -0
  108. package/server/discrete-input-native-lifetime.test.ts +150 -0
  109. package/server/display-loss-recovery.test.ts +294 -0
  110. package/server/emulation-restoration.test.ts +262 -0
  111. package/server/input-cleanup.ts +23 -0
  112. package/server/input-generation.test.ts +31 -0
  113. package/server/input-generation.ts +36 -0
  114. package/server/jpeg-capture-demand.test.ts +76 -0
  115. package/server/jpeg-capture-demand.ts +98 -0
  116. package/server/jpeg-frame.ts +29 -0
  117. package/server/keyboard-input.test.ts +61 -0
  118. package/server/keyboard-input.ts +104 -0
  119. package/server/live-input-policy.test.ts +958 -0
  120. package/server/live-input-runtime.test.ts +732 -0
  121. package/server/live-input-supervisor.test.ts +147 -0
  122. package/server/mcp-entry.ts +44 -2
  123. package/server/native-video-backpressure.test.ts +228 -0
  124. package/server/native-video-buffer.test.ts +141 -0
  125. package/server/native-video-buffer.ts +114 -0
  126. package/server/native-video-capture.test.ts +599 -0
  127. package/server/native-video-capture.ts +497 -0
  128. package/server/native-video-extension-source.test.ts +201 -0
  129. package/server/native-video-extension-source.ts +364 -0
  130. package/server/native-video-extension.test.ts +41 -0
  131. package/server/native-video-extension.ts +83 -0
  132. package/server/native-video-packet.ts +9 -0
  133. package/server/native-video-settings.test.ts +245 -0
  134. package/server/navigation-budget.ts +8 -0
  135. package/server/navigation-commit.ts +125 -0
  136. package/server/navigation-policy.test.ts +153 -0
  137. package/server/process-identity.ts +58 -0
  138. package/server/reattach-held-input.test.ts +143 -0
  139. package/server/runtime-frame-reuse.test.ts +295 -0
  140. package/server/runtime-jpeg-demand.test.ts +200 -0
  141. package/server/runtime-jpeg-shutdown.test.ts +210 -0
  142. package/server/runtime-jpeg-source-order.test.ts +201 -0
  143. package/server/runtime-launch-environment.test.ts +111 -0
  144. package/server/runtime-navigation.test.ts +327 -0
  145. package/server/runtime-owner-display.test.ts +152 -0
  146. package/server/runtime-owner.ts +340 -31
  147. package/server/runtime-page-state.test.ts +249 -0
  148. package/server/runtime-protocol.ts +32 -2
  149. package/server/runtime-target-retention.test.ts +181 -0
  150. package/server/runtime-video-opt-in.test.ts +82 -0
  151. package/server/runtime-video.test.ts +444 -0
  152. package/server/screencast-source-time.ts +46 -0
  153. package/server/stale-wheel-admission.test.ts +136 -0
  154. package/server/supervisor-client.ts +28 -4
  155. package/server/supervisor-heartbeat.test.ts +80 -0
  156. package/server/supervisor.ts +553 -55
  157. package/server/tab-close-confirmation.test.ts +97 -0
  158. package/server/video-source-recovery.test.ts +20 -0
  159. package/server/video-source-recovery.ts +64 -0
  160. package/server/viewer-capture-lifetime.ts +51 -0
  161. package/server/virtual-display.test.ts +184 -0
  162. package/server/virtual-display.ts +236 -0
  163. package/shared/browser-display-preferences.ts +46 -0
  164. package/shared/browser-video.ts +83 -0
  165. package/shared/browser.ts +286 -65
  166. package/shared/capture-density.ts +16 -0
  167. package/shared/capture-settings.ts +17 -0
  168. package/shared/device-presets.ts +90 -0
  169. package/shared/resolution-menu.test.ts +120 -0
  170. package/shared/resolution-menu.ts +72 -0
  171. package/shared/video-settings.test.ts +38 -0
  172. package/shared/video-settings.ts +34 -0
  173. package/shared/viewport-limits.ts +5 -0
@@ -0,0 +1,46 @@
1
+ /**
2
+ * Host-persisted display favorites through Paseo's settings service. These values
3
+ * affect viewing quality and picker convenience, never input ownership, page
4
+ * navigation, emulation or pixel density. Version1 migration retains the bitrate
5
+ * formerly coupled to JPEG quality.
6
+ * Retired IDs remain parseable; the current catalogue prunes them at the view.
7
+ */
8
+ import { defineSettings } from "@getpaseo/plugin";
9
+ import { z } from "zod";
10
+ import { DEFAULT_CAPTURE_QUALITY } from "./capture-settings";
11
+ import { DEVICE_PRESETS } from "./device-presets";
12
+ import {
13
+ resolveVideoEncoderSettings,
14
+ videoBitrateSchema,
15
+ videoFrameRateSchema,
16
+ } from "./video-settings";
17
+
18
+ export const browserDisplayPreferencesSchema = z.object({
19
+ captureQuality: z.enum(["low", "medium", "high", "maximum"]).default(DEFAULT_CAPTURE_QUALITY),
20
+ videoBitrate: videoBitrateSchema.default(12_000_000),
21
+ videoFps: videoFrameRateSchema.default(30),
22
+ favoritePresetIds: z.array(z.string().min(1).max(128)).max(DEVICE_PRESETS.length).default([]),
23
+ });
24
+
25
+ export type BrowserDisplayPreferences = z.output<typeof browserDisplayPreferencesSchema>;
26
+
27
+ /** One revisioned document per plugin host, shared across that host's panels. */
28
+ export const browserDisplayPreferences = defineSettings({
29
+ id: "display-preferences",
30
+ scope: "host",
31
+ version: 2,
32
+ migrate(values, fromVersion) {
33
+ if (fromVersion !== 1 || !values || typeof values !== "object") return values;
34
+ const previous = values as Record<string, unknown>;
35
+ const quality =
36
+ previous.captureQuality === "low" || previous.captureQuality === "medium"
37
+ ? previous.captureQuality
38
+ : "high";
39
+ return {
40
+ ...previous,
41
+ videoBitrate: resolveVideoEncoderSettings({ quality }).bitrate,
42
+ videoFps: 30,
43
+ };
44
+ },
45
+ schema: browserDisplayPreferencesSchema,
46
+ });
@@ -0,0 +1,83 @@
1
+ /**
2
+ * Encoded browser-tab video over the existing authenticated Paseo RPC transport.
3
+ * Opt-in on the daemon and decoded by WebCodecs on the web; every other client retains JPEG.
4
+ * Packets never grant control; only a decoded
5
+ * presentation with current browser authority can provide an input target.
6
+ */
7
+ import { defineRpc } from "@getpaseo/plugin";
8
+ import { z } from "zod";
9
+ import { browserFrameSchema, browserStateSchema } from "./browser";
10
+ import { videoBitrateSchema, videoFrameRateSchema } from "./video-settings";
11
+ import { MAX_VIEWPORT } from "./viewport-limits";
12
+
13
+ export const VIDEO_MAX_PACKET_BYTES = 2 * 1024 * 1024;
14
+ export const VIDEO_MAX_BATCH_BYTES = 4 * 1024 * 1024;
15
+ export const VIDEO_MAX_BATCH_PACKETS = 32;
16
+ export const VIDEO_MAX_WAIT_MS = 500;
17
+ export const VIDEO_FRAME_RATE = 30;
18
+ /** Maximum future source-clock skew admitted by the calibrated native capture. */
19
+ export const VIDEO_SOURCE_CLOCK_TOLERANCE_MS = 50;
20
+
21
+ const opaqueIdSchema = browserFrameSchema.shape.frameId;
22
+ const sequenceSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER);
23
+ const base64PacketSchema = z
24
+ .string()
25
+ .min(1)
26
+ .max(Math.ceil(VIDEO_MAX_PACKET_BYTES / 3) * 4);
27
+
28
+ /** Geometry and generation shared by JPEG and decoded video input admission. */
29
+ export const browserFrameAuthoritySchema = browserFrameSchema.omit({
30
+ mimeType: true,
31
+ transport: true,
32
+ dataBase64: true,
33
+ byteLength: true,
34
+ });
35
+ export type BrowserFrameAuthority = z.infer<typeof browserFrameAuthoritySchema>;
36
+
37
+ /** Native encoded output; timestamps are source VideoFrame microseconds, not wall-clock time. */
38
+ export const nativeVideoPacketSchema = z.object({
39
+ streamId: opaqueIdSchema,
40
+ captureGeneration: sequenceSchema,
41
+ sequence: sequenceSchema,
42
+ timestampUs: sequenceSchema,
43
+ type: z.enum(["key", "delta"]),
44
+ codec: z.string().min(1).max(64),
45
+ width: z.number().int().positive().max(MAX_VIEWPORT.width),
46
+ height: z.number().int().positive().max(MAX_VIEWPORT.height),
47
+ descriptionBase64: z.string().max(16_384).optional(),
48
+ dataBase64: base64PacketSchema,
49
+ capturedAt: z.string().datetime(),
50
+ });
51
+ export type NativeVideoPacket = z.infer<typeof nativeVideoPacketSchema>;
52
+
53
+ export const browserVideoPacketSchema = nativeVideoPacketSchema.extend({
54
+ frame: browserFrameAuthoritySchema,
55
+ /** False when the host refused this capture as an input receipt (it predates
56
+ * acknowledged input). It may still be painted. Absent from older hosts. */
57
+ actionable: z.boolean().optional(),
58
+ });
59
+ export type BrowserVideoPacket = z.infer<typeof browserVideoPacketSchema>;
60
+
61
+ export const readBrowserVideoRpc = defineRpc({
62
+ name: "shared-browser.video.read",
63
+ input: z.object({
64
+ viewerToken: opaqueIdSchema,
65
+ quality: z.enum(["low", "medium", "high"]).default("high"),
66
+ bitrate: videoBitrateSchema.optional(),
67
+ fps: videoFrameRateSchema.optional(),
68
+ streamId: opaqueIdSchema.nullable().default(null),
69
+ afterSequence: sequenceSchema.default(0),
70
+ requestKeyFrame: z.boolean().default(false),
71
+ waitMs: z.number().int().min(0).max(VIDEO_MAX_WAIT_MS).default(250),
72
+ }),
73
+ output: z.object({
74
+ state: browserStateSchema,
75
+ status: z.enum(["ready", "waiting", "reset", "unsupported"]),
76
+ streamId: opaqueIdSchema.nullable(),
77
+ packets: z.array(browserVideoPacketSchema).max(VIDEO_MAX_BATCH_PACKETS),
78
+ reason: z.string().max(256).optional(),
79
+ reasonCode: z.enum(["encoder-capacity", "video-disabled"]).optional(),
80
+ }),
81
+ });
82
+ export type BrowserVideoReadInput = z.infer<typeof readBrowserVideoRpc.input>;
83
+ export type BrowserVideoReadReply = z.infer<typeof readBrowserVideoRpc.output>;
package/shared/browser.ts CHANGED
@@ -1,71 +1,28 @@
1
+ import { captureDensitySchema } from "./capture-density";
2
+
3
+ export { type CaptureDensity, canUseCaptureDensity, captureDensitySchema } from "./capture-density";
4
+
1
5
  import { defineRpc } from "@getpaseo/plugin";
2
6
  import { z } from "zod";
7
+ import {
8
+ DEFAULT_CAPTURE_QUALITY,
9
+ FRAME_MAX_BASE64_CHARS,
10
+ FRAME_MAX_BYTES,
11
+ } from "./capture-settings";
12
+ import { DEVICE_PRESET_IDS } from "./device-presets";
13
+ import { MAX_VIEWPORT, MIN_VIEWPORT } from "./viewport-limits";
14
+
15
+ export {
16
+ DEFAULT_CAPTURE_QUALITY,
17
+ FRAME_MAX_BASE64_CHARS,
18
+ FRAME_MAX_BYTES,
19
+ } from "./capture-settings";
20
+ export { DEVICE_PRESET_IDS, DEVICE_PRESETS, type DevicePresetId } from "./device-presets";
3
21
 
4
22
  export const DEFAULT_VIEWPORT = { width: 1280, height: 800 } as const;
5
- export const MIN_VIEWPORT = { width: 320, height: 480 } as const;
6
- export const MAX_VIEWPORT = { width: 1600, height: 1200 } as const;
7
- export const FRAME_MAX_BYTES = 800_000;
8
- export const FRAME_MAX_BASE64_CHARS = Math.ceil(FRAME_MAX_BYTES / 3) * 4;
9
-
10
- export const DEVICE_PRESETS = [
11
- {
12
- id: "desktop-chrome",
13
- label: "Desktop Chrome",
14
- shortLabel: "Desktop",
15
- viewport: { width: 1280, height: 720 },
16
- deviceScaleFactor: 1,
17
- isMobile: false,
18
- hasTouch: false,
19
- platform: "Win32",
20
- userAgent:
21
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.8010.12 Safari/537.36",
22
- },
23
- {
24
- id: "iphone-15-pro",
25
- label: "iPhone 15 Pro",
26
- shortLabel: "iPhone 15",
27
- viewport: { width: 393, height: 659 },
28
- deviceScaleFactor: 3,
29
- isMobile: true,
30
- hasTouch: true,
31
- platform: "iPhone",
32
- userAgent:
33
- "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6 Mobile/15E148 Safari/604.1",
34
- },
35
- {
36
- id: "pixel-7",
37
- label: "Pixel 7",
38
- shortLabel: "Pixel 7",
39
- viewport: { width: 412, height: 839 },
40
- deviceScaleFactor: 2.625,
41
- isMobile: true,
42
- hasTouch: true,
43
- platform: "Linux armv81",
44
- userAgent:
45
- "Mozilla/5.0 (Linux; Android 14; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.8010.12 Mobile Safari/537.36",
46
- },
47
- {
48
- id: "ipad-pro-11",
49
- label: "iPad Pro 11",
50
- shortLabel: "iPad 11",
51
- viewport: { width: 834, height: 1194 },
52
- deviceScaleFactor: 2,
53
- isMobile: true,
54
- hasTouch: true,
55
- platform: "iPad",
56
- userAgent:
57
- "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6 Mobile/15E148 Safari/604.1",
58
- },
59
- ] as const;
60
-
61
- export const DEVICE_PRESET_IDS = [
62
- "desktop-chrome",
63
- "iphone-15-pro",
64
- "pixel-7",
65
- "ipad-pro-11",
66
- ] as const;
67
-
68
- export type DevicePresetId = (typeof DEVICE_PRESET_IDS)[number];
23
+ export const MAX_BROWSER_TABS = 8;
24
+ export { MAX_VIEWPORT, MIN_VIEWPORT } from "./viewport-limits";
25
+
69
26
  const devicePresetIdSchema = z.enum(DEVICE_PRESET_IDS);
70
27
 
71
28
  const workspaceIdSchema = z
@@ -81,6 +38,20 @@ const opaqueTokenSchema = z
81
38
  const generationSchema = z.number().int().nonnegative();
82
39
  const epochSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER);
83
40
  const runtimeIdSchema = opaqueTokenSchema;
41
+ export const browserTabIdSchema = z
42
+ .string()
43
+ .min(1)
44
+ .max(128)
45
+ .regex(/^[A-Za-z0-9_-]+$/);
46
+
47
+ export const browserTabSchema = z.object({
48
+ id: browserTabIdSchema,
49
+ title: z.string().max(1_024),
50
+ url: z.string().max(8_192),
51
+ viewerCount: z.number().int().nonnegative(),
52
+ controllerLabel: z.string().max(64).nullable(),
53
+ });
54
+ export type BrowserTab = z.output<typeof browserTabSchema>;
84
55
 
85
56
  export const browserRecoveryStateSchema = z.enum([
86
57
  "available",
@@ -102,6 +73,7 @@ export const browserStateSchema = z.object({
102
73
  canGoBack: z.boolean(),
103
74
  canGoForward: z.boolean(),
104
75
  viewport: viewportSchema,
76
+ captureScale: z.number().min(1).max(2).default(1),
105
77
  navigationGeneration: generationSchema,
106
78
  viewportGeneration: generationSchema,
107
79
  devicePresetId: devicePresetIdSchema.nullable(),
@@ -111,7 +83,9 @@ export const browserStateSchema = z.object({
111
83
  controllerExpiresAt: z.string().datetime().nullable(),
112
84
  viewerCount: z.number().int().nonnegative(),
113
85
  error: z.string().max(2_048).nullable(),
86
+ notice: z.string().max(256).optional(),
114
87
  runtimeId: runtimeIdSchema.optional(),
88
+ tabId: browserTabIdSchema.optional(),
115
89
  runtimeCreatedAt: epochSchema.optional(),
116
90
  bridgeEpoch: epochSchema.optional(),
117
91
  recoveryState: browserRecoveryStateSchema.optional(),
@@ -138,6 +112,7 @@ export const attachBrowserRpc = defineRpc({
138
112
  input: z.object({
139
113
  workspaceId: workspaceIdSchema,
140
114
  viewerLabel: z.string().trim().min(1).max(64),
115
+ tabId: browserTabIdSchema.optional(),
141
116
  }),
142
117
  output: z.object({
143
118
  viewerToken: opaqueTokenSchema,
@@ -145,12 +120,53 @@ export const attachBrowserRpc = defineRpc({
145
120
  }),
146
121
  });
147
122
 
123
+ export const listBrowserTabsRpc = defineRpc({
124
+ name: "shared-browser.tabs.list",
125
+ input: z.object({ viewerToken: opaqueTokenSchema }),
126
+ output: z.object({ tabs: z.array(browserTabSchema).max(MAX_BROWSER_TABS) }),
127
+ });
128
+
129
+ export const createBrowserTabRpc = defineRpc({
130
+ name: "shared-browser.tabs.create",
131
+ input: z.object({ viewerToken: opaqueTokenSchema }),
132
+ output: z.object({ tabId: browserTabIdSchema }),
133
+ });
134
+
135
+ export const closeBrowserTabRpc = defineRpc({
136
+ name: "shared-browser.tabs.close",
137
+ input: z.object({
138
+ viewerToken: opaqueTokenSchema,
139
+ controlToken: opaqueTokenSchema,
140
+ tabId: browserTabIdSchema,
141
+ }),
142
+ output: z.object({ closed: z.literal(true) }),
143
+ });
144
+
148
145
  export const detachBrowserRpc = defineRpc({
149
146
  name: "shared-browser.detach",
150
147
  input: z.object({ viewerToken: opaqueTokenSchema }),
151
148
  output: z.object({ detached: z.boolean() }),
152
149
  });
153
150
 
151
+ /** Stop the current workspace runtime only when its controller still owns this session. */
152
+ export const closeBrowserRpc = defineRpc({
153
+ name: "shared-browser.close",
154
+ input: z.object({
155
+ viewerToken: opaqueTokenSchema,
156
+ controlToken: opaqueTokenSchema,
157
+ sessionId: opaqueTokenSchema,
158
+ runtimeId: runtimeIdSchema,
159
+ }),
160
+ output: z.object({ closed: z.literal(true) }),
161
+ });
162
+
163
+ /** Reopen a deliberately closed workspace browser using its retained profile. */
164
+ export const reopenBrowserRpc = defineRpc({
165
+ name: "shared-browser.reopen",
166
+ input: z.object({ workspaceId: workspaceIdSchema }),
167
+ output: z.object({ opened: z.literal(true) }),
168
+ });
169
+
154
170
  export const listOpenBrowserWorkspacesRpc = defineRpc({
155
171
  name: "shared-browser.presence",
156
172
  input: z.object({}),
@@ -161,7 +177,7 @@ export const captureBrowserRpc = defineRpc({
161
177
  name: "shared-browser.capture",
162
178
  input: z.object({
163
179
  viewerToken: opaqueTokenSchema,
164
- quality: z.enum(["low", "medium", "high"]).default("medium"),
180
+ quality: z.enum(["low", "medium", "high", "maximum"]).default(DEFAULT_CAPTURE_QUALITY),
165
181
  knownFrameId: opaqueTokenSchema.nullable().default(null),
166
182
  }),
167
183
  output: z.object({
@@ -225,6 +241,22 @@ export const resizeBrowserRpc = defineRpc({
225
241
  controlToken: opaqueTokenSchema,
226
242
  expected: expectedStateSchema,
227
243
  viewport: viewportSchema,
244
+ /** Panel following changes layout size without resetting device behavior or density. */
245
+ preserveEmulation: z.boolean().optional(),
246
+ /** Resolve automatic density and panel geometry in one controlled transition. */
247
+ captureDensity: captureDensitySchema.optional(),
248
+ }),
249
+ output: z.object({ state: browserStateSchema }),
250
+ });
251
+
252
+ /** Independent, controller-authorized density selection; no page navigation or preset substitution. */
253
+ export const setCaptureDensityRpc = defineRpc({
254
+ name: "shared-browser.capture.density",
255
+ input: z.object({
256
+ viewerToken: opaqueTokenSchema,
257
+ controlToken: opaqueTokenSchema,
258
+ expected: expectedStateSchema,
259
+ density: captureDensitySchema,
228
260
  }),
229
261
  output: z.object({ state: browserStateSchema }),
230
262
  });
@@ -236,6 +268,8 @@ export const applyDevicePresetRpc = defineRpc({
236
268
  controlToken: opaqueTokenSchema,
237
269
  expected: expectedStateSchema,
238
270
  presetId: devicePresetIdSchema,
271
+ /** Mode-only changes retain current CSS dimensions and JPEG capture density. */
272
+ preserveDisplay: z.boolean().optional(),
239
273
  }),
240
274
  output: z.object({ state: browserStateSchema }),
241
275
  });
@@ -312,6 +346,193 @@ export const sendBrowserInputRpc = defineRpc({
312
346
  output: z.object({ state: browserStateSchema }),
313
347
  });
314
348
 
349
+ /** Safe CSS cursor names only. Custom cursor URLs never reach the local client. */
350
+ export const browserCursorSchema = z.enum([
351
+ "default",
352
+ "none",
353
+ "pointer",
354
+ "text",
355
+ "vertical-text",
356
+ "crosshair",
357
+ "move",
358
+ "grab",
359
+ "grabbing",
360
+ "wait",
361
+ "progress",
362
+ "help",
363
+ "not-allowed",
364
+ "no-drop",
365
+ "copy",
366
+ "alias",
367
+ "context-menu",
368
+ "cell",
369
+ "all-scroll",
370
+ "col-resize",
371
+ "row-resize",
372
+ "n-resize",
373
+ "s-resize",
374
+ "e-resize",
375
+ "w-resize",
376
+ "ne-resize",
377
+ "nw-resize",
378
+ "se-resize",
379
+ "sw-resize",
380
+ "ew-resize",
381
+ "ns-resize",
382
+ "nesw-resize",
383
+ "nwse-resize",
384
+ "zoom-in",
385
+ "zoom-out",
386
+ ]);
387
+ export type BrowserCursor = z.output<typeof browserCursorSchema>;
388
+
389
+ const touchPointSchema = displayedPointSchema.extend({
390
+ id: z.number().int().min(0).max(2_147_483_647),
391
+ });
392
+ /** Native keyboard modifiers use CDP's bits: Alt 1, Control 2, Meta 4, Shift 8. */
393
+ /** Bound simultaneous physical-key cleanup work without restricting layout codes. */
394
+ export const MAX_HELD_BROWSER_KEYS = 32;
395
+
396
+ export const browserGestureKeySchema = z
397
+ .object({
398
+ kind: z.literal("key"),
399
+ type: z.enum(["down", "up"]),
400
+ key: z.string().min(1).max(64),
401
+ code: z
402
+ .string()
403
+ .min(1)
404
+ .max(64)
405
+ .regex(/^[A-Za-z][A-Za-z0-9]*$/),
406
+ modifiers: z.number().int().min(0).max(15),
407
+ repeat: z.boolean().default(false),
408
+ text: z.string().min(1).max(32).optional(),
409
+ })
410
+ .superRefine((event, context) => {
411
+ if (event.type === "up" && (event.repeat || event.text !== undefined)) {
412
+ context.addIssue({ code: "custom", message: "Key release cannot repeat or insert text" });
413
+ }
414
+ if (event.text !== undefined && (event.modifiers & 7) !== 0) {
415
+ context.addIssue({ code: "custom", message: "Shortcut keys cannot insert printable text" });
416
+ }
417
+ });
418
+ export type BrowserGestureKeyEvent = z.output<typeof browserGestureKeySchema>;
419
+
420
+ /**
421
+ * Ordered live input, in displayed-image coordinates. Touch start/move contain
422
+ * the complete active ID set; removing contacts in move releases those contacts.
423
+ * End/cancel contain no points. Wheel deltas are bounded browser CSS pixels.
424
+ */
425
+ // Pointer snapshots use the same CDP modifier bits as key events. Omission
426
+ // preserves native/older callers' held-key inference; explicit zero means none.
427
+ const pointerModifiersSchema = z.number().int().min(0).max(15).optional();
428
+ export const browserGestureEventSchema = z.discriminatedUnion("kind", [
429
+ browserGestureKeySchema,
430
+ z.object({ kind: z.literal("text"), text: z.string().min(1).max(16_000) }),
431
+ z.object({ kind: z.literal("leave") }),
432
+ z.object({
433
+ kind: z.literal("move"),
434
+ point: displayedPointSchema,
435
+ modifiers: pointerModifiersSchema,
436
+ }),
437
+ z.object({
438
+ kind: z.literal("down"),
439
+ point: displayedPointSchema,
440
+ modifiers: pointerModifiersSchema,
441
+ button: z.enum(["left", "right", "middle"]).default("left"),
442
+ clickCount: z.union([z.literal(1), z.literal(2)]).default(1),
443
+ }),
444
+ z.object({
445
+ kind: z.literal("up"),
446
+ point: displayedPointSchema,
447
+ modifiers: pointerModifiersSchema,
448
+ button: z.enum(["left", "right", "middle"]).default("left"),
449
+ clickCount: z.union([z.literal(1), z.literal(2)]).default(1),
450
+ }),
451
+ z.object({
452
+ kind: z.literal("scroll"),
453
+ point: displayedPointSchema,
454
+ modifiers: pointerModifiersSchema,
455
+ deltaX: z.number().finite().min(-4_000).max(4_000),
456
+ deltaY: z.number().finite().min(-4_000).max(4_000),
457
+ }),
458
+ z
459
+ .object({
460
+ kind: z.literal("touch"),
461
+ type: z.enum(["start", "move", "end", "cancel"]),
462
+ points: z.array(touchPointSchema).max(5),
463
+ })
464
+ .superRefine((event, context) => {
465
+ if ((event.type === "end" || event.type === "cancel") !== (event.points.length === 0)) {
466
+ context.addIssue({
467
+ code: "custom",
468
+ message: "Touch start/move require points; end/cancel require none",
469
+ });
470
+ }
471
+ if (new Set(event.points.map((point) => point.id)).size !== event.points.length) {
472
+ context.addIssue({ code: "custom", message: "Touch identifiers must be unique" });
473
+ }
474
+ }),
475
+ ]);
476
+ export type BrowserGestureEvent = z.output<typeof browserGestureEventSchema>;
477
+
478
+ const gestureContextSchema = z.object({
479
+ viewerToken: opaqueTokenSchema,
480
+ controlToken: opaqueTokenSchema,
481
+ expected: expectedStateSchema.extend({ runtimeId: runtimeIdSchema, bridgeEpoch: epochSchema }),
482
+ });
483
+ const gestureContinuationSchema = gestureContextSchema.extend({
484
+ gestureId: opaqueTokenSchema,
485
+ sequence: z.number().int().positive().max(100_000),
486
+ });
487
+ /** Begin pins a decoded recent frame but sends no physical input. One live channel per controller. Keyboard events may share either pointer channel. */
488
+ export const beginBrowserGestureRpc = defineRpc({
489
+ name: "shared-browser.gesture.begin",
490
+ input: gestureContextSchema.extend({
491
+ target: targetFrameSchema,
492
+ pointerKind: z.enum(["mouse", "touch"]),
493
+ }),
494
+ output: z.union([
495
+ z.object({
496
+ state: browserStateSchema,
497
+ gestureId: opaqueTokenSchema,
498
+ nextSequence: z.number().int().positive(),
499
+ }),
500
+ // This receipt exists only before input.begin, so no physical action or channel is replayed.
501
+ z.object({ state: browserStateSchema, admission: z.literal("stale-frame") }),
502
+ ]),
503
+ });
504
+ /**
505
+ * Strict sequence continues the original frame context despite this channel's
506
+ * input invalidations. Independent mouse down/initial touch start require target;
507
+ * additional contacts in a held touch gesture continue its pinned geometry.
508
+ */
509
+ export const updateBrowserGestureRpc = defineRpc({
510
+ name: "shared-browser.gesture.update",
511
+ input: gestureContinuationSchema.extend({
512
+ event: browserGestureEventSchema,
513
+ target: targetFrameSchema.optional(),
514
+ }),
515
+ // The native input was acknowledged and navigated on its original attachment.
516
+ // This closes the old channel; it never authorizes a continuation or a replay.
517
+ output: z.object({
518
+ state: browserStateSchema,
519
+ gestureId: opaqueTokenSchema,
520
+ nextSequence: z.number().int().positive(),
521
+ cursor: browserCursorSchema.nullable(),
522
+ completion: z.literal("navigation").optional(),
523
+ }),
524
+ });
525
+ /**
526
+ * End/cancel releases the original held input without retrying an uncertain action.
527
+ * cancel permits a stale sequence after a lost reply, but still requires the exact
528
+ * owned gesture/controller. Normal end requires the acknowledged next sequence.
529
+ */
530
+ export const endBrowserGestureRpc = defineRpc({
531
+ name: "shared-browser.gesture.end",
532
+ input: gestureContinuationSchema.extend({ cancel: z.boolean().default(false) }),
533
+ output: z.object({ state: browserStateSchema, cursor: browserCursorSchema.nullable() }),
534
+ });
535
+
315
536
  export type BrowserState = z.output<typeof browserStateSchema>;
316
537
  export type BrowserFrame = z.output<typeof browserFrameSchema>;
317
538
  export type BrowserInputEvent = z.output<typeof browserInputEventSchema>;
@@ -0,0 +1,16 @@
1
+ /** Density changes physical capture pixels, never CSS layout or pointer coordinates. */
2
+ import { z } from "zod";
3
+ import { MAX_VIEWPORT } from "./viewport-limits";
4
+ export const captureDensitySchema = z.union([z.literal(1), z.literal(2)]);
5
+ export type CaptureDensity = z.infer<typeof captureDensitySchema>;
6
+
7
+ /** UI and server share the exact bound; an unsupported choice must not resize the page. */
8
+ export function canUseCaptureDensity(
9
+ viewport: { width: number; height: number },
10
+ density: CaptureDensity,
11
+ ): boolean {
12
+ return (
13
+ viewport.width * density <= MAX_VIEWPORT.width &&
14
+ viewport.height * density <= MAX_VIEWPORT.height
15
+ );
16
+ }
@@ -0,0 +1,17 @@
1
+ /** JPEG quality and byte bounds shared by the human viewer, agent adapter and browser runtime. */
2
+ export const DEFAULT_CAPTURE_QUALITY = "medium" as const;
3
+ export const JPEG_QUALITY = { low: 40, medium: 65, high: 85, maximum: 100 } as const;
4
+ export type CaptureQuality = keyof typeof JPEG_QUALITY;
5
+ export const DEFAULT_JPEG_QUALITY = JPEG_QUALITY[DEFAULT_CAPTURE_QUALITY];
6
+
7
+ // Preserve the original RPC frame bound; large captures reduce JPEG quality to fit.
8
+ export const FRAME_MAX_BYTES = 800_000;
9
+ export const FRAME_MAX_BASE64_CHARS = Math.ceil(FRAME_MAX_BYTES / 3) * 4;
10
+
11
+ /** Actual image pixels are independent of CSS layout and input coordinates for sharper phone capture. */
12
+ export function captureDimensions(viewport: { width: number; height: number }, captureScale = 1) {
13
+ return {
14
+ width: Math.round(viewport.width * captureScale),
15
+ height: Math.round(viewport.height * captureScale),
16
+ };
17
+ }
@@ -0,0 +1,90 @@
1
+ /** Device emulation facts and display sizes shared by the picker and server policy. */
2
+ const DESKTOP_CHROME_EMULATION = {
3
+ deviceScaleFactor: 1,
4
+ captureScale: 1,
5
+ isMobile: false,
6
+ hasTouch: false,
7
+ platform: "Win32",
8
+ userAgent:
9
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.8010.12 Safari/537.36",
10
+ } as const;
11
+
12
+ /** Larger desktop presets change resolution while retaining the same input and browser behavior. */
13
+ function createDesktopPreset<const Id extends string>(id: Id, width: number, height: number) {
14
+ const name = width === height ? `${width} square` : `${height}p`;
15
+
16
+ return {
17
+ ...DESKTOP_CHROME_EMULATION,
18
+ id,
19
+ label: name,
20
+ shortLabel: name,
21
+ viewport: { width, height },
22
+ } as const;
23
+ }
24
+
25
+ const PIXEL_7_EMULATION = {
26
+ id: "pixel-7",
27
+ label: "Pixel 7",
28
+ shortLabel: "Pixel 7",
29
+ viewport: { width: 412, height: 839 },
30
+ deviceScaleFactor: 2.625,
31
+ captureScale: 1,
32
+ isMobile: true,
33
+ hasTouch: true,
34
+ platform: "Linux armv81",
35
+ userAgent:
36
+ "Mozilla/5.0 (Linux; Android 14; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.8010.12 Mobile Safari/537.36",
37
+ } as const;
38
+
39
+ export const DEVICE_PRESETS = [
40
+ createDesktopPreset("desktop-chrome", 1280, 720),
41
+ createDesktopPreset("desktop-1280x800", 1280, 800),
42
+ createDesktopPreset("desktop-1280x1280", 1280, 1280),
43
+ createDesktopPreset("desktop-1440x810", 1440, 810),
44
+ createDesktopPreset("desktop-1440x900", 1440, 900),
45
+ createDesktopPreset("desktop-1440x1440", 1440, 1440),
46
+ createDesktopPreset("desktop-1920x1080", 1920, 1080),
47
+ createDesktopPreset("desktop-2560x1440", 2560, 1440),
48
+ createDesktopPreset("desktop-1920x1200", 1920, 1200),
49
+ createDesktopPreset("desktop-2560x1600", 2560, 1600),
50
+ createDesktopPreset("desktop-1920x1920", 1920, 1920),
51
+ createDesktopPreset("desktop-2560x2560", 2560, 2560),
52
+ {
53
+ id: "iphone-15-pro",
54
+ label: "iPhone 15 Pro",
55
+ shortLabel: "iPhone 15",
56
+ viewport: { width: 393, height: 659 },
57
+ deviceScaleFactor: 3,
58
+ captureScale: 1,
59
+ isMobile: true,
60
+ hasTouch: true,
61
+ platform: "iPhone",
62
+ userAgent:
63
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6 Mobile/15E148 Safari/604.1",
64
+ },
65
+ PIXEL_7_EMULATION,
66
+ {
67
+ ...PIXEL_7_EMULATION,
68
+ id: "pixel-7-sharp",
69
+ label: "Pixel 7 (high resolution)",
70
+ shortLabel: "Pixel 7 HD",
71
+ captureScale: 2,
72
+ },
73
+ {
74
+ id: "ipad-pro-11",
75
+ label: "iPad Pro 11",
76
+ shortLabel: "iPad 11",
77
+ viewport: { width: 834, height: 1194 },
78
+ deviceScaleFactor: 2,
79
+ captureScale: 1,
80
+ isMobile: true,
81
+ hasTouch: true,
82
+ platform: "iPad",
83
+ userAgent:
84
+ "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6 Mobile/15E148 Safari/604.1",
85
+ },
86
+ ] as const;
87
+
88
+ // The schema and picker derive from one catalogue, so adding a preset cannot leave it unselectable.
89
+ export const DEVICE_PRESET_IDS = DEVICE_PRESETS.map((preset) => preset.id);
90
+ export type DevicePresetId = (typeof DEVICE_PRESET_IDS)[number];