@omercnet/paseo-shared-browser 1.1.0 → 1.2.0-next.214.1

package/README.md

@@ -93,15 +93,19 @@ Before upgrading the plugin on Windows, close active Shared Browser sessions and
 daemon; Windows does not allow the installer to replace runtime executables that are still running.
 
 Open a workspace, search the Command Center for **Open Shared Browser**, or tap the **Shared
-Browser** composer pill while a workspace session is open.
+Browser** composer pill while a workspace session is open. On Paseo 0.11 or newer, a **Shared
+Browser (n)** row in the sidebar footer appears while browser sessions are open; it lists them and
+jumps to the chosen workspace's browser panel.
 
 ## Agent MCP access
 
 The plugin automatically injects its stdio MCP adapter only when a new, non-internal agent is
 created with a provider that accepts external MCP servers. Agents that already exist, resumed
-sessions, imported sessions, and Paseo's internal agents are not modified. Paseo's OMP provider
-uses native host-tool injection and rejects external MCP servers, so OMP agents are left unchanged.
-Pi agents continue to receive the adapter, but they require Pi's optional MCP support to launch it.
+sessions, imported sessions, and Paseo's internal agents are not modified. Paseo's built-in OMP
+provider accepts session MCP servers from Paseo 0.11, so new OMP agents receive the adapter there.
+On Paseo 0.9 and 0.10 the built-in OMP adapter rejects external MCP servers, so OMP agents are left
+unchanged. Pi agents continue to receive the adapter, but they require Pi's optional MCP support to
+launch it.
 
 The injected MCP server exposes exactly these tools: `shared_browser_status`,
 `shared_browser_capture`, `shared_browser_acquire_control`, `shared_browser_release_control`,
@@ -180,5 +184,5 @@ persistence, and archive teardown.
 Release Please maintains the version, changelog, component tag, and GitHub release from
 Conventional Commits in the monorepo.
 
-Both the Paseo daemon and app must satisfy `^0.9.0 || ^0.10.0`. The client surface uses React Native primitives
+Both the Paseo daemon and app must satisfy `^0.9.0 || ^0.10.0 || ^0.11.0`. The client surface uses React Native primitives
 and works in desktop, web, iOS, and Android Paseo clients.

package/client/browser.tsx

@@ -29,6 +29,8 @@ import {
   type BrowserInputEvent,
   type BrowserState,
   captureBrowserRpc,
+  containedRect,
+  createImageSizeLoader,
   DEVICE_PRESETS,
   type DevicePresetId,
   detachBrowserRpc,
@@ -41,7 +43,9 @@ import {
   releaseControlRpc,
   resizeBrowserRpc,
   sendBrowserInputRpc,
+  toViewportPoint,
 } from "../shared/browser";
+import { FrameLifecycle } from "./frame-lifecycle";
 
 const SPACE = {
   xxs: 2,
@@ -88,12 +92,7 @@ interface Size {
   height: number;
 }
 
-interface DisplayRect extends Size {
-  x: number;
-  y: number;
-}
-
-interface DisplayPoint extends Size {
+interface DisplayPoint {
   x: number;
   y: number;
 }
@@ -115,21 +114,6 @@ const SPECIAL_KEYS: readonly { key: SpecialKey; label: string }[] = [
   { key: "Space", label: "Space" },
 ];
 
-function containedRect(container: Size, image: Size): DisplayRect | null {
-  if (container.width <= 0 || container.height <= 0 || image.width <= 0 || image.height <= 0) {
-    return null;
-  }
-  const scale = Math.min(container.width / image.width, container.height / image.height);
-  const width = image.width * scale;
-  const height = image.height * scale;
-  return {
-    x: (container.width - width) / 2,
-    y: (container.height - height) / 2,
-    width,
-    height,
-  };
-}
-
 function errorMessage(error: unknown): string {
   if (error instanceof Error && error.message.trim()) return error.message;
   return "The shared browser request failed.";
@@ -1006,14 +990,15 @@ export function SharedBrowserPanel({
   const mountedRef = useRef(false);
   const activeViewerTokenRef = useRef<string | null>(null);
   const stateRef = useRef<BrowserState | null>(null);
-  const frameRef = useRef<BrowserFrame | null>(null);
+  const lifecycleRef = useRef(new FrameLifecycle());
+  const lifecycle = lifecycleRef.current;
   const lastPointRef = useRef<{ x: number; y: number } | null>(null);
   const dragStartRef = useRef<{ x: number; y: number } | null>(null);
-  const mutationEpochRef = useRef(0);
   const captureInFlightRef = useRef(false);
 
   const [state, setState] = useState<BrowserState | null>(null);
   const [frame, setFrame] = useState<BrowserFrame | null>(null);
+  const [inputBusy, setInputBusy] = useState(false);
   const [controlToken, setControlToken] = useState<string | null>(null);
   const [operationError, setOperationError] = useState<string | null>(null);
   const [imageError, setImageError] = useState(false);
@@ -1041,7 +1026,7 @@ export function SharedBrowserPanel({
   const acceptState = useCallback((next: BrowserState) => {
     const previous = stateRef.current;
     if (previous && !isBrowserStateCurrent(previous, next)) return false;
-    const currentFrame = frameRef.current;
+    const currentFrame = lifecycle.frame;
     if (previous && didBrowserRuntimeRestart(previous, next)) {
       setRuntimeNotice(
         "Browser restarted. The preserved viewer connection now targets the new runtime.",
@@ -1049,7 +1034,7 @@ export function SharedBrowserPanel({
       setControlToken(null);
     }
     if (currentFrame && !isFrameCurrent(currentFrame, next)) {
-      frameRef.current = null;
+      lifecycle.drop();
       setFrame(null);
       lastPointRef.current = null;
     }
@@ -1091,9 +1076,10 @@ export function SharedBrowserPanel({
 
   useEffect(() => {
     stateRef.current = null;
-    frameRef.current = null;
+    lifecycle.reset();
     setState(null);
     setFrame(null);
+    setInputBusy(false);
     setControlToken(null);
     setOperationError(null);
     setRuntimeNotice(null);
@@ -1103,8 +1089,8 @@ export function SharedBrowserPanel({
     queryKey: ["shared-browser", "capture", viewerToken],
     queryFn: async () => {
       if (!viewerToken) throw new Error("The browser viewer is not attached.");
-      const mutationEpoch = mutationEpochRef.current;
-      const knownFrame = frameRef.current;
+      const mutationEpoch = lifecycle.epoch;
+      const knownFrame = lifecycle.frame;
       captureInFlightRef.current = true;
       try {
         const result = await captureBrowser({
@@ -1130,17 +1116,14 @@ export function SharedBrowserPanel({
 
   useEffect(() => {
     const result = captureQuery.data;
-    if (
-      !result ||
-      result.mutationEpoch !== mutationEpochRef.current ||
-      !acceptState(result.state)
-    ) {
+    if (!result || !lifecycle.isCurrent(result.mutationEpoch) || !acceptState(result.state)) {
       return;
     }
     if (result.frame && isFrameCurrent(result.frame, result.state)) {
-      frameRef.current = result.frame;
+      lifecycle.accept(result.mutationEpoch, result.frame);
       setImageError(false);
-      setFrame(result.frame);
+      setFrame(lifecycle.visible);
+      setInputBusy(lifecycle.busy);
     }
   }, [acceptState, captureQuery.data]);
 
@@ -1229,6 +1212,9 @@ export function SharedBrowserPanel({
     onSuccess: (result) => mutationSucceeded(result.state),
     onError: mutationFailed,
   });
+  const settleInput = () => {
+    lifecycle.settle();
+  };
   const inputMutation = useMutation({
     mutationFn: sendBrowserInput,
     retry: false,
@@ -1237,9 +1223,13 @@ export function SharedBrowserPanel({
         const sentText = variables.event.text;
         setTypeDraft((current) => (current === sentText ? "" : current));
       }
+      settleInput();
       mutationSucceeded(result.state);
     },
-    onError: mutationFailed,
+    onError: (error) => {
+      settleInput();
+      mutationFailed(error);
+    },
   });
 
   const anyMutationPending =
@@ -1251,18 +1241,37 @@ export function SharedBrowserPanel({
     inputMutation.isPending;
   const canControl = Boolean(viewerToken && controlToken && state?.controller === "self");
   const currentFrame = frame && state && isFrameCurrent(frame, state) ? frame : null;
-  const canSendInput = canControl && Boolean(currentFrame) && !inputMutation.isPending;
+  const canSendInput =
+    canControl && Boolean(currentFrame) && !inputMutation.isPending && !inputBusy;
 
+  const frameUri = useMemo(
+    () => (currentFrame ? `data:${currentFrame.mimeType};base64,${currentFrame.dataBase64}` : null),
+    [currentFrame],
+  );
+  const [imageSize, setImageSize] = useState<{ uri: string; size: Size } | null>(null);
+  const loadImageSize = useMemo(
+    () =>
+      createImageSizeLoader(
+        (uri, success, failure) => Image.getSize(uri, success, failure),
+        (uri, size) => setImageSize(size ? { uri, size } : null),
+      ),
+    [],
+  );
+  useEffect(() => {
+    if (frameUri) loadImageSize(frameUri);
+  }, [frameUri, loadImageSize]);
+  // The bitmap can disagree with the frame metadata; its own aspect ratio decides the rectangle.
   const displayRect = useMemo(
     () =>
       currentFrame
-        ? containedRect(containerSize, { width: currentFrame.width, height: currentFrame.height })
+        ? containedRect(
+            containerSize,
+            imageSize?.uri === frameUri
+              ? imageSize.size
+              : { width: currentFrame.width, height: currentFrame.height },
+          )
         : null,
-    [containerSize, currentFrame],
-  );
-  const frameUri = useMemo(
-    () => (currentFrame ? `data:${currentFrame.mimeType};base64,${currentFrame.dataBase64}` : null),
-    [currentFrame],
+    [containerSize, currentFrame, frameUri, imageSize],
   );
 
   const controlContext = useCallback(() => {
@@ -1283,7 +1292,7 @@ export function SharedBrowserPanel({
   const inputContext = useCallback(() => {
     const context = controlContext();
     const current = stateRef.current;
-    const targetFrame = frameRef.current;
+    const targetFrame = lifecycle.frame;
     if (!context || !current || !targetFrame || !isFrameCurrent(targetFrame, current)) return null;
     return {
       ...context,
@@ -1316,12 +1325,13 @@ export function SharedBrowserPanel({
 
   const sendEvent = useCallback(
     (event: BrowserInputEvent) => {
+      if (lifecycle.busy) return;
       const context = requireInputContext();
-      if (!context || inputMutation.isPending) return;
-      mutationEpochRef.current += 1;
+      if (!context || !lifecycle.begin()) return;
+      setInputBusy(true);
       inputMutation.mutate({ ...context, event });
     },
-    [inputMutation, requireInputContext],
+    [inputMutation, lifecycle, requireInputContext],
   );
 
   const pointFromEvent = useCallback(
@@ -1329,7 +1339,7 @@ export function SharedBrowserPanel({
       if (!displayRect) return null;
       const x = Math.min(displayRect.width, Math.max(0, event.nativeEvent.locationX));
       const y = Math.min(displayRect.height, Math.max(0, event.nativeEvent.locationY));
-      return { x, y, width: displayRect.width, height: displayRect.height };
+      return { x, y };
     },
     [displayRect],
   );
@@ -1338,17 +1348,18 @@ export function SharedBrowserPanel({
     (event: GestureResponderEvent) => {
       const start = dragStartRef.current;
       const end = pointFromEvent(event);
+      const viewport = stateRef.current?.viewport;
       dragStartRef.current = null;
-      if (!start || !end) return;
+      if (!start || !end || !displayRect || !viewport) return;
+      const toViewport = (point: DisplayPoint) => toViewportPoint(point, displayRect, viewport);
       lastPointRef.current = { x: end.x, y: end.y };
       const distance = Math.hypot(end.x - start.x, end.y - start.y);
       if (distance >= DRAG_THRESHOLD) {
         if (swipeMode === "scroll") {
-          const current = stateRef.current;
-          const scale = current ? current.viewport.width / end.width : 1;
+          const scale = viewport.width / displayRect.width;
           sendEvent({
             kind: "scroll",
-            point: { ...start, width: end.width, height: end.height },
+            point: toViewport(start),
             deltaX: clampScrollDelta((start.x - end.x) * scale),
             deltaY: clampScrollDelta((start.y - end.y) * scale),
           });
@@ -1356,20 +1367,20 @@ export function SharedBrowserPanel({
         }
         sendEvent({
           kind: "drag",
-          start: { ...start, width: end.width, height: end.height },
-          end,
+          start: toViewport(start),
+          end: toViewport(end),
           button: interactionMode === "right" ? "right" : "left",
         });
         return;
       }
       sendEvent({
         kind: "click",
-        point: end,
+        point: toViewport(end),
         button: interactionMode === "right" ? "right" : "left",
         clickCount: interactionMode === "double" ? 2 : 1,
       });
     },
-    [interactionMode, pointFromEvent, sendEvent, swipeMode],
+    [displayRect, interactionMode, pointFromEvent, sendEvent, swipeMode],
   );
 
   const panResponder = useMemo(
@@ -1400,7 +1411,7 @@ export function SharedBrowserPanel({
   const takeControl = useCallback(
     (takeover: boolean) => {
       if (!viewerToken || acquireMutation.isPending) return;
-      mutationEpochRef.current += 1;
+      lifecycle.bump();
       acquireMutation.mutate({ viewerToken, takeover });
     },
     [acquireMutation, viewerToken],
@@ -1415,7 +1426,7 @@ export function SharedBrowserPanel({
     ) {
       return;
     }
-    mutationEpochRef.current += 1;
+    lifecycle.bump();
     releaseMutation.mutate({ viewerToken: viewer, controlToken });
   }, [controlToken, releaseMutation]);
 
@@ -1429,11 +1440,11 @@ export function SharedBrowserPanel({
           setOperationError("Enter an address to navigate.");
           return;
         }
-        mutationEpochRef.current += 1;
+        lifecycle.bump();
         navigateMutation.mutate({ ...context, action: { kind: "goto", url: nextUrl } });
         return;
       }
-      mutationEpochRef.current += 1;
+      lifecycle.bump();
       navigateMutation.mutate({ ...context, action: { kind: action } });
     },
     [navigateMutation, requireControlContext],
@@ -1457,7 +1468,7 @@ export function SharedBrowserPanel({
       );
       return;
     }
-    mutationEpochRef.current += 1;
+    lifecycle.bump();
     resizeMutation.mutate({ ...context, viewport: { width, height } });
   }, [requireControlContext, resizeMutation, viewportHeight, viewportWidth]);
 
@@ -1465,7 +1476,7 @@ export function SharedBrowserPanel({
     (presetId: DevicePresetId) => {
       const context = requireControlContext();
       if (!context || deviceMutation.isPending) return;
-      mutationEpochRef.current += 1;
+      lifecycle.bump();
       setDevicePickerOpen(false);
       deviceMutation.mutate({ ...context, presetId });
     },
@@ -1474,13 +1485,14 @@ export function SharedBrowserPanel({
 
   const scroll = useCallback(
     (deltaX: number, deltaY: number) => {
-      if (!displayRect) return;
+      const viewport = stateRef.current?.viewport;
+      if (!displayRect || !viewport) return;
       const previous = lastPointRef.current;
       const x = previous && previous.x <= displayRect.width ? previous.x : displayRect.width / 2;
       const y = previous && previous.y <= displayRect.height ? previous.y : displayRect.height / 2;
       sendEvent({
         kind: "scroll",
-        point: { x, y, width: displayRect.width, height: displayRect.height },
+        point: toViewportPoint({ x, y }, displayRect, viewport),
         deltaX,
         deltaY,
       });
@@ -1495,7 +1507,7 @@ export function SharedBrowserPanel({
 
   const reconnect = useCallback(() => {
     if (attachQuery.isFetching) return;
-    mutationEpochRef.current += 1;
+    lifecycle.bump();
     setReconnecting(true);
     setControlToken(null);
     setOperationError(null);
@@ -1700,7 +1712,7 @@ export function SharedBrowserPanel({
                 }
                 accessibilityRole="image"
                 onError={() => {
-                  frameRef.current = null;
+                  lifecycle.frame = null;
                   setImageError(true);
                   void captureQuery.refetch({ cancelRefetch: false });
                 }}

package/client/frame-lifecycle.ts

@@ -0,0 +1,65 @@
+import type { BrowserFrame } from "../shared/browser";
+
+/**
+ * Tracks which captured frame input may target. After an input is submitted
+ * and again when it settles, the actionable frame is dropped and the epoch
+ * advances, so captures that began earlier are ignored. Input stays blocked
+ * until a capture taken after settlement is accepted. The last accepted image
+ * stays visible meanwhile.
+ */
+export class FrameLifecycle {
+  epoch = 0;
+  /** Frame input may target; null while a fresh capture is awaited. */
+  frame: BrowserFrame | null = null;
+  /** Last accepted frame, kept on screen while input is blocked. */
+  visible: BrowserFrame | null = null;
+  private gate: "idle" | "pending" | "refreshing" = "idle";
+
+  get busy(): boolean {
+    return this.gate !== "idle";
+  }
+
+  bump(): void {
+    this.epoch += 1;
+  }
+
+  isCurrent(captureEpoch: number): boolean {
+    return captureEpoch === this.epoch;
+  }
+
+  /** Claims the input slot; false if an input is already in flight or awaiting a fresh frame. */
+  begin(): boolean {
+    if (this.busy) return false;
+    this.gate = "pending";
+    this.frame = null;
+    this.bump();
+    return true;
+  }
+
+  /** Called when an input succeeds or fails. Never replays the action. */
+  settle(): void {
+    this.frame = null;
+    this.bump();
+    this.gate = "refreshing";
+  }
+
+  /** Accepts a capture started at `captureEpoch`; false if it predates the latest boundary. */
+  accept(captureEpoch: number, frame: BrowserFrame): boolean {
+    if (!this.isCurrent(captureEpoch)) return false;
+    this.frame = frame;
+    this.visible = frame;
+    if (this.gate === "refreshing") this.gate = "idle";
+    return true;
+  }
+
+  /** Forgets all frames, e.g. when the session no longer matches them. */
+  drop(): void {
+    this.frame = null;
+    this.visible = null;
+  }
+
+  reset(): void {
+    this.drop();
+    this.gate = "idle";
+  }
+}

package/client/sidebar.tsx

@@ -0,0 +1,134 @@
+import {
+  type PluginClientContext,
+  type PluginPopoverProps,
+  type PluginSidebarItemProps,
+  useRpc,
+  useWorkspace,
+} from "@getpaseo/plugin/client";
+import { SidebarRow } from "@getpaseo/plugin/client/ui";
+import { useQuery } from "@tanstack/react-query";
+import { useMemo } from "react";
+import { Pressable, StyleSheet, Text, View } from "react-native";
+import { listOpenBrowserWorkspacesRpc } from "../shared/browser";
+
+const PANEL_ID = "shared-browser";
+const PRESENCE_POLL_MS = 5_000;
+const TOUCH_TARGET = 44;
+const ROW_HEIGHT = 36;
+const RADIUS = 8;
+
+function useOpenBrowserWorkspaceIds(): readonly string[] {
+  const listOpen = useRpc(listOpenBrowserWorkspacesRpc);
+  const query = useQuery({
+    queryKey: ["shared-browser", "presence"],
+    queryFn: async () => (await listOpen({})).workspaceIds,
+    refetchInterval: PRESENCE_POLL_MS,
+    retry: false,
+  });
+  return query.data ?? [];
+}
+
+function SessionRow({
+  workspaceId,
+  theme,
+  compact,
+  onPress,
+}: {
+  workspaceId: string;
+  theme: PluginPopoverProps["theme"];
+  compact: boolean;
+  onPress(): void;
+}) {
+  const name = useWorkspace(workspaceId, (workspace) => workspace.title ?? workspace.name);
+  const styles = useMemo(
+    () =>
+      StyleSheet.create({
+        row: {
+          minHeight: compact ? TOUCH_TARGET : ROW_HEIGHT,
+          justifyContent: "center",
+          paddingHorizontal: 12,
+          borderRadius: RADIUS,
+        },
+        label: { color: theme.colors.foreground, fontSize: 14 },
+      }),
+    [theme, compact],
+  );
+  // A workspace this client has not loaded cannot host the panel; openPanel would throw.
+  if (name === null) return null;
+  return (
+    <Pressable
+      accessibilityRole="button"
+      accessibilityLabel={`Open Shared Browser for ${name}`}
+      onPress={onPress}
+      style={({ pressed }) => [
+        styles.row,
+        pressed ? { backgroundColor: theme.colors.surface1 } : null,
+      ]}
+    >
+      <Text numberOfLines={1} style={styles.label}>
+        {name}
+      </Text>
+    </Pressable>
+  );
+}
+
+function createSessionsPopover(client: PluginClientContext) {
+  return function SessionsPopover({ theme, layout, close }: PluginPopoverProps) {
+    const workspaceIds = useOpenBrowserWorkspaceIds();
+    const styles = useMemo(
+      () =>
+        StyleSheet.create({
+          container: { padding: layout.compact ? 16 : 8, gap: 4, minWidth: 220 },
+          title: {
+            color: theme.colors.foregroundMuted,
+            fontSize: 12,
+            paddingHorizontal: 12,
+            paddingBottom: 4,
+          },
+        }),
+      [theme, layout.compact],
+    );
+    return (
+      <View style={styles.container}>
+        <Text style={styles.title}>Open browser sessions</Text>
+        {workspaceIds.map((workspaceId) => (
+          <SessionRow
+            key={workspaceId}
+            workspaceId={workspaceId}
+            theme={theme}
+            compact={layout.compact}
+            onPress={() => {
+              close();
+              client.openPanel(PANEL_ID, { workspaceId });
+            }}
+          />
+        ))}
+      </View>
+    );
+  };
+}
+
+function createSidebarItem(client: PluginClientContext) {
+  const SessionsPopover = createSessionsPopover(client);
+  return function SharedBrowserSidebarItem({ openPopover }: PluginSidebarItemProps) {
+    const count = useOpenBrowserWorkspaceIds().length;
+    if (count === 0) return null;
+    return (
+      <SidebarRow
+        icon="PanelsTopLeft"
+        label={`Shared Browser (${count})`}
+        onPress={() => openPopover(SessionsPopover)}
+      />
+    );
+  };
+}
+
+/** Sidebar footer row listing open sessions. Hosts before 0.11 lack the API and get no row. */
+export function contributeSharedBrowserSidebar(client: PluginClientContext): () => void {
+  if (typeof client.addSidebarFooterItem !== "function") return () => {};
+  return client.addSidebarFooterItem({
+    id: "shared-browser-sessions",
+    title: "Shared Browser",
+    Component: createSidebarItem(client),
+  });
+}

package/index.client.tsx

@@ -1,5 +1,6 @@
 import type { PluginClientContext } from "@getpaseo/plugin/client";
 import { contributeSharedBrowserClient, SharedBrowserPanel } from "./client/browser";
+import { contributeSharedBrowserSidebar } from "./client/sidebar";
 
 export default function contribute(client: PluginClientContext) {
   client.addWorkspacePanel({
@@ -31,5 +32,10 @@ export default function contribute(client: PluginClientContext) {
     },
   });
 
-  return contributeSharedBrowserClient(client);
+  const removeSidebar = contributeSharedBrowserSidebar(client);
+  const cleanup = contributeSharedBrowserClient(client);
+  return () => {
+    removeSidebar();
+    return cleanup();
+  };
 }

package/index.server.ts

@@ -77,8 +77,12 @@ export default function contribute(server: PluginServerContext) {
   server.handle(resizeBrowserRpc, handleResizeBrowser);
   server.handle(applyDevicePresetRpc, handleApplyDevicePreset);
   server.handle(sendBrowserInputRpc, handleSendBrowserInput);
+  // OMP accepts session MCP servers from Paseo 0.11.0-beta.1, the release that added
+  // registerUsageSource; 0.9/0.10 reject agent creation when an OMP config carries MCP servers.
+  const ompAcceptsMcp = typeof server.registerUsageSource === "function";
   server.before("agent.create", async ({ request }) => {
-    if (request.config.internal || request.config.provider === "omp") return request;
+    if (request.config.internal) return request;
+    if (request.config.provider === "omp" && !ompAcceptsMcp) return request;
     const ticket = randomBytes(32).toString("base64url");
     if (!(await issueTicketWithinDeadline(ticket))) return request;
     return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@omercnet/paseo-shared-browser",
-  "version": "1.1.0",
+  "version": "1.2.0-next.214.1",
   "description": "One real Chromium session per Paseo workspace, shared live across every connected client.",
   "keywords": [
     "browser",
@@ -43,12 +43,12 @@
     "check:write": "biome check --write .",
     "typecheck": "tsc --noEmit",
     "prepare:runtime": "node scripts/prepare-runtime.mjs",
-    "test": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts",
-    "test:unit": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts",
+    "test": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts",
+    "test:unit": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts",
     "test:smoke": "vitest run tests/browser.smoke.test.ts --testTimeout=30000"
   },
   "dependencies": {
-    "@getpaseo/plugin": "0.10.1",
+    "@getpaseo/plugin": "0.11.0-beta.3",
     "@modelcontextprotocol/sdk": "1.30.0",
     "agent-browser": "0.37.1",
     "esbuild": "0.28.2",

package/paseo-plugin.json

@@ -1,6 +1,6 @@
 {
   "id": "shared-browser",
   "description": "One real Chromium session per Paseo workspace, shared live across every connected client.",
-  "requirements": { "paseo": "^0.9.0 || ^0.10.0" },
+  "requirements": { "paseo": "^0.9.0 || ^0.10.0 || ^0.11.0" },
   "build": [["node", "scripts/prepare-dependencies.mjs"], ["npm", "run", "prepare:runtime"]]
 }

package/server/agent-browser-runtime.ts

@@ -183,6 +183,26 @@ function findString(value: unknown, keys: readonly string[]): string | null {
   return null;
 }
 
+export function jpegDimensions(base64: string): { width: number; height: number } | null {
+  const bytes = Buffer.from(base64, "base64");
+  let offset = 2;
+  while (bytes[0] === 0xff && bytes[1] === 0xd8 && offset + 9 <= bytes.length) {
+    if (bytes[offset] !== 0xff) return null;
+    const marker = bytes[offset + 1];
+    // SOFn carries the frame size; DHT (c4), JPG (c8) and DAC (cc) share the range but do not.
+    if (
+      marker !== undefined &&
+      marker >= 0xc0 &&
+      marker <= 0xcf &&
+      ![0xc4, 0xc8, 0xcc].includes(marker)
+    )
+      return { width: bytes.readUInt16BE(offset + 7), height: bytes.readUInt16BE(offset + 5) };
+    if (marker === 0xda) return null;
+    offset += 2 + bytes.readUInt16BE(offset + 2);
+  }
+  return null;
+}
+
 export class AgentBrowserRuntime {
   readonly binaryPath: string;
   readonly executablePath: string;
@@ -318,7 +338,7 @@ export class AgentBrowserRuntime {
     await this.requireConnection().send("Target.activateTarget", { targetId }, { mutation: true });
     this.page = await attachToTarget(this.requireConnection(), targetId);
     this.targetId = targetId;
-    await this.bindPageEvents(this.page);
+    await this.preparePage(this.page);
     if (this.screencastActive) await this.startScreencastSession(this.page);
   }
 
@@ -425,34 +445,7 @@ export class AgentBrowserRuntime {
   async emulate(device: DeviceEmulation): Promise<void> {
     this.assertViewport(device);
     await this.withInvalidatedScreencast(async (page) => {
-      await page.send(
-        "Emulation.setDeviceMetricsOverride",
-        {
-          width: device.width,
-          height: device.height,
-          deviceScaleFactor: device.deviceScaleFactor,
-          mobile: device.mobile,
-          screenWidth: device.screenWidth ?? device.width,
-          screenHeight: device.screenHeight ?? device.height,
-          screenOrientation: {
-            type: device.width > device.height ? "landscapePrimary" : "portraitPrimary",
-            angle: device.width > device.height ? 90 : 0,
-          },
-        },
-        { mutation: true },
-      );
-      await page.send(
-        "Emulation.setTouchEmulationEnabled",
-        { enabled: device.touch, maxTouchPoints: device.touch ? 5 : 1 },
-        { mutation: true },
-      );
-      if (device.userAgent) {
-        await page.send(
-          "Emulation.setUserAgentOverride",
-          { userAgent: device.userAgent, platform: device.platform ?? "" },
-          { mutation: true },
-        );
-      }
+      await this.applyEmulation(page, device);
       this.viewport = { ...device };
     });
   }
@@ -483,7 +476,17 @@ export class AgentBrowserRuntime {
       throw new RangeError("maxBytes must be positive");
     const streamed = this.screencastFrame ?? (await this.waitForFrame(waitMs));
     if (streamed && streamed.byteLength <= maxBytes) return streamed;
-    const page = await this.requirePage();
+    let frame = await this.screenshot(await this.requirePage(), maxBytes, quality);
+    if (!this.sizeMismatch(frame)) return frame;
+    // The browser stopped honouring the cached viewport. Re-apply it once; never loop.
+    await this.emulate(this.requireViewport());
+    frame = await this.screenshot(await this.requirePage(), maxBytes, quality);
+    const mismatch = this.sizeMismatch(frame);
+    if (mismatch) throw new Error(`Browser returned a ${mismatch} after viewport resync`);
+    return frame;
+  }
+
+  private async screenshot(page: CdpSession, maxBytes: number, quality: number) {
     for (const candidate of [quality, 50, 35, 20, 10, 1]) {
       const boundedQuality = Math.max(1, Math.min(100, Math.round(candidate)));
       const result = await page.send<{ data: string }>("Page.captureScreenshot", {
@@ -502,12 +505,25 @@ export class AgentBrowserRuntime {
           height: viewport.height,
           transport: "screenshot",
           capturedAt: new Date().toISOString(),
-        };
+        } satisfies RuntimeFrame;
       }
     }
     throw new Error(`JPEG screenshot exceeds ${maxBytes} bytes at minimum quality`);
   }
 
+  private sizeMismatch(frame: RuntimeFrame): string | null {
+    const { width, height, deviceScaleFactor: scale } = this.requireViewport();
+    const size = jpegDimensions(frame.dataBase64);
+    // Chromium rounds fractional device pixels, so allow one pixel of slack.
+    if (
+      size &&
+      Math.abs(size.width - width * scale) <= 1 &&
+      Math.abs(size.height - height * scale) <= 1
+    )
+      return null;
+    return `${size ? `${size.width}x${size.height}` : "unreadable"} JPEG for a ${width}x${height} viewport at ${scale}x`;
+  }
+
   async mouseMove(x: number, y: number): Promise<void> {
     this.assertPoint(x, y);
     await (await this.requirePage()).send(
@@ -786,13 +802,46 @@ export class AgentBrowserRuntime {
     return this.viewport;
   }
 
-  private async bindPageEvents(page: CdpSession): Promise<void> {
+  private async preparePage(page: CdpSession): Promise<void> {
     page.on("Page.screencastFrame", this.onScreencastFrame);
     page.on("event", (event: CdpEvent) => {
       if (event.method === "Inspector.targetCrashed") this.invalidateScreencastFrame();
     });
     await Promise.all([page.send("Page.enable"), page.send("Runtime.enable")]);
     await this.navigationHistory(page);
+    // Emulation is scoped to the CDP session, so every fresh attach starts without it.
+    if (this.viewport) await this.applyEmulation(page, this.viewport);
+  }
+
+  private async applyEmulation(page: CdpSession, device: DeviceEmulation): Promise<void> {
+    await page.send(
+      "Emulation.setDeviceMetricsOverride",
+      {
+        width: device.width,
+        height: device.height,
+        deviceScaleFactor: device.deviceScaleFactor,
+        mobile: device.mobile,
+        screenWidth: device.screenWidth ?? device.width,
+        screenHeight: device.screenHeight ?? device.height,
+        screenOrientation: {
+          type: device.width > device.height ? "landscapePrimary" : "portraitPrimary",
+          angle: device.width > device.height ? 90 : 0,
+        },
+      },
+      { mutation: true },
+    );
+    await page.send(
+      "Emulation.setTouchEmulationEnabled",
+      { enabled: device.touch, maxTouchPoints: device.touch ? 5 : 1 },
+      { mutation: true },
+    );
+    if (device.userAgent) {
+      await page.send(
+        "Emulation.setUserAgentOverride",
+        { userAgent: device.userAgent, platform: device.platform ?? "" },
+        { mutation: true },
+      );
+    }
   }
 
   private async startScreencastSession(page: CdpSession): Promise<void> {
@@ -837,7 +886,7 @@ export class AgentBrowserRuntime {
       { mutation: true },
     );
     const replacement = await attachToTarget(connection, target.targetId);
-    await this.bindPageEvents(replacement);
+    await this.preparePage(replacement);
     if (this.page === previous) {
       this.page = replacement;
       this.targetId = target.targetId;

package/shared/browser.ts

@@ -365,3 +365,70 @@ export function mapDisplayedPoint(
     y: Math.min(viewport.height - 1, Math.max(0, (point.y / point.height) * viewport.height)),
   };
 }
+
+export interface FrameSize {
+  width: number;
+  height: number;
+}
+
+export interface FrameRect extends FrameSize {
+  x: number;
+  y: number;
+}
+
+export function containedRect(container: FrameSize, image: FrameSize): FrameRect | null {
+  if (container.width <= 0 || container.height <= 0 || image.width <= 0 || image.height <= 0) {
+    return null;
+  }
+  const scale = Math.min(container.width / image.width, container.height / image.height);
+  const width = image.width * scale;
+  const height = image.height * scale;
+  return {
+    x: (container.width - width) / 2,
+    y: (container.height - height) / 2,
+    width,
+    height,
+  };
+}
+
+/** Converts a point in the displayed image to the canonical viewport, scaling both axes by viewport.width / rect.width. */
+export function toViewportPoint(
+  point: MappedPoint,
+  rect: FrameSize,
+  viewport: Viewport,
+): MappedPoint & FrameSize {
+  const scale = viewport.width / rect.width;
+  return {
+    x: point.x * scale,
+    y: Math.min(viewport.height, point.y * scale),
+    width: viewport.width,
+    height: viewport.height,
+  };
+}
+
+type GetImageSize = (
+  uri: string,
+  success: (width: number, height: number) => void,
+  failure: () => void,
+) => void;
+
+/** Looks up the intrinsic size once per distinct URI and drops results superseded by a newer URI. */
+export function createImageSizeLoader(
+  getSize: GetImageSize,
+  onSize: (uri: string, size: FrameSize | null) => void,
+): (uri: string) => void {
+  let current: string | null = null;
+  return (uri) => {
+    if (uri === current) return;
+    current = uri;
+    getSize(
+      uri,
+      (width, height) => {
+        if (current === uri) onSize(uri, { width, height });
+      },
+      () => {
+        if (current === uri) onSize(uri, null);
+      },
+    );
+  };
+}