@omercnet/paseo-shared-browser 1.2.2-next.239.1 → 1.2.2-next.240.1

package/STREAMING_DESIGN.md

@@ -69,7 +69,12 @@ admit clicks indefinitely. Motion, releases and keys of a held gesture use the o
 channel without waiting for new pixels. Agent commands retain strict fresh-frame
 targeting.
 
-A visible frame is not automatically an actionable receipt. Delayed reads may
+A visible frame is not automatically an actionable receipt. The host flags each
+video packet `actionable` or not (a capture older than the last acknowledged input
+is painted but never a receipt), and the viewer starts a new epoch when the host
+acknowledges a receipt-revoking input, so reads answered before that revocation
+cannot authorize the next press. After an acknowledged press the viewer has no
+actionable video receipt until a newer packet paints; a following press waits for it. Delayed reads may
 continue painting within the same source and geometry while retaining their
 original input revision. Initial-admission waiters wake only for actionable
 pixels. A document or viewport transition, controller loss, cancellation,

package/client/browser-input-queue.test.ts

@@ -60,6 +60,7 @@ function fixture(
   const errors: unknown[] = [];
   const cursors: unknown[] = [];
   const navigationCompleted: boolean[] = [];
+  const acknowledgements: string[] = [];
   const states: BrowserState[] = [];
   let projectState: ((state: BrowserState) => void) | null = null;
   let finished = 0;
@@ -94,6 +95,7 @@ function fixture(
       finished += 1;
     },
     onNavigationComplete: () => navigationCompleted.push(true),
+    onAcknowledged: () => acknowledgements.push("ack"),
     ...(waitForFrame ? { waitForFrame } : {}),
   });
   return {
@@ -105,6 +107,7 @@ function fixture(
     beginGate,
     transport,
     navigationCompleted,
+    acknowledgements,
     states,
     finishes: () => finished,
     projectState(callback: (state: BrowserState) => void) {
@@ -1062,6 +1065,56 @@ it.each(["viewer", "controller", "session", "runtime", "bridge", "document", "vi
   },
 );
 
+it("reports host acknowledgement of every receipt-revoking input, never plain hover", async () => {
+  const f = fixture(undefined, true);
+  f.queue.enqueue(move(1));
+  f.beginGate.resolve({ state, gestureId: "opaque", nextSequence: 1 });
+  await flush();
+  f.queue.enqueue(move(2));
+  await flush();
+  expect(f.acknowledgements).toEqual([]);
+  f.queue.enqueue({ kind: "down", button: "left", point: point(2), clickCount: 1 });
+  await flush();
+  expect(f.acknowledgements).toEqual(["ack"]);
+  f.queue.enqueue(move(3)); // a held drag keeps revoking
+  f.queue.enqueue({ kind: "up", button: "left", point: point(3), clickCount: 1 });
+  await flush();
+  expect(f.acknowledgements).toHaveLength(3);
+  f.queue.enqueue({ kind: "scroll", point: point(3), deltaX: 0, deltaY: 5 });
+  await flush();
+  expect(f.acknowledgements).toHaveLength(4);
+  f.queue.cancel();
+});
+
+it("waits for the next decoded frame when video revoked its receipt, then presses with that frame", async () => {
+  // Video input authority disappears (null) right after an acknowledged press and
+  // returns only when the next packet paints; the queue must wait, not refuse.
+  let waited = 0;
+  const f: ReturnType<typeof fixture> = fixture(async (afterFrameId) => {
+    waited += 1;
+    expect(afterFrameId).toBe(authority.target.frameId);
+    f.changeAuthority(freshAuthority);
+  }, true);
+  f.queue.enqueue({ kind: "down", button: "left", point: point(1), clickCount: 1 });
+  f.beginGate.resolve({ state, gestureId: "opaque", nextSequence: 1 });
+  await flush();
+  f.queue.enqueue({ kind: "up", button: "left", point: point(1), clickCount: 1 });
+  await flush();
+  f.changeAuthority(null);
+  expect(f.queue.enqueue({ kind: "down", button: "left", point: point(2), clickCount: 1 })).toBe(
+    true,
+  );
+  await flush();
+  expect(waited).toBe(1);
+  expect(f.errors).toEqual([]);
+  expect(f.sent.map((input) => input.event.kind)).toEqual(["down", "up", "down"]);
+  expect(f.sent.map((input) => input.target?.frameId)).toEqual([
+    authority.target.frameId,
+    undefined,
+    freshAuthority.target.frameId,
+  ]);
+  f.queue.cancel();
+});
 it("gives each independent touch contact its own decoded receipt; fingers within a gesture and text reuse the channel", async () => {
   const f: ReturnType<typeof fixture> = fixture(async () => {
     f.changeAuthority(freshAuthority);

package/client/browser-input-queue.ts

@@ -52,6 +52,13 @@ interface QueueOptions {
   onFinish(): void;
   /** Reset local held contacts after an acknowledged navigation, without cancelling this queue again. */
   onNavigationComplete?(): void;
+  /**
+   * Called after the host acknowledges a non-hover event. Receipts decoded from
+   * anything requested before this point predate the input's effect, so the viewer
+   * must stop treating them as actionable even if they were requested after the
+   * local press began.
+   */
+  onAcknowledged?(): void;
   /** Resolves only after a different actionable decoded receipt commits, never visual-only pixels or a timer. */
   waitForFrame?(afterFrameId: string, maxWaitMs?: number): Promise<void>;
 }
@@ -336,11 +343,17 @@ export function createBrowserInputQueue(options: QueueOptions) {
         let pressTarget: BrowserGestureAuthority["target"] | undefined;
         if (needsFreshPress(command.event, owned)) {
           let fresh = options.authority();
-          if (fresh && invalidatedFrameId === fresh.target.frameId && options.waitForFrame) {
-            // Earlier input revoked the visible receipt. Wait a bounded time for a
-            // newer decoded one; elapsed time alone never admits a press, and the
+          // Video revokes its receipt as soon as input is acknowledged, leaving no
+          // actionable authority until the next frame paints; JPEG keeps the spent
+          // frame visible. Both mean this press needs a newer decoded receipt.
+          const visibleId = fresh?.target.frameId ?? owned.authority.target.frameId;
+          const spent = fresh
+            ? invalidatedFrameId === fresh.target.frameId
+            : invalidatedFrameId !== null;
+          if (spent && options.waitForFrame) {
+            // Wait a bounded time; elapsed time alone never admits a press, and the
             // server refuses a spent or expired receipt regardless.
-            await options.waitForFrame(fresh.target.frameId);
+            await options.waitForFrame(visibleId);
             if (currentEpoch !== epoch) break;
             fresh = options.authority();
           }
@@ -349,7 +362,7 @@ export function createBrowserInputQueue(options: QueueOptions) {
               throw new Error("Browser input context changed. Release the gesture and try again.");
             }
             pressTarget = fresh.target;
-          } else if (owned.authority.target.frameId !== invalidatedFrameId) {
+          } else if (!spent) {
             // Display was revoked after admission but before this first press; the
             // receipt begin just validated is still unspent. The server re-checks it.
             pressTarget = owned.authority.target;
@@ -367,10 +380,10 @@ export function createBrowserInputQueue(options: QueueOptions) {
         if (command.event.kind === "down") owned.heldButtons.add(command.event.button);
         if (command.event.kind === "up") owned.heldButtons.delete(command.event.button);
         // Mirror the server: everything except hover revokes the visible receipt.
-        if (
+        const revokesReceipts =
           (command.event.kind !== "move" && command.event.kind !== "leave") ||
-          owned.heldButtons.size > 0
-        ) {
+          owned.heldButtons.size > 0;
+        if (revokesReceipts) {
           invalidatedFrameId =
             options.authority()?.target.frameId ?? owned.authority.target.frameId;
         }
@@ -413,6 +426,7 @@ export function createBrowserInputQueue(options: QueueOptions) {
         if (!sameAuthority(owned.authority, currentControl())) {
           throw new Error("Browser input context changed. Release the gesture and try again.");
         }
+        if (revokesReceipts) options.onAcknowledged?.();
         options.onCursor(result.cursor);
       }
     } catch (error) {

package/client/browser-video-currency.ts

@@ -0,0 +1,45 @@
+/** Source, document and geometry validity of a viewer frame, shared by the panel and its tests. */
+import type { BrowserState } from "../shared/browser";
+import type { BrowserFrameAuthority, BrowserVideoPacket } from "../shared/browser-video";
+
+export function isFrameCurrent(frame: BrowserFrameAuthority, state: BrowserState): boolean {
+  return (
+    frame.sessionId === state.sessionId &&
+    (!frame.runtimeId || !state.runtimeId || frame.runtimeId === state.runtimeId) &&
+    (frame.captureEpoch === undefined ||
+      state.bridgeEpoch === undefined ||
+      frame.captureEpoch === state.bridgeEpoch) &&
+    frame.navigationGeneration === state.navigationGeneration &&
+    frame.viewportGeneration === state.viewportGeneration
+  );
+}
+
+export interface VideoSourceContext {
+  mounted: boolean;
+  viewerActive: boolean;
+  epoch: number;
+  committedViewport: { sessionId: string; generation: number } | null;
+  state: BrowserState | null;
+}
+
+/**
+ * Whether a packet may be decoded and painted. This is source validity only: a
+ * display-only packet (actionable === false) is still a link in the codec's delta
+ * chain, so the input-only flag must never be consulted here. Input admission is
+ * the video hook's separate inputCurrent guard.
+ */
+export function isVideoSourceCurrent(
+  context: VideoSourceContext,
+  packet: BrowserVideoPacket,
+  epoch: number,
+): boolean {
+  return (
+    context.mounted &&
+    context.viewerActive &&
+    epoch === context.epoch &&
+    context.committedViewport?.sessionId === packet.frame.sessionId &&
+    context.committedViewport?.generation === packet.frame.viewportGeneration &&
+    context.state !== null &&
+    isFrameCurrent(packet.frame, context.state)
+  );
+}

package/client/browser.tsx

@@ -84,6 +84,7 @@ import {
   BrowserMenuSeparator,
   BrowserToolbarMenu,
 } from "./browser-toolbar-menu";
+import { isFrameCurrent, isVideoSourceCurrent } from "./browser-video-currency";
 import { BrowserVideoSurface } from "./browser-video-surface";
 import { isBrowserClosedError, isExpiredBrowserViewerError } from "./browser-viewer-recovery";
 import { createFrameLifecycle } from "./frame-lifecycle";
@@ -182,18 +183,6 @@ function isFrameDisplayCurrent(frame: BrowserFrameAuthority, state: BrowserState
   );
 }
 
-function isFrameCurrent(frame: BrowserFrameAuthority, state: BrowserState): boolean {
-  return (
-    frame.sessionId === state.sessionId &&
-    (!frame.runtimeId || !state.runtimeId || frame.runtimeId === state.runtimeId) &&
-    (frame.captureEpoch === undefined ||
-      state.bridgeEpoch === undefined ||
-      frame.captureEpoch === state.bridgeEpoch) &&
-    frame.navigationGeneration === state.navigationGeneration &&
-    frame.viewportGeneration === state.viewportGeneration
-  );
-}
-
 export function SharedBrowserPanel({
   theme,
   host,
@@ -505,13 +494,17 @@ export function SharedBrowserPanel({
     epoch: () => inputLifecycle.epoch,
     viewport: () => stateRef.current?.viewport ?? null,
     isCurrent: (packet, epoch) =>
-      mountedRef.current &&
-      activeViewerTokenRef.current === viewerToken &&
-      epoch === inputLifecycle.epoch &&
-      committedViewport.current?.sessionId === packet.frame.sessionId &&
-      committedViewport.current?.generation === packet.frame.viewportGeneration &&
-      stateRef.current !== null &&
-      isFrameCurrent(packet.frame, stateRef.current),
+      isVideoSourceCurrent(
+        {
+          mounted: mountedRef.current,
+          viewerActive: activeViewerTokenRef.current === viewerToken,
+          epoch: inputLifecycle.epoch,
+          committedViewport: committedViewport.current,
+          state: stateRef.current,
+        },
+        packet,
+        epoch,
+      ),
     isDisplayCurrent: (packet) => {
       const current = stateRef.current;
       return (
@@ -962,6 +955,11 @@ export function SharedBrowserPanel({
       lastPointRef.current = { x: point.x, y: point.y };
     },
     onActivity: handleInputActivity,
+    onInputAcknowledged: () => {
+      // Pixels requested before the host acknowledged this input cannot authorize
+      // the next press, even when their request began after the local press.
+      inputLifecycle.bump();
+    },
     onInputBoundary: () => {
       inputLifecycle.bump();
     },

package/client/use-browser-canvas-input.ts

@@ -45,6 +45,8 @@ interface CanvasOptions {
   onPoint(point: BrowserGesturePoint): void;
   onActivity(active: boolean): void;
   onInputBoundary(): void;
+  /** The host acknowledged an input that revoked every earlier frame receipt. */
+  onInputAcknowledged?(): void;
 }
 
 /** Native touch coordinates are relative to the controlled overlay, with stable ID mapping. */
@@ -120,6 +122,9 @@ export function useBrowserCanvasInput(options: CanvasOptions) {
         clearIdle();
         if (alive.current) current.current.onError(error);
       },
+      onAcknowledged: () => {
+        if (alive.current) current.current.onInputAcknowledged?.();
+      },
       onNavigationComplete: () => {
         // Server already closed the acknowledged channel. Quarantine local
         // holds before state projection; never replay their old-page release.

package/client/use-browser-video.test.ts

@@ -1,10 +1,12 @@
 /** Deliberately defer React commits to exercise canvas paint and RPC replacement windows. */
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { BrowserState } from "../shared/browser";
 import type {
   BrowserVideoPacket,
   BrowserVideoReadInput,
   BrowserVideoReadReply,
 } from "../shared/browser-video";
+import { isVideoSourceCurrent } from "./browser-video-currency";
 import type { BrowserVideoDecodeEnvironment } from "./browser-video-decoder";
 
 interface HookSlot {
@@ -211,11 +213,30 @@ function setup(retainDocumentDisplay = false) {
       fps,
       epoch: () => mutationEpoch,
       viewport: () => ({ width: 1280, height: 800 }),
+      // The mounted panel's real source predicate, fed a fake panel context, so a drift
+      // in what the panel decides (for example consulting the input-only actionable flag)
+      // fails here instead of in a copied predicate.
       isCurrent: (value, epoch) =>
         current &&
-        epoch === mutationEpoch &&
-        Object.entries(identity).every(
-          ([key, expected]) => value.frame[key as keyof typeof identity] === expected,
+        isVideoSourceCurrent(
+          {
+            mounted: true,
+            viewerActive: true,
+            epoch: mutationEpoch,
+            committedViewport: {
+              sessionId: identity.sessionId,
+              generation: identity.viewportGeneration,
+            },
+            state: {
+              sessionId: identity.sessionId,
+              runtimeId: identity.runtimeId,
+              bridgeEpoch: identity.captureEpoch,
+              navigationGeneration: identity.navigationGeneration,
+              viewportGeneration: identity.viewportGeneration,
+            } as unknown as BrowserState,
+          },
+          value,
+          epoch,
         ),
       ...(retainDocumentDisplay
         ? {
@@ -278,6 +299,19 @@ function setup(retainDocumentDisplay = false) {
         packets: [{ ...packet(sequence), type: keyFrame ? "key" : "delta" }],
       } as BrowserVideoReadReply);
     },
+    replyPackets(index: number, sequences: number[], display: number[] = []) {
+      harness.reads[index]!.resolve({
+        status: "ready",
+        streamId: "s".repeat(32),
+        state: {},
+        packets: sequences.map((sequence) => ({
+          ...packet(sequence),
+          type: sequence === 1 ? "key" : "delta",
+          // The host paints these but refuses them as input receipts.
+          ...(display.includes(sequence) ? { actionable: false } : { actionable: true }),
+        })),
+      } as BrowserVideoReadReply);
+    },
     disabled(index: number) {
       harness.reads[index]!.resolve({
         status: "unsupported",
@@ -521,6 +555,41 @@ describe("video presentation lifecycle", () => {
     expect(result.error).toBeNull();
     expect(result.frontRef.current).not.toBeNull();
   });
+  it("decodes and paints a same-source display-only packet without breaking the delta chain or granting input", async () => {
+    const f = setup();
+    const front: (number | null)[] = [];
+    for (const sequence of [1, 2, 3]) {
+      f.replyPackets(sequence - 1, [sequence], [2]);
+      await drain();
+      f.paint();
+      const result = f.render();
+      f.commit();
+      front.push(result.frontRef.current?.sequence ?? null);
+      await vi.advanceTimersByTimeAsync(41);
+    }
+    // All three packets paint, so the codec chain 1 -> 2 -> 3 is intact.
+    expect(f.drawn).toEqual([1000, 2000, 3000]);
+    // The display-only packet never publishes input authority and does not leave the
+    // older actionable receipt current; the next actionable packet does.
+    expect(f.presented).not.toContain(2);
+    expect(f.presented).toContain(3);
+    expect(front).toEqual([1, null, 3]);
+    // Each read continued the same stream without asking for a recovery keyframe.
+    expect(harness.reads.slice(1, 4).map((read) => read.input.requestKeyFrame)).toEqual([
+      false,
+      false,
+      false,
+    ]);
+  });
+  it("still drops wrong-document display-only packets without decoding or painting them", async () => {
+    const f = setup();
+    f.identity("navigationGeneration", 2);
+    f.replyPackets(0, [1, 2], [2]);
+    await drain();
+    f.paint();
+    expect(f.drawn).toEqual([]);
+    expect(f.presented).toEqual([]);
+  });
   it("a host that disabled video stops reads and releases the decoder without an error", async () => {
     const f = setup();
     f.disabled(0);

package/client/use-browser-video.ts

@@ -71,6 +71,13 @@ export function useBrowserVideo(options: BrowserVideoOptions) {
     fallbackRef.current = ++fallbackCounter.current;
     setFallbackRevision(fallbackRef.current);
   };
+  /** Source/document/geometry validity: enough to decode and paint a packet. */
+  const sourceCurrent = (packet: BrowserVideoPacket, epoch: number) =>
+    latest.current.isCurrent(packet, epoch);
+  /** Input eligibility: source validity plus a host receipt. A display-only packet
+   * (actionable === false) may be decoded and painted but never admits a press. */
+  const inputCurrent = (packet: BrowserVideoPacket, epoch: number) =>
+    packet.actionable !== false && sourceCurrent(packet, epoch);
   const displayCurrent = (packet: BrowserVideoPacket) =>
     latest.current.isDisplayCurrent
       ? latest.current.isDisplayCurrent(packet)
@@ -78,7 +85,7 @@ export function useBrowserVideo(options: BrowserVideoOptions) {
   const front = presentation?.packet ?? null;
   const setFront = (value: BrowserVideoPacket | null, epoch = 0) => {
     displayedPacket.current = value;
-    if (!value || latest.current.isCurrent(value, epoch)) {
+    if (!value || inputCurrent(value, epoch)) {
       fallbackRef.current = 0;
       setFallbackRevision(0);
     }
@@ -121,7 +128,7 @@ export function useBrowserVideo(options: BrowserVideoOptions) {
       if (
         !fallbackRef.current &&
         presentation.authorityRevision === authorityRevision.current &&
-        latest.current.isCurrent(presentation.packet, presentation.epoch)
+        inputCurrent(presentation.packet, presentation.epoch)
       ) {
         frontRef.current = presentation.packet;
         latest.current.onPresented(presentation.packet, presentation.epoch);
@@ -167,9 +174,9 @@ export function useBrowserVideo(options: BrowserVideoOptions) {
         isCurrent: (packet, epoch) => !stopped && latest.current.isCurrent(packet, epoch),
         // Same source/document packets preserve the codec's dependency chain,
         // including a reply that began before a local press. They carry that old
-        // read epoch through paint and cannot authorize another press.
-        isSourceCurrent: (packet) =>
-          !stopped && latest.current.isCurrent(packet, latest.current.epoch()),
+        // read epoch through paint and cannot authorize another press. A display-only
+        // (non-actionable) packet is still source-valid and must keep decoding.
+        isSourceCurrent: (packet) => !stopped && sourceCurrent(packet, latest.current.epoch()),
         onPresented(packet, epoch) {
           if (stopped) return;
           lastPaintedAt = Date.now();
@@ -177,7 +184,7 @@ export function useBrowserVideo(options: BrowserVideoOptions) {
           setError(null);
           // Once this canvas is visible, its pixels change before React commits.
           // Publish authority synchronously with that draw, never the old front.
-          if (visible.current && latest.current.isCurrent(packet, epoch)) {
+          if (visible.current && inputCurrent(packet, epoch)) {
             frontRef.current = packet;
             latest.current.onPresented(packet, epoch);
           } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@omercnet/paseo-shared-browser",
-  "version": "1.2.2-next.239.1",
+  "version": "1.2.2-next.240.1",
   "description": "One real Chromium session per Paseo workspace, shared live across every connected client.",
   "keywords": [
     "browser",

package/server/browser-policy.ts

@@ -749,8 +749,10 @@ export class SessionManager {
               };
               session.videoReceipts.set(receipt, { frame, capturedAtMonotonicMs });
             }
-            this.rememberFrame(session, frame, capturedAtMonotonicMs);
-            packets.push({ ...packet, frame });
+            // False when input revoked this capture's time: it may still be painted,
+            // but it can never authorize a press, and the viewer must not try.
+            const actionable = this.rememberFrame(session, frame, capturedAtMonotonicMs);
+            packets.push({ ...packet, frame, actionable });
           }
         }
         while (session.videoReceipts.size > MAX_RECENT_FRAMES) {
@@ -1746,12 +1748,12 @@ export class SessionManager {
     frame: BrowserFrameAuthority,
     capturedAtMonotonicMs?: number,
     observedAt = this.now(),
-  ): void {
+  ): boolean {
     if (capturedAtMonotonicMs !== undefined && session.videoInputNotBefore !== null) {
       const exclusiveFloor = session.videoInputNotBefore + VIDEO_SOURCE_CLOCK_TOLERANCE_MS;
       if (!Number.isFinite(capturedAtMonotonicMs) || capturedAtMonotonicMs <= exclusiveFloor) {
         session.recentFrames.delete(frame.frameId);
-        return;
+        return false;
       }
     }
     this.pruneRecentFrames(session);
@@ -1770,6 +1772,7 @@ export class SessionManager {
       if (!oldest) break;
       session.recentFrames.delete(oldest);
     }
+    return session.recentFrames.has(frame.frameId);
   }
 
   /**

package/server/browser-video-input-authority.test.ts

@@ -153,6 +153,8 @@ it.each([-1, 0, 40])(
       const late = await pendingRead;
       pendingRead = null;
       expect(late.packets[0]!.frame.frameId).toBe(frame.frameId);
+      // Late pixels are painted but flagged: the viewer must not try to press on them.
+      expect(late.packets[0]!.actionable).toBe(false);
       // Late pixels still paint, but cannot re-authorize discrete agent input.
       await expect(strictInput()).rejects.toThrow("frame is stale");
       expect(calls).not.toContain("mouse.down");
@@ -185,6 +187,7 @@ it.each([-1, 0, 40])(
       packet.capturedAt = new Date(now).toISOString();
       const fresh = await manager.readVideo(videoInput);
       const freshFrame = fresh.packets[0]!.frame;
+      expect(fresh.packets[0]!.actionable).toBe(true);
       const accepted = await manager.beginGesture({
         viewerToken,
         controlToken,

package/shared/browser-video.ts

@@ -52,6 +52,9 @@ export type NativeVideoPacket = z.infer<typeof nativeVideoPacketSchema>;
 
 export const browserVideoPacketSchema = nativeVideoPacketSchema.extend({
   frame: browserFrameAuthoritySchema,
+  /** False when the host refused this capture as an input receipt (it predates
+   * acknowledged input). It may still be painted. Absent from older hosts. */
+  actionable: z.boolean().optional(),
 });
 export type BrowserVideoPacket = z.infer<typeof browserVideoPacketSchema>;