@push.rocks/smartbrowser 4.1.0 → 4.3.0

package/ts_web/classes.livebrowservideoreceiver.ts

@@ -0,0 +1,250 @@
+import type { ILiveBrowserState, ILiveVideoSource } from '@push.rocks/smartpuppeteer';
+import type { ILiveBrowserCanvasError, ILiveBrowserCanvasOperationOptions } from './interfaces.livebrowsercanvas.js';
+import type { ILiveBrowserVideoClient, ILiveBrowserVideoStatistics } from './interfaces.livebrowservideo.js';
+
+interface IVideoReceiverOptions {
+  video: HTMLVideoElement;
+  client: ILiveBrowserVideoClient;
+  operationTimeoutMs: number;
+  onUnavailable(): void;
+  onPresented(sourceArg: ILiveVideoSource): void;
+  onError(errorArg: ILiveBrowserCanvasError): void;
+}
+
+const sourceKey = (sourceArg: ILiveVideoSource): string => JSON.stringify([
+  sourceArg.tabId, sourceArg.generation, sourceArg.viewportRevision,
+  sourceArg.viewport.width, sourceArg.viewport.height, sourceArg.viewport.deviceScaleFactor,
+]);
+
+/** One native receiver. Every replacement closes the previous host peer before opening another. */
+export class LiveBrowserVideoReceiver {
+  private epoch = 0;
+  private desiredKey?: string;
+  private controller?: AbortController;
+  private peer?: RTCPeerConnection;
+  private frameCallback?: number;
+  private connectionTimer?: ReturnType<typeof setTimeout>;
+  private sampleTimer?: ReturnType<typeof setTimeout>;
+  private tail: Promise<void> = Promise.resolve();
+  private remoteMayExist = false;
+  private statistics?: ILiveBrowserVideoStatistics;
+  private decodedBeforePeer = 0;
+  private droppedBeforePeer = 0;
+
+  constructor(private readonly options: IVideoReceiverOptions) {
+    const video = options.video;
+    video.autoplay = true;
+    video.muted = true;
+    video.playsInline = true;
+    video.style.objectFit = 'contain';
+  }
+
+  public getStatistics(): ILiveBrowserVideoStatistics | undefined {
+    return this.statistics ? { ...this.statistics } : undefined;
+  }
+
+  public getCounters(): { framesReceived: number; framesDecoded: number; framesSkipped: number } {
+    const decoded = this.decodedBeforePeer + (this.statistics?.framesDecoded ?? 0);
+    const dropped = this.droppedBeforePeer + (this.statistics?.framesDropped ?? 0);
+    return { framesReceived: decoded + dropped, framesDecoded: decoded, framesSkipped: dropped };
+  }
+
+  public update(stateArg: ILiveBrowserState): void {
+    const tab = stateArg.tabs.find((tabArg) => tabArg.id === stateArg.activeTabId);
+    const source: ILiveVideoSource | undefined = document.visibilityState !== 'hidden'
+      && stateArg.status === 'running' && tab?.status === 'open' && tab.streaming
+      ? { tabId: tab.id, generation: tab.generation, viewportRevision: stateArg.viewportRevision,
+          viewport: { ...stateArg.viewport } }
+      : undefined;
+    const key = source ? sourceKey(source) : undefined;
+    if (key === this.desiredKey) return;
+    this.interrupt();
+    this.desiredKey = key;
+    const epoch = this.epoch;
+    const controller = new AbortController();
+    this.controller = controller;
+    this.enqueue(async () => {
+      await this.closeRemote();
+      if (!source || epoch !== this.epoch) return;
+      try {
+        await this.connect(source, controller.signal, epoch);
+      } catch (error) {
+        if (epoch !== this.epoch || controller.signal.aborted) return;
+        this.fail('video_negotiation_failed', error, epoch);
+      }
+    });
+  }
+
+  /** Synchronously prevents old frames or input from surviving a transport/source change. */
+  public interrupt(): void {
+    this.epoch++;
+    this.desiredKey = undefined;
+    this.controller?.abort(new Error('Video source was superseded'));
+    this.controller = undefined;
+    this.options.onUnavailable();
+    if (this.frameCallback !== undefined) this.options.video.cancelVideoFrameCallback(this.frameCallback);
+    this.frameCallback = undefined;
+    clearTimeout(this.connectionTimer);
+    clearTimeout(this.sampleTimer);
+    this.connectionTimer = undefined;
+    this.sampleTimer = undefined;
+    this.peer?.close();
+    this.peer = undefined;
+    this.options.video.pause();
+    this.options.video.srcObject = null;
+    this.decodedBeforePeer += this.statistics?.framesDecoded ?? 0;
+    this.droppedBeforePeer += this.statistics?.framesDropped ?? 0;
+    this.statistics = undefined;
+  }
+
+  public async stop(): Promise<void> {
+    this.interrupt();
+    await this.enqueue(() => this.closeRemote());
+  }
+
+  private enqueue(operationArg: () => Promise<void>): Promise<void> {
+    const operation = this.tail.then(operationArg, operationArg);
+    this.tail = operation.catch((error) => {
+      this.options.onError({ code: 'renderer_cleanup_failed', message: `Video peer cleanup failed: ${String(error)}`, cause: error });
+    });
+    return operation;
+  }
+
+  private async closeRemote(): Promise<void> {
+    if (!this.remoteMayExist) return;
+    await this.operation((optionsArg) => this.options.client.closeVideoPeer(optionsArg));
+    this.remoteMayExist = false;
+  }
+
+  private async connect(sourceArg: ILiveVideoSource, signalArg: AbortSignal, epochArg: number): Promise<void> {
+    this.remoteMayExist = true;
+    const offer = await this.operation((optionsArg) => this.options.client.openVideoPeer(optionsArg), signalArg);
+    if (sourceKey(offer.source) !== sourceKey(sourceArg)) throw new Error('Video offer does not match the current browser source');
+    if (signalArg.aborted || epochArg !== this.epoch) return;
+    const peer = new RTCPeerConnection({ iceServers: offer.iceServers, iceTransportPolicy: offer.iceTransportPolicy });
+    this.peer = peer;
+    this.statistics = { connectionState: 'new', framesDecoded: 0, framesDropped: 0, bytesReceived: 0, bitrate: 0 };
+    peer.addEventListener('connectionstatechange', () => {
+      if (epochArg !== this.epoch) return;
+      if (this.statistics) this.statistics.connectionState = peer.connectionState;
+      if (peer.connectionState === 'failed' || peer.connectionState === 'disconnected') {
+        this.fail('video_connection_failed', new Error(`Video connection ${peer.connectionState}`), epochArg);
+      }
+    });
+    let presented = false;
+    peer.addEventListener('track', (eventArg) => {
+      if (epochArg !== this.epoch || eventArg.track.kind !== 'video') return;
+      if ('jitterBufferTarget' in eventArg.receiver) eventArg.receiver.jitterBufferTarget = 0;
+      this.options.video.srcObject = new MediaStream([eventArg.track]);
+      this.frameCallback = this.options.video.requestVideoFrameCallback(() => {
+        this.frameCallback = undefined;
+        if (epochArg !== this.epoch) return;
+        clearTimeout(this.connectionTimer);
+        this.connectionTimer = undefined;
+        presented = true;
+        this.options.onPresented(sourceArg);
+      });
+      void this.options.video.play().catch((error) => this.fail('video_connection_failed', error, epochArg));
+    });
+    await this.operation(async ({ signal }) => {
+      await peer.setRemoteDescription(offer.description);
+      if (signal.aborted) throw signal.reason;
+      await peer.setLocalDescription(await peer.createAnswer());
+      await this.waitForIce(peer, signal);
+    }, signalArg);
+    if (signalArg.aborted || epochArg !== this.epoch) return;
+    const description = peer.localDescription;
+    if (!description?.sdp || description.type !== 'answer' || description.sdp.length > 48 * 1024) {
+      throw new Error('Invalid video answer');
+    }
+    await this.operation((optionsArg) => this.options.client.answerVideoPeer(offer.negotiationId,
+      { type: 'answer', sdp: description.sdp }, optionsArg), signalArg);
+    if (signalArg.aborted || epochArg !== this.epoch) return;
+    if (!presented) this.connectionTimer = setTimeout(() => this.fail('video_connection_failed',
+      new Error('No video frame received within 30 seconds'), epochArg), 30000);
+    void this.sample(peer, signalArg, epochArg);
+  }
+
+  private waitForIce(peerArg: RTCPeerConnection, signalArg: AbortSignal): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const cleanup = () => {
+        peerArg.removeEventListener('icegatheringstatechange', check);
+        signalArg.removeEventListener('abort', abort);
+      };
+      const abort = () => { cleanup(); reject(signalArg.reason); };
+      const check = () => {
+        if (signalArg.aborted) abort();
+        else if (peerArg.iceGatheringState === 'complete') { cleanup(); resolve(); }
+      };
+      peerArg.addEventListener('icegatheringstatechange', check);
+      signalArg.addEventListener('abort', abort, { once: true });
+      check();
+    });
+  }
+
+  private fail(codeArg: 'video_negotiation_failed' | 'video_connection_failed', errorArg: unknown, epochArg: number): void {
+    if (epochArg !== this.epoch) return;
+    this.interrupt();
+    void this.enqueue(() => this.closeRemote()).catch(() => undefined);
+    this.options.onError({ code: codeArg, message: String(errorArg), cause: errorArg });
+  }
+
+  private async sample(peerArg: RTCPeerConnection, signalArg: AbortSignal, epochArg: number,
+    previousArg?: { timestamp: number; bytes: number; delay: number; emitted: number }): Promise<void> {
+    let previous = previousArg;
+    try {
+      const [report, sender] = await this.operation(async (optionsArg) => Promise.all([
+        peerArg.getStats(), this.options.client.getVideoStatistics(optionsArg),
+      ]), signalArg);
+      if (epochArg !== this.epoch || !this.statistics) return;
+      this.statistics.sender = sender;
+      report.forEach((entryArg) => {
+        if (entryArg.type === 'inbound-rtp' && entryArg.kind === 'video') {
+          const interval = previous ? entryArg.timestamp - previous.timestamp : 0;
+          this.statistics!.framesDecoded = entryArg.framesDecoded ?? 0;
+          this.statistics!.framesDropped = entryArg.framesDropped ?? 0;
+          this.statistics!.bytesReceived = entryArg.bytesReceived ?? 0;
+          this.statistics!.framesPerSecond = entryArg.framesPerSecond;
+          this.statistics!.decoderImplementation = entryArg.decoderImplementation;
+          this.statistics!.powerEfficientDecoder = entryArg.powerEfficientDecoder;
+          this.statistics!.codec = report.get(entryArg.codecId)?.mimeType;
+          this.statistics!.bitrate = interval > 0 ? Math.max(0, (entryArg.bytesReceived - previous!.bytes) * 8000 / interval) : 0;
+          const emitted = entryArg.jitterBufferEmittedCount ?? 0;
+          const delay = entryArg.jitterBufferDelay ?? 0;
+          if (previous && emitted > previous.emitted) this.statistics!.jitterBufferMs =
+            (delay - previous.delay) * 1000 / (emitted - previous.emitted);
+          previous = { timestamp: entryArg.timestamp, bytes: entryArg.bytesReceived ?? 0, delay, emitted };
+        }
+        if (entryArg.type === 'transport' && entryArg.selectedCandidatePairId) {
+          const pair = report.get(entryArg.selectedCandidatePairId);
+          this.statistics!.roundTripMs = pair?.currentRoundTripTime === undefined ? undefined : pair.currentRoundTripTime * 1000;
+          this.statistics!.candidateType = report.get(pair?.remoteCandidateId)?.candidateType;
+        }
+      });
+    } catch (error) {
+      if (epochArg !== this.epoch || signalArg.aborted) return;
+      this.options.onError({ code: 'video_statistics_failed', message: `Could not read video statistics: ${String(error)}`, cause: error });
+    }
+    if (epochArg === this.epoch && !signalArg.aborted) {
+      this.sampleTimer = setTimeout(() => { void this.sample(peerArg, signalArg, epochArg, previous); }, 1000);
+    }
+  }
+
+  private operation<T>(operationArg: (optionsArg: ILiveBrowserCanvasOperationOptions) => Promise<T>,
+    parentSignalArg?: AbortSignal): Promise<T> {
+    const controller = new AbortController();
+    const abort = () => controller.abort(parentSignalArg?.reason);
+    parentSignalArg?.addEventListener('abort', abort, { once: true });
+    if (parentSignalArg?.aborted) abort();
+    const timer = setTimeout(() => controller.abort(new Error('Video operation timed out')), this.options.operationTimeoutMs);
+    return new Promise<T>((resolve, reject) => {
+      const onAbort = () => reject(controller.signal.reason);
+      controller.signal.addEventListener('abort', onAbort, { once: true });
+      if (controller.signal.aborted) onAbort();
+      else Promise.resolve().then(() => operationArg({ signal: controller.signal })).then(resolve, reject);
+    }).finally(() => {
+      clearTimeout(timer);
+      parentSignalArg?.removeEventListener('abort', abort);
+    });
+  }
+}

package/ts_web/classes.livebrowservideorenderer.ts

@@ -0,0 +1,9 @@
+import { LiveBrowserRenderer } from './classes.livebrowserrenderer.js';
+import type { ILiveBrowserVideoRendererOptions } from './interfaces.livebrowservideo.js';
+
+/** Native video presentation; shares input, viewport and lifecycle handling with the canvas renderer. */
+export class LiveBrowserVideoRenderer extends LiveBrowserRenderer {
+  constructor(optionsArg: ILiveBrowserVideoRendererOptions) {
+    super(optionsArg);
+  }
+}

package/ts_web/index.ts

@@ -25,3 +25,7 @@ export type {
   TLiveBrowserStatus,
   TLiveBrowserTabStatus,
 } from '@push.rocks/smartpuppeteer';
+
+export { LiveBrowserVideoRenderer } from './classes.livebrowservideorenderer.js';
+export type * from './interfaces.livebrowservideo.js';
+export type { ILiveVideoDescription, ILiveVideoOffer, ILiveVideoSource, ILiveVideoStatistics, ILiveVideoAcceleration } from '@push.rocks/smartpuppeteer';

package/ts_web/interfaces.livebrowsercanvas.ts

@@ -26,10 +26,11 @@ export interface ILiveBrowserCanvasClient {
     requestArg: ILiveBrowserFrameAcknowledgementRequest,
     optionsArg: ILiveBrowserCanvasOperationOptions,
   ): Promise<ILiveBrowserFrameAcknowledgement>;
+  /** Shared hosts return their effective viewport, which may be constrained by another viewer. */
   setViewport(
     viewportArg: ILiveBrowserViewport,
     optionsArg: ILiveBrowserCanvasOperationOptions,
-  ): Promise<void>;
+  ): Promise<void | ILiveBrowserCanvasViewportResult>;
   dispatchMouse(
     inputArg: ILiveBrowserMouseInput,
     optionsArg: ILiveBrowserCanvasOperationOptions,
@@ -48,12 +49,21 @@ export interface ILiveBrowserCanvasClient {
   ): Promise<void>;
 }
 
+export interface ILiveBrowserCanvasViewportResult {
+  viewport: ILiveBrowserViewport;
+  /** The authoritative revision, including when the aggregate viewport did not change. */
+  viewportRevision: number;
+}
+
 export interface ILiveBrowserCanvasOperationOptions {
   /** Adapters must propagate this signal and reject promptly when it aborts. */
   signal: AbortSignal;
 }
 
 export type TLiveBrowserCanvasErrorCode =
+  | 'video_negotiation_failed'
+  | 'video_connection_failed'
+  | 'video_statistics_failed'
   | 'frame_acknowledgement_failed'
   | 'frame_render_failed'
   | 'input_dispatch_failed'

package/ts_web/interfaces.livebrowservideo.ts

@@ -0,0 +1,46 @@
+import type {
+  ILiveVideoDescription,
+  ILiveVideoOffer,
+  ILiveVideoStatistics,
+} from '@push.rocks/smartpuppeteer';
+import type {
+  ILiveBrowserCanvasClient,
+  ILiveBrowserCanvasOperationOptions,
+  ILiveBrowserCanvasRendererOptions,
+  ILiveBrowserCanvasRendererStatistics,
+} from './interfaces.livebrowsercanvas.js';
+
+/** Signaling is authorized by the host's existing viewer lease. */
+export interface ILiveBrowserVideoClient extends Omit<ILiveBrowserCanvasClient, 'acknowledgeFrame'> {
+  openVideoPeer(optionsArg: ILiveBrowserCanvasOperationOptions): Promise<Omit<ILiveVideoOffer, 'peerId'>>;
+  answerVideoPeer(negotiationIdArg: string, descriptionArg: ILiveVideoDescription,
+    optionsArg: ILiveBrowserCanvasOperationOptions): Promise<void>;
+  closeVideoPeer(optionsArg: ILiveBrowserCanvasOperationOptions): Promise<void>;
+  getVideoStatistics(optionsArg: ILiveBrowserCanvasOperationOptions): Promise<ILiveVideoStatistics>;
+}
+
+export interface ILiveBrowserVideoRendererOptions extends Omit<ILiveBrowserCanvasRendererOptions,
+  'canvas' | 'client' | 'onFrameRendered' | 'frameDecodeTimeoutMs'> {
+  video: HTMLVideoElement;
+  client: ILiveBrowserVideoClient;
+}
+
+export interface ILiveBrowserVideoStatistics {
+  connectionState: RTCPeerConnectionState;
+  framesDecoded: number;
+  framesDropped: number;
+  bytesReceived: number;
+  bitrate: number;
+  framesPerSecond?: number;
+  roundTripMs?: number;
+  jitterBufferMs?: number;
+  codec?: string;
+  decoderImplementation?: string;
+  powerEfficientDecoder?: boolean;
+  candidateType?: string;
+  sender?: ILiveVideoStatistics;
+}
+
+export interface ILiveBrowserVideoRendererStatistics extends ILiveBrowserCanvasRendererStatistics {
+  video?: ILiveBrowserVideoStatistics;
+}

package/readme.hints.md

@@ -1,34 +0,0 @@
-# SmartBrowser Hints
-
-## Dependencies (as of 2026-08-24)
-- `@push.rocks/smartpdf` v5.x uses SmartPuppeteer 2/Puppeteer 25 browser types. `SmartPdf` remains lazy and starts only for PDF methods.
-- `@push.rocks/smartpuppeteer` v2.6 owns `LiveBrowserSession`, the canonical transport-neutral live browser contracts, bounded CDP frame acknowledgement flow, and authenticated-proxy service-worker coverage.
-- Tests import `@git.zone/tstest/tapbundle`; browser tests use the `.chromium.ts` suffix.
-- The Node.js entry requires Node.js 22.12 or newer. This repository uses pnpm 11.21 for development and release tooling.
-- Puppeteer `page.screenshot({ encoding: 'binary' })` returns `Uint8Array`, not `Buffer` - wrap with `Buffer.from()`
-
-## Build
-- `pnpm run build` uses `tsbuild tsfolders` and emits both `dist_ts` and `dist_ts_web`.
-- `@push.rocks/smartbrowser/web` must remain runtime-isolated from Puppeteer, SmartPDF, Buffer, and Node.js built-ins. Only type imports from SmartPuppeteer are allowed.
-- `pnpm test` runs Node and Chromium files through tstest 4. The Chromium renderer tests claim a 2D context in `installCanvas()` so pixels can be read back; the renderer then keeps its 2D path. One test uses a fresh canvas to cover the `bitmaprenderer` path.
-
-## Canvas Renderer Invariants
-- The client facade is transport-neutral. Authentication, authorization, binary wire encoding, session ownership, and egress policy belong to the consuming adapter.
-- Input always carries the identity of the actually displayed frame, never merely the newest state event.
-- Input and frame rendering require the active tab state to report `streaming: true`; a stopped stream clears the displayed frame even when its identity fields have not advanced.
-- Cached state may lag a new frame generation. A state identity ahead of the displayed frame invalidates it; same-identity metadata updates do not.
-- One frame may decode while only the newest subsequent frame waits; a superseded queued frame is never decoded. Active-run frames must pass full static validation and then have a sequence strictly above the per-run high-water value. Duplicate and out-of-order frames suspend without acknowledgement; resume resets the high-water value.
-- Validated frames are acknowledged at receipt, before decoding. The newest decoded bitmap is presented once per animation frame (16 ms timer while hidden), the canvas backing store is reallocated only when frame dimensions change, and a fresh canvas gets an `ImageBitmapRenderingContext`; a canvas that already owns a 2D context keeps the 2D path. Decoded-dimension integrity is still checked before presentation.
-- The renderer makes exactly one acknowledgement request for each valid frame while its run remains active. Invalid-protocol frames do not start acknowledgement work. An acknowledgement that throws, rejects, or times out suspends the run; `{ accepted: false }` settles the local attempt and does not prove upstream retirement.
-- Every asynchronous client method receives `ILiveBrowserCanvasOperationOptions` with a renderer-owned `AbortSignal`. Operation deadline, run suspension, and stop all abort the signal; adapters must propagate it through the underlying transport.
-- Renderer starts, suspensions, resumes, and stops are serialized by run epoch. Suspension aborts owned work and drops state, frames, queued input, pressed state, and old-generation recovery. Resume creates a fresh epoch and requires newly read state plus a newly displayed frame before input.
-- Native `createImageBitmap()` jobs are tracked outside per-run cleanup. A new start or resume waits for every prior raw decode to settle, and late bitmaps are closed after timeout or interruption. Start or resume can remain pending indefinitely if the browser never settles native decode work.
-- Acknowledgement failures, input-release failures, acknowledgement-capacity exhaustion, and client/decode timeouts suspend only the current run. Discrete input-queue capacity exhaustion is a non-fatal `input_queue_capacity_exceeded` rejection of the new command; coalescable wheel and move commands never raise it. Malformed, oversized, or inconsistent frame protocol and deterministic decode-integrity failures are also terminal to that run. Recovery is explicit through `resume()`; none of these paths retries automatically.
-- Internal renderer ceilings are 16 pending acknowledgements; 4 in-flight input commands, 128 queued discrete input commands, and 32 queued coalescable input commands; 12,288 per frame dimension and 8,294,400 frame pixels; 34,226,176 encoded bytes; and a 4,096 x 4,096 viewport, device scale factor 3, and 8,294,400 physical pixels.
-- Repeated blur, visibility, and pointer-loss resets coalesce into one bounded reset operation.
-- Input dispatch pipelines up to 4 commands in queue order; input idle means an empty queue and nothing in flight. Wheel deltas accumulate outside the queue and flush at most once per animation frame and only while no wheel dispatch is in flight; a discrete command forces the flush first so order is kept. Consecutive moves coalesce to the newest position. `getStatistics()` exposes cumulative frame and input counters.
-- During an active run, pressed input is retained until release succeeds, with one retry for transient release failures. Input remains blocked while a release is unresolved. Suspension discards pressed state without dispatching releases into a later run.
-- A rejected non-timeout input dispatch clears the ambiguous displayed frame and requires a newly rendered matching frame. A timed-out dispatch suspends the run and requires explicit resume plus a fresh frame.
-- Resize uses a caller-supplied stable CSS element. `ResizeObserver` observations are debounced 100 ms trailing, `syncViewport()` is immediate. Requests are latest-only and serialized; input is released before resizing and remains blocked until a frame meeting the target revision fence is drawn.
-- Intrinsic canvas dimensions come from encoded frame dimensions. Pointer coordinates map to the frame's logical CSS viewport.
-- Pointer and wheel listeners remain on the canvas. Keyboard and composition listeners use `focusTarget`, which defaults to the canvas. Call `insertText()` for text committed by an external IME or dedicated input control.