@restregular/dsh-visual-browser 0.1.0 → 0.2.0

package/lib/cdp.js

@@ -1,192 +1,208 @@
-/**
- * Minimal Chrome DevTools Protocol client.
- *
- * The plugin ships without dependencies: Node.js 22+ provides a WHATWG
- * `WebSocket`, which is all a flattened CDP session needs. Every command is a
- * JSON message with an `id`; flattened target sessions add `sessionId` to the
- * command and echo it on the matching response.
- */
-
-/** A CDP failure with a stable code so tools and routes can report a reason. */
-export class CdpError extends Error {
-  constructor(code, message) {
-    super(message);
-    this.name = 'CdpError';
-    this.code = code;
-  }
-}
-
-const OPEN_TIMEOUT_MS = 15_000;
-const COMMAND_TIMEOUT_MS = 30_000;
-
-export class CdpConnection {
-  #socket;
-  #nextId = 1;
-  #pending = new Map();
-  #listeners = new Map();
-  #closed = false;
-
-  constructor(socket) {
-    this.#socket = socket;
-    socket.addEventListener('message', (event) => this.#onMessage(event.data));
-    socket.addEventListener('close', () => this.#failAll(new CdpError('CDP_CLOSED', 'the DevTools connection closed')));
-    socket.addEventListener('error', () => {
-      /* The close event carries the outcome; keep this listener quiet. */
-    });
-  }
-
-  /** Connect to a browser-level or page-level WebSocket endpoint. */
-  static async open(url, { timeoutMs = OPEN_TIMEOUT_MS } = {}) {
-    const Socket = globalThis.WebSocket;
-    if (typeof Socket !== 'function') {
-      throw new CdpError('CDP_UNAVAILABLE', 'this Node.js runtime has no built-in WebSocket (Node 22 or newer is required)');
-    }
-    const socket = new Socket(url);
-    await new Promise((resolve, reject) => {
-      const timer = setTimeout(() => {
-        cleanup();
-        try {
-          socket.close();
-        } catch {
-          /* already unusable */
-        }
-        reject(new CdpError('CDP_TIMEOUT', `timed out connecting to ${url}`));
-      }, timeoutMs);
-      const cleanup = () => {
-        clearTimeout(timer);
-        socket.removeEventListener('open', onOpen);
-        socket.removeEventListener('error', onError);
-      };
-      const onOpen = () => {
-        cleanup();
-        resolve();
-      };
-      const onError = () => {
-        cleanup();
-        reject(new CdpError('CDP_CONNECT_FAILED', `could not connect to ${url}`));
-      };
-      socket.addEventListener('open', onOpen);
-      socket.addEventListener('error', onError);
-    });
-    return new CdpConnection(socket);
-  }
-
-  get closed() {
-    return this.#closed;
-  }
-
-  /** Send one protocol command; `sessionId` targets a flattened session. */
-  send(method, params = {}, sessionId) {
-    if (this.#closed) return Promise.reject(new CdpError('CDP_CLOSED', 'the DevTools connection closed'));
-    const id = this.#nextId++;
-    const payload = { id, method, params };
-    if (sessionId !== undefined) payload.sessionId = sessionId;
-    return new Promise((resolve, reject) => {
-      const timer = setTimeout(() => {
-        this.#pending.delete(id);
-        reject(new CdpError('CDP_COMMAND_TIMEOUT', `${method} did not answer within ${COMMAND_TIMEOUT_MS}ms`));
-      }, COMMAND_TIMEOUT_MS);
-      this.#pending.set(id, { resolve, reject, method, timer });
-      try {
-        this.#socket.send(JSON.stringify(payload));
-      } catch (error) {
-        clearTimeout(timer);
-        this.#pending.delete(id);
-        reject(new CdpError('CDP_SEND_FAILED', messageOf(error)));
-      }
-    });
-  }
-
-  /** Subscribe to one protocol event; the returned function unsubscribes. */
-  on(method, listener) {
-    const bucket = this.#listeners.get(method) ?? new Set();
-    bucket.add(listener);
-    this.#listeners.set(method, bucket);
-    return () => {
-      bucket.delete(listener);
-      if (bucket.size === 0) this.#listeners.delete(method);
-    };
-  }
-
-  /** Resolve on the next `method` event, or reject on timeout. */
-  once(method, timeoutMs) {
-    return new Promise((resolve, reject) => {
-      const timer = setTimeout(() => {
-        off();
-        reject(new CdpError('CDP_EVENT_TIMEOUT', `waited ${timeoutMs}ms for ${method}`));
-      }, timeoutMs);
-      const off = this.on(method, (params, sessionId) => {
-        clearTimeout(timer);
-        off();
-        resolve({ params, sessionId });
-      });
-    });
-  }
-
-  close() {
-    if (this.#closed) return;
-    this.#failAll(new CdpError('CDP_CLOSED', 'the DevTools connection closed'));
-    try {
-      this.#socket.close();
-    } catch {
-      /* already closed */
-    }
-  }
-
-  #onMessage(raw) {
-    let message;
-    try {
-      message = JSON.parse(typeof raw === 'string' ? raw : String(raw));
-    } catch {
-      return;
-    }
-    if (message && typeof message.id === 'number') {
-      const pending = this.#pending.get(message.id);
-      if (pending === undefined) return;
-      this.#pending.delete(message.id);
-      clearTimeout(pending.timer);
-      if (message.error) {
-        pending.reject(
-          new CdpError(
-            `CDP_${String(message.error.code ?? 'ERROR')}`,
-            `${pending.method}: ${message.error.message ?? 'protocol error'}`,
-          ),
-        );
-      } else {
-        pending.resolve(message.result);
-      }
-      return;
-    }
-    if (!message || typeof message.method !== 'string') return;
-    const bucket = this.#listeners.get(message.method);
-    if (bucket === undefined) return;
-    for (const listener of [...bucket]) {
-      try {
-        listener(message.params, message.sessionId);
-      } catch {
-        /* a failing listener must not break the socket */
-      }
-    }
-  }
-
-  #failAll(error) {
-    if (this.#closed) return;
-    this.#closed = true;
-    for (const [id, pending] of this.#pending) {
-      clearTimeout(pending.timer);
-      this.#pending.delete(id);
-      pending.reject(error);
-    }
-    this.#listeners.clear();
-  }
-}
-
-export function messageOf(error) {
-  if (error instanceof Error) return error.message;
-  if (typeof error === 'string') return error;
-  try {
-    return JSON.stringify(error);
-  } catch {
-    return String(error);
-  }
-}
+/**
+ * Minimal Chrome DevTools Protocol client.
+ *
+ * The plugin ships without dependencies: Node.js 22+ provides a WHATWG
+ * `WebSocket`, which is all a flattened CDP session needs. Every command is a
+ * JSON message with an `id`; flattened target sessions add `sessionId` to the
+ * command and echo it on the matching response.
+ */
+
+/** A CDP failure with a stable code so tools and routes can report a reason. */
+export class CdpError extends Error {
+  constructor(code, message) {
+    super(message);
+    this.name = 'CdpError';
+    this.code = code;
+  }
+}
+
+const OPEN_TIMEOUT_MS = 15_000;
+const COMMAND_TIMEOUT_MS = 30_000;
+
+export class CdpConnection {
+  #socket;
+  #nextId = 1;
+  #pending = new Map();
+  #listeners = new Map();
+  #closeListeners = new Set();
+  #closed = false;
+
+  constructor(socket) {
+    this.#socket = socket;
+    socket.addEventListener('message', (event) => this.#onMessage(event.data));
+    socket.addEventListener('close', () => this.#failAll(new CdpError('CDP_CLOSED', 'the DevTools connection closed')));
+    socket.addEventListener('error', () => {
+      /* The close event carries the outcome; keep this listener quiet. */
+    });
+  }
+
+  /** Connect to a browser-level or page-level WebSocket endpoint. */
+  static async open(url, { timeoutMs = OPEN_TIMEOUT_MS } = {}) {
+    const Socket = globalThis.WebSocket;
+    if (typeof Socket !== 'function') {
+      throw new CdpError('CDP_UNAVAILABLE', 'this Node.js runtime has no built-in WebSocket (Node 22 or newer is required)');
+    }
+    const socket = new Socket(url);
+    await new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        cleanup();
+        try {
+          socket.close();
+        } catch {
+          /* already unusable */
+        }
+        reject(new CdpError('CDP_TIMEOUT', `timed out connecting to ${url}`));
+      }, timeoutMs);
+      const cleanup = () => {
+        clearTimeout(timer);
+        socket.removeEventListener('open', onOpen);
+        socket.removeEventListener('error', onError);
+      };
+      const onOpen = () => {
+        cleanup();
+        resolve();
+      };
+      const onError = () => {
+        cleanup();
+        reject(new CdpError('CDP_CONNECT_FAILED', `could not connect to ${url}`));
+      };
+      socket.addEventListener('open', onOpen);
+      socket.addEventListener('error', onError);
+    });
+    return new CdpConnection(socket);
+  }
+
+  get closed() {
+    return this.#closed;
+  }
+
+  /** Send one protocol command; `sessionId` targets a flattened session. */
+  send(method, params = {}, sessionId) {
+    if (this.#closed) return Promise.reject(new CdpError('CDP_CLOSED', 'the DevTools connection closed'));
+    const id = this.#nextId++;
+    const payload = { id, method, params };
+    if (sessionId !== undefined) payload.sessionId = sessionId;
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.#pending.delete(id);
+        reject(new CdpError('CDP_COMMAND_TIMEOUT', `${method} did not answer within ${COMMAND_TIMEOUT_MS}ms`));
+      }, COMMAND_TIMEOUT_MS);
+      this.#pending.set(id, { resolve, reject, method, timer });
+      try {
+        this.#socket.send(JSON.stringify(payload));
+      } catch (error) {
+        clearTimeout(timer);
+        this.#pending.delete(id);
+        reject(new CdpError('CDP_SEND_FAILED', messageOf(error)));
+      }
+    });
+  }
+
+  /** Subscribe to one protocol event; the returned function unsubscribes. */
+  on(method, listener) {
+    const bucket = this.#listeners.get(method) ?? new Set();
+    bucket.add(listener);
+    this.#listeners.set(method, bucket);
+    return () => {
+      bucket.delete(listener);
+      if (bucket.size === 0) this.#listeners.delete(method);
+    };
+  }
+
+  /** Resolve on the next `method` event, or reject on timeout. */
+  once(method, timeoutMs) {
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        off();
+        reject(new CdpError('CDP_EVENT_TIMEOUT', `waited ${timeoutMs}ms for ${method}`));
+      }, timeoutMs);
+      const off = this.on(method, (params, sessionId) => {
+        clearTimeout(timer);
+        off();
+        resolve({ params, sessionId });
+      });
+    });
+  }
+
+  /** Notified once with the failure that closed the socket. */
+  onClose(listener) {
+    this.#closeListeners.add(listener);
+    return () => this.#closeListeners.delete(listener);
+  }
+
+  close() {
+    if (this.#closed) return;
+    this.#failAll(new CdpError('CDP_CLOSED', 'the DevTools connection closed'));
+    try {
+      this.#socket.close();
+    } catch {
+      /* already closed */
+    }
+  }
+
+  #onMessage(raw) {
+    let message;
+    try {
+      message = JSON.parse(typeof raw === 'string' ? raw : String(raw));
+    } catch {
+      return;
+    }
+    if (message && typeof message.id === 'number') {
+      const pending = this.#pending.get(message.id);
+      if (pending === undefined) return;
+      this.#pending.delete(message.id);
+      clearTimeout(pending.timer);
+      if (message.error) {
+        pending.reject(
+          new CdpError(
+            `CDP_${String(message.error.code ?? 'ERROR')}`,
+            `${pending.method}: ${message.error.message ?? 'protocol error'}`,
+          ),
+        );
+      } else {
+        pending.resolve(message.result);
+      }
+      return;
+    }
+    if (!message || typeof message.method !== 'string') return;
+    const bucket = this.#listeners.get(message.method);
+    if (bucket === undefined) return;
+    for (const listener of [...bucket]) {
+      try {
+        listener(message.params, message.sessionId);
+      } catch {
+        /* a failing listener must not break the socket */
+      }
+    }
+  }
+
+  #failAll(error) {
+    if (this.#closed) return;
+    this.#closed = true;
+    for (const [id, pending] of this.#pending) {
+      clearTimeout(pending.timer);
+      this.#pending.delete(id);
+      pending.reject(error);
+    }
+    this.#listeners.clear();
+    const closeListeners = [...this.#closeListeners];
+    this.#closeListeners.clear();
+    for (const listener of closeListeners) {
+      try {
+        listener(error);
+      } catch {
+        /* a close listener must not block teardown */
+      }
+    }
+  }
+}
+
+export function messageOf(error) {
+  if (error instanceof Error) return error.message;
+  if (typeof error === 'string') return error;
+  try {
+    return JSON.stringify(error);
+  } catch {
+    return String(error);
+  }
+}

package/lib/routes.js

@@ -1,15 +1,15 @@
 /**
  * Same-origin HTTP API behind the Visual Browser panel.
  *
- * The panel polls `/state` for the timeline and pulls `/frame?seq=` for the
- * picture; user clicks, wheel events and keys go back through the same
- * BrowserManager methods the agent tools use. Frames stay on the host, so a
- * 1 Hz picture never costs a session token.
+ * Three jobs: a tiny `/state` JSON poll for the operation log, one long-lived
+ * `/stream` response that carries Chrome's own screencast frames as a binary
+ * frame stream, and `/input` batches that go straight to the DevTools input
+ * domain. No screenshot per click, no per-frame request, no base64 hop.
  */
 
 import { BrowserError } from './browser.js';
 
-const BODY_LIMIT = 64 * 1024;
+const BODY_LIMIT = 128 * 1024;
 
 class HttpError extends Error {
   constructor(status, code, message) {
@@ -28,17 +28,10 @@ function sendJson(res, status, payload) {
   res.end(body);
 }
 
-function sendFrame(res, frame) {
-  if (frame === null || frame === undefined) {
-    res.statusCode = 204;
-    res.end();
-    return;
-  }
-  res.statusCode = 200;
-  res.setHeader('content-type', 'image/jpeg');
+function sendEmpty(res, status) {
+  res.statusCode = status;
   res.setHeader('cache-control', 'no-store');
-  res.setHeader('content-length', String(frame.jpeg.length));
-  res.end(frame.jpeg);
+  res.end();
 }
 
 async function readJsonBody(req) {
@@ -75,6 +68,78 @@ function failure(res, error) {
   sendJson(res, status, { ok: false, error: { code, message } });
 }
 
+/**
+ * Hold the response open as one binary frame stream:
+ *
+ *   [uint32 LE payload length][uint32 LE sequence][JPEG bytes]...
+ *
+ * The panel reads it with `fetch` streaming and decodes each JPEG with
+ * `createImageBitmap` onto a canvas. Parsing the framing ourselves keeps the
+ * picture deterministic across browser builds and skips the base64 hop an
+ * event stream would need.
+ */
+function openStream(manager, res) {
+  if (typeof manager.subscribeFrames !== 'function') {
+    sendEmpty(res, 503);
+    return;
+  }
+  if (!manager.running) {
+    sendEmpty(res, 204);
+    return;
+  }
+  res.statusCode = 200;
+  // A custom type stays out of every compressible mime table on purpose: the
+  // host's gzip middleware must never buffer a live picture.
+  res.setHeader('content-type', 'application/x-dsh-visual-browser-frames');
+  // no-transform keeps the host's gzip middleware out of a live picture.
+  res.setHeader('cache-control', 'no-store, no-transform');
+  res.setHeader('pragma', 'no-cache');
+  res.setHeader('x-accel-buffering', 'no');
+  res.setHeader('connection', 'keep-alive');
+  res.flushHeaders?.();
+
+  let closed = false;
+  let paused = false;
+  let unsubscribe = null;
+  let sequence = 0;
+
+  const cleanup = () => {
+    if (closed) return;
+    closed = true;
+    try {
+      unsubscribe?.();
+    } catch {
+      /* the browser may be gone */
+    }
+    unsubscribe = null;
+  };
+
+  const writeFrame = (jpeg) => {
+    if (closed || paused) return;
+    try {
+      const header = Buffer.allocUnsafe(8);
+      header.writeUInt32LE(jpeg.length, 0);
+      header.writeUInt32LE(++sequence, 4);
+      const headOk = res.write(header);
+      const bodyOk = res.write(jpeg);
+      if (!headOk || !bodyOk) {
+        // Slow consumer: drop frames until the socket drains instead of growing
+        // an unbounded queue of stale pictures.
+        paused = true;
+        res.once('drain', () => {
+          paused = false;
+        });
+      }
+    } catch {
+      cleanup();
+    }
+  };
+
+  unsubscribe = manager.subscribeFrames(writeFrame);
+  res.on('close', cleanup);
+  res.on('error', cleanup);
+}
+
 /**
  * Handle one request under `prefix`. Exported so a smoke test can drive it with
  * a plain request/response pair and a stub manager.
@@ -90,7 +155,7 @@ export async function handle(manager, prefix, req, res) {
   }
   let route = url.pathname;
   if (route.startsWith(prefix)) route = route.slice(prefix.length);
-  if (route === '' || route === '/') route = '/';
+  if (route === '') route = '/';
 
   try {
     if (method === 'GET' && route === '/') {
@@ -101,15 +166,15 @@ export async function handle(manager, prefix, req, res) {
       sendJson(res, 200, { ok: true, state: manager.state() });
       return;
     }
-    if (method === 'GET' && route === '/frame') {
-      const seq = Number(url.searchParams.get('seq') ?? '0');
-      sendFrame(res, manager.frameFor(seq));
+    if (method === 'GET' && route === '/stream') {
+      openStream(manager, res);
       return;
     }
-    if (method === 'GET' && route === '/live') {
-      const state = await manager.refreshFrame();
-      const frame = manager.frameFor(state.frameSeq);
-      sendFrame(res, frame);
+    if (method === 'POST' && route === '/input') {
+      const body = await readJsonBody(req);
+      const events = Array.isArray(body.events) ? body.events : [];
+      const result = await manager.dispatchInput(events);
+      sendJson(res, 200, { ok: true, applied: result?.applied ?? 0, state: manager.state() });
       return;
     }
     if (method === 'POST' && route === '/start') {
@@ -117,6 +182,7 @@ export async function handle(manager, prefix, req, res) {
       const target = typeof body.url === 'string' ? body.url.trim() : '';
       if (target !== '') await manager.open(target);
       else await manager.ensureStarted();
+      manager.refreshLiveView?.();
       sendJson(res, 200, { ok: true, state: manager.state() });
       return;
     }
@@ -125,33 +191,12 @@ export async function handle(manager, prefix, req, res) {
       sendJson(res, 200, { ok: true, state: manager.state() });
       return;
     }
-    if (method === 'POST' && route === '/capture') {
-      const state = await manager.refreshFrame();
-      sendJson(res, 200, { ok: true, state });
-      return;
-    }
-    if (method === 'POST' && route === '/click') {
-      const body = await readJsonBody(req);
-      await manager.clickAt({ x: body.x, y: body.y });
-      sendJson(res, 200, { ok: true, state: manager.state() });
-      return;
-    }
-    if (method === 'POST' && route === '/wheel') {
-      const body = await readJsonBody(req);
-      await manager.wheelAt({ deltaY: body.deltaY, deltaX: body.deltaX });
-      sendJson(res, 200, { ok: true, state: manager.state() });
-      return;
-    }
-    if (method === 'POST' && route === '/key') {
-      const body = await readJsonBody(req);
-      await manager.keyFromUser(String(body.key ?? 'Enter'));
-      sendJson(res, 200, { ok: true, state: manager.state() });
-      return;
-    }
     if (method === 'POST' && route === '/nav') {
       const body = await readJsonBody(req);
       const action = String(body.action ?? '');
-      if (!['back', 'forward', 'reload'].includes(action)) throw new HttpError(400, 'BAD_ACTION', 'action must be back, forward, or reload');
+      if (!['back', 'forward', 'reload'].includes(action)) {
+        throw new HttpError(400, 'BAD_ACTION', 'action must be back, forward, or reload');
+      }
       await manager.act({ action });
       sendJson(res, 200, { ok: true, state: manager.state() });
       return;
@@ -159,6 +204,15 @@ export async function handle(manager, prefix, req, res) {
     sendJson(res, 404, { ok: false, error: { code: 'NOT_FOUND', message: `No route for ${method} ${route}` } });
   } catch (error) {
     manager.noteError?.(error);
+    if (res.headersSent) {
+      // A stream that failed after its headers went out can only be torn down.
+      try {
+        res.end();
+      } catch {
+        /* already closed */
+      }
+      return;
+    }
     failure(res, error);
   }
 }

package/locale/en.json

@@ -1,6 +1,6 @@
 {
   "meta": {
     "title": "Visual Browser",
-    "description": "A real Chrome for the Agent plus a live view for you: the Agent reads a numbered, token-light list of interactive elements instead of the raw DOM, and every action lands in a panel with the page frame, a replayable timeline, and click-through control."
+    "description": "One extra door under the right Sidebar's built-in Browser: a live picture of the Chrome the Agent drives, with real click, drag, wheel and keyboard control, plus a collapsible, scrollable action log — no screenshot polling, no frame replay."
   }
 }

package/locale/zh.json

@@ -1,6 +1,6 @@
 {
   "meta": {
     "title": "可视化浏览器",
-    "description": "给 Agent 一个真实的 Chrome，也给你一块实时画面：Agent 只读「可交互元素编号列表」这种精简视图而不是整页 DOM，每次操作都会在侧边栏留下画面、可回放的操作时间线，你也可以直接点画面接管浏览器。"
+    "description": "在右侧边栏「浏览器」下面多一个入口：实时映射 Agent 正在操作的 Chrome 画面，点击、拖动、滚轮、键盘都能直接操作，操作记录可折叠、可滚动，不再需要逐帧截图或回放。"
   }
 }