@inetafrica/open-claudia 3.5.24 → 3.5.26

package/core/browser-drive.js

@@ -0,0 +1,619 @@
+// The Browser app's phone and web view (bot apps v2): while a person drives the bot's browser and
+// Chrome runs in its own container (AGENT_BROWSER_CDP), the bot's tab takes the driver's screen.
+// Its size and pixel density, a phone's page layout and touch on a phone, a desktop's on the web.
+// This module's own DevTools connection then carries the driver's whole session:
+//   - the picture: a screencast at the driver's size, so text is sharp on their screen;
+//   - their input: mouse, keys, touch and typed text, straight to Chrome, in order;
+//   - the page's text fields (where they are, what kind), so the driver's own keyboard opens the
+//     moment they tap one.
+// Handing back (stop) closes the connection. Chrome drops the size, layout and touch with it, and
+// the window goes back to the size it had.
+//
+// Why the window is resized too: Chrome's screencast pictures the window, so a phone-size view in
+// a desktop-size window comes out boxed in a corner after the next page load (proven 10 Oct on
+// Chrome 149). Order matters: the view first, then the window, or the first pictures stay at 1x.
+// The browser's identity (user agent) never changes: sites the bot or the person signed into
+// must see the same browser after the hand back.
+//
+// Only one drive at a time (the driver lock is single). Nothing here decides who may drive:
+// core/browser-control.js does, and calls start / fit / dispatch / stop.
+
+const WebSocket = require("ws");
+const view = require("./browser-view");
+
+const QUALITY = 65;
+const MAX_FPS = 30;
+// A drive's picture is at most this many device pixels (a phone at 2x is about 1.3 million).
+const MAX_PIXELS = 1600000;
+const MIN_SIDE = 240;
+const MAX_SIDE = 1920;
+const MAX_DPR = 3;
+const CALL_TIMEOUT_MS = 4000;
+const CONNECT_TIMEOUT_MS = 3000;
+// The text fields are read this long after the page stops changing (and after a tap).
+const FIELDS_SETTLE_MS = 250;
+// After a page load the window is nudged this long after the load's events settle.
+const NUDGE_DELAY_MS = 120;
+const MAX_FIELDS = 60;
+const TEXT_MAX = 500;
+
+function defaultDeps() {
+  return {
+    now: () => Date.now(),
+    endpoint: () => view.cdpEndpoint(),
+    fetchJson: async (url) => {
+      const ctrl = new AbortController();
+      const timer = setTimeout(() => ctrl.abort(), CONNECT_TIMEOUT_MS);
+      if (timer.unref) timer.unref();
+      try {
+        const res = await fetch(url, { signal: ctrl.signal });
+        if (!res.ok) throw new Error(`HTTP ${res.status}`);
+        return await res.json();
+      } finally {
+        clearTimeout(timer);
+      }
+    },
+    connect: (url) => new WebSocket(url, { perMessageDeflate: false }),
+    targetOf: (session) => view.sessionTargetId(session),
+  };
+}
+
+let deps = defaultDeps();
+let current = null;
+
+// ---------------------------------------------------------------------------
+// The driver's screen, checked and capped
+
+function round2(n) {
+  return Math.round(n * 100) / 100;
+}
+
+// { width, height, dpr, mobile, touch } in CSS pixels, or null when it is not a screen.
+function checkViewport(vp) {
+  if (!vp || typeof vp !== "object" || Array.isArray(vp)) return null;
+  const width = Math.round(Number(vp.width));
+  const height = Math.round(Number(vp.height));
+  if (!Number.isFinite(width) || !Number.isFinite(height)) return null;
+  if (width < MIN_SIDE || height < MIN_SIDE || width > MAX_SIDE || height > MAX_SIDE) return null;
+  let dpr = Number(vp.dpr);
+  if (!Number.isFinite(dpr) || dpr < 1) dpr = 1;
+  dpr = Math.min(dpr, MAX_DPR, Math.sqrt(MAX_PIXELS / (width * height)));
+  dpr = Math.max(1, round2(dpr));
+  return { width, height, dpr, mobile: vp.mobile === true, touch: vp.touch === true };
+}
+
+// ---------------------------------------------------------------------------
+// One DevTools connection
+
+async function openConnection() {
+  const endpoint = deps.endpoint();
+  if (!endpoint) throw Object.assign(new Error("Chrome is not in its own container"), { code: "unsupported" });
+  const version = await deps.fetchJson(`${endpoint}/json/version`);
+  const reported = version && typeof version.webSocketDebuggerUrl === "string" ? version.webSocketDebuggerUrl : "";
+  let path;
+  try {
+    path = new URL(reported).pathname;
+  } catch (_) {
+    path = "";
+  }
+  if (!/^\/devtools\/browser\/[A-Za-z0-9-]{1,80}$/.test(path)) throw new Error("Chrome gave no DevTools address");
+  // Loopback only: the address is built from AGENT_BROWSER_CDP, never from what Chrome reports.
+  const ws = deps.connect(`ws://${new URL(endpoint).host}${path}`);
+  const conn = { ws, nextId: 0, calls: new Map(), handlers: [], closed: false };
+  ws.on("message", (data) => {
+    let msg;
+    try { msg = JSON.parse(String(data)); } catch (_) { return; }
+    if (msg && msg.id && conn.calls.has(msg.id)) {
+      const call = conn.calls.get(msg.id);
+      conn.calls.delete(msg.id);
+      clearTimeout(call.timer);
+      if (msg.error) call.reject(new Error(`${call.method}: ${msg.error.message || "failed"}`));
+      else call.resolve(msg.result || {});
+      return;
+    }
+    if (msg && msg.method) for (const fn of conn.handlers) fn(msg);
+  });
+  const fail = () => {
+    if (conn.closed) return;
+    conn.closed = true;
+    for (const call of conn.calls.values()) { clearTimeout(call.timer); call.reject(new Error("DevTools connection closed")); }
+    conn.calls.clear();
+    for (const fn of conn.handlers) fn({ method: "_closed" });
+  };
+  ws.on("close", fail);
+  ws.on("error", fail);
+  await new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error("DevTools connection timed out")), CONNECT_TIMEOUT_MS);
+    if (timer.unref) timer.unref();
+    ws.on("open", () => { clearTimeout(timer); resolve(); });
+    ws.on("error", (e) => { clearTimeout(timer); reject(e instanceof Error ? e : new Error("DevTools connection failed")); });
+  });
+  return conn;
+}
+
+// A call that waits for Chrome's answer.
+function call(conn, method, params = {}, sessionId = null) {
+  if (!conn || conn.closed) return Promise.reject(new Error("DevTools connection closed"));
+  const id = ++conn.nextId;
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      conn.calls.delete(id);
+      reject(new Error(`${method} timed out`));
+    }, CALL_TIMEOUT_MS);
+    if (timer.unref) timer.unref();
+    conn.calls.set(id, { resolve, reject, timer, method });
+    try {
+      conn.ws.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) }));
+    } catch (e) {
+      clearTimeout(timer);
+      conn.calls.delete(id);
+      reject(e);
+    }
+  });
+}
+
+// A call whose answer nobody waits for (input, frame acks): sent in order, failures only logged.
+function fire(conn, method, params, sessionId) {
+  call(conn, method, params, sessionId).catch((e) => {
+    if (!conn.closed) console.error(`[browser] ${e.message}`);
+  });
+}
+
+function closeConnection(conn) {
+  if (!conn || conn.closed) return;
+  conn.closed = true;
+  for (const c of conn.calls.values()) { clearTimeout(c.timer); c.reject(new Error("DevTools connection closed")); }
+  conn.calls.clear();
+  try { conn.ws.close(); } catch (_) { /* already closed */ }
+}
+
+// ---------------------------------------------------------------------------
+// The page's text fields: where they are on the driver's screen (CSS pixels of the view, after
+// any zoom), what kind, and whether one has the focus. Never a value.
+
+const FIELDS_SCRIPT = `(() => {
+  const vv = window.visualViewport || { offsetLeft: 0, offsetTop: 0, scale: 1, width: innerWidth, height: innerHeight };
+  const sw = vv.width * vv.scale, sh = vv.height * vv.scale;
+  const SKIP = { hidden: 1, checkbox: 1, radio: 1, button: 1, submit: 1, reset: 1, image: 1, file: 1, range: 1, color: 1 };
+  const KINDS = { password: 1, email: 1, number: 1, tel: 1, url: 1, search: 1 };
+  const attr = (el, n, max) => String(el.getAttribute(n) || "").slice(0, max);
+  const out = [];
+  let focused = -1;
+  const active = document.activeElement;
+  const list = document.querySelectorAll('input, textarea, [contenteditable=""], [contenteditable="true"], [role="textbox"], [role="searchbox"], [role="combobox"]');
+  for (const el of list) {
+    if (out.length >= ${MAX_FIELDS}) break;
+    if (el.disabled || el.readOnly) continue;
+    let kind = "text";
+    if (el.tagName === "INPUT") {
+      const t = String(el.type || "text").toLowerCase();
+      if (SKIP[t]) continue;
+      if (KINDS[t]) kind = t;
+    } else if (el.tagName === "TEXTAREA") kind = "multiline";
+    const r = el.getBoundingClientRect();
+    if (r.width < 4 || r.height < 4) continue;
+    const x = (r.left - vv.offsetLeft) * vv.scale, y = (r.top - vv.offsetTop) * vv.scale;
+    const w = r.width * vv.scale, h = r.height * vv.scale;
+    if (x + w < 0 || y + h < 0 || x > sw || y > sh) continue;
+    const st = getComputedStyle(el);
+    if (st.visibility === "hidden" || st.display === "none" || Number(st.opacity) === 0) continue;
+    if (el === active) focused = out.length;
+    out.push({ x: Math.round(x), y: Math.round(y), w: Math.round(w), h: Math.round(h), kind,
+      ac: attr(el, "autocomplete", 40), im: attr(el, "inputmode", 16), enter: attr(el, "enterkeyhint", 10) });
+  }
+  const editable = !!(active && (active.isContentEditable || active.tagName === "TEXTAREA" ||
+    (active.tagName === "INPUT" && !SKIP[String(active.type || "text").toLowerCase()])));
+  return JSON.stringify({ fields: out, focused, editable });
+})()`;
+
+function cleanFields(raw) {
+  let parsed;
+  try { parsed = JSON.parse(String(raw)); } catch (_) { return null; }
+  if (!parsed || !Array.isArray(parsed.fields)) return null;
+  const word = (v, max) => (typeof v === "string" ? v.slice(0, max).replace(/[^\w -]/g, "") : "");
+  const fields = parsed.fields.slice(0, MAX_FIELDS).filter((f) => f && [f.x, f.y, f.w, f.h].every(Number.isFinite)).map((f) => ({
+    x: Math.round(f.x), y: Math.round(f.y), w: Math.round(f.w), h: Math.round(f.h),
+    kind: word(f.kind, 12) || "text", ac: word(f.ac, 40), im: word(f.im, 16), enter: word(f.enter, 10),
+  }));
+  const focused = Number.isInteger(parsed.focused) && parsed.focused >= 0 && parsed.focused < fields.length ? parsed.focused : -1;
+  return { fields, focused, editable: parsed.editable === true };
+}
+
+function scheduleFields(drive, ms = FIELDS_SETTLE_MS) {
+  if (!drive || drive.stopped) return;
+  clearTimeout(drive.fieldsTimer);
+  drive.fieldsTimer = setTimeout(() => readFields(drive), ms);
+  if (drive.fieldsTimer.unref) drive.fieldsTimer.unref();
+}
+
+async function readFields(drive) {
+  if (!drive || drive.stopped) return;
+  try {
+    const res = await call(drive.conn, "Runtime.evaluate", { expression: FIELDS_SCRIPT, returnByValue: true, silent: true }, drive.sid);
+    const value = res && res.result ? res.result.value : null;
+    const fields = cleanFields(value);
+    if (!fields || drive.stopped) return;
+    const key = JSON.stringify(fields);
+    if (key === drive.fieldsKey) return;
+    drive.fieldsKey = key;
+    view.driveFeed.emit("fields", { session: drive.session.name, ...fields });
+  } catch (_) {
+    // a page in the middle of loading: the next frame reads them again
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Frames
+
+function onFrame(drive, params) {
+  if (drive.stopped || !params || typeof params.data !== "string") return;
+  const md = params.metadata || {};
+  const vp = drive.viewport;
+  const frame = {
+    data: params.data,
+    // The driver's page is vp.width x vp.height CSS pixels; the picture is that times vp.dpr.
+    metadata: {
+      deviceWidth: vp.width,
+      deviceHeight: vp.height,
+      pageScaleFactor: Number.isFinite(md.pageScaleFactor) ? md.pageScaleFactor : 1,
+      scrollOffsetX: Number.isFinite(md.scrollOffsetX) ? md.scrollOffsetX : 0,
+      scrollOffsetY: Number.isFinite(md.scrollOffsetY) ? md.scrollOffsetY : 0,
+      timestamp: Number.isFinite(md.timestamp) ? Math.round(md.timestamp * 1000) : deps.now(),
+      fit: true,
+    },
+    seq: ++drive.seq,
+    at: deps.now(),
+  };
+  drive.lastFrame = frame;
+  view.driveFeed.last = { session: drive.session.name, frame };
+  view.driveFeed.emit("frame", { session: drive.session.name, frame });
+  // Chrome sends the next picture after this one is acknowledged: at most MAX_FPS a second.
+  const wait = Math.max(0, drive.lastAckAt + 1000 / MAX_FPS - deps.now());
+  const ack = () => {
+    drive.lastAckAt = deps.now();
+    if (!drive.stopped) fire(drive.conn, "Page.screencastFrameAck", { sessionId: params.sessionId }, drive.sid);
+  };
+  if (wait > 0) {
+    const t = setTimeout(ack, wait);
+    if (t.unref) t.unref();
+  } else ack();
+  scheduleFields(drive);
+}
+
+// After a page load, Chrome 150 on Linux draws the view at 1x again: the view keeps its density,
+// the picture does not (proven 10 Oct on the canary's Chrome 150.0.7871.124: 2x until the load,
+// 1x after; the same view sent again, a new screencast, or the view cleared and set again change
+// nothing; the window resized by a pixel and back brings 2x back). Chrome 149 on a Mac did the
+// opposite, 1x until the first load. So after every main-frame navigation and load, the window
+// is nudged by a pixel and back.
+function nudgeWindow(drive) {
+  if (!drive || drive.stopped || !drive.viewport || drive.windowId === null) return;
+  clearTimeout(drive.nudgeTimer);
+  drive.nudgeTimer = setTimeout(async () => {
+    if (drive.stopped || !drive.viewport) return;
+    const vp = drive.viewport;
+    const width = Math.round(vp.width * vp.dpr) + drive.insets.width;
+    const height = Math.round(vp.height * vp.dpr) + drive.insets.height;
+    try {
+      await call(drive.conn, "Browser.setWindowBounds", { windowId: drive.windowId, bounds: { width: width + 1, height, windowState: "normal" } });
+      await call(drive.conn, "Browser.setWindowBounds", { windowId: drive.windowId, bounds: { width, height, windowState: "normal" } });
+    } catch (_) {
+      // the next load nudges again
+    }
+  }, NUDGE_DELAY_MS);
+  if (drive.nudgeTimer.unref) drive.nudgeTimer.unref();
+}
+
+// ---------------------------------------------------------------------------
+// start / fit / stop
+
+async function applyViewport(drive, vp) {
+  const { conn, sid } = drive;
+  // The view first, then the window (the other way round, the first pictures stay at 1x).
+  await call(conn, "Emulation.setDeviceMetricsOverride", {
+    width: vp.width,
+    height: vp.height,
+    deviceScaleFactor: vp.dpr,
+    mobile: vp.mobile,
+    scale: vp.dpr,
+    screenWidth: vp.width,
+    screenHeight: vp.height,
+  }, sid);
+  await call(conn, "Browser.setWindowBounds", {
+    windowId: drive.windowId,
+    bounds: {
+      width: Math.round(vp.width * vp.dpr) + drive.insets.width,
+      height: Math.round(vp.height * vp.dpr) + drive.insets.height,
+      windowState: "normal",
+    },
+  });
+  await call(conn, "Emulation.setTouchEmulationEnabled", { enabled: vp.touch, ...(vp.touch ? { maxTouchPoints: 5 } : {}) }, sid);
+  drive.viewport = vp;
+  // A new size needs a new screencast: its picture size is fixed when it starts.
+  if (drive.screencasting) await call(conn, "Page.stopScreencast", {}, sid).catch(() => {});
+  await call(conn, "Page.startScreencast", {
+    format: "jpeg",
+    quality: QUALITY,
+    maxWidth: Math.round(vp.width * vp.dpr),
+    maxHeight: Math.round(vp.height * vp.dpr),
+    everyNthFrame: 1,
+  }, sid);
+  drive.screencasting = true;
+}
+
+// session: { name, dir } (an agent-browser session in container mode); viewport: the driver's
+// screen; savedBounds: the window's size before an earlier drive of this same lock (after a bot
+// restart), so the hand back restores that and not the phone size.
+// -> { ok: true, fit, bounds } | { ok: false, error }
+async function start({ session, viewport, savedBounds = null } = {}) {
+  const vp = checkViewport(viewport);
+  if (!vp) return { ok: false, error: "bad-viewport" };
+  if (!session || !session.name) return { ok: false, error: "not-running" };
+  if (!deps.endpoint()) return { ok: false, error: "unsupported" };
+  const targetId = deps.targetOf(session);
+  if (!targetId) return { ok: false, error: "no-tab" };
+  if (current) await stop();
+  let conn;
+  try {
+    conn = await openConnection();
+  } catch (e) {
+    console.error(`[browser] phone/web view unavailable: ${e.message}`);
+    return { ok: false, error: e.code === "unsupported" ? "unsupported" : "browser-unavailable" };
+  }
+  const drive = {
+    session: { name: session.name, dir: session.dir || null },
+    targetId,
+    conn,
+    sid: null,
+    windowId: null,
+    savedBounds: null,
+    insets: { width: 0, height: 0 },
+    viewport: null,
+    screencasting: false,
+    stopped: false,
+    lastFrame: null,
+    lastAckAt: 0,
+    seq: 0,
+    fieldsTimer: null,
+    fieldsKey: "",
+    touchBase: null,
+    nudgeTimer: null,
+  };
+  conn.handlers.push((msg) => {
+    if (msg.method === "Page.screencastFrame" && msg.sessionId === drive.sid) onFrame(drive, msg.params);
+    else if (msg.sessionId === drive.sid && ((msg.method === "Page.frameNavigated" && msg.params && msg.params.frame && !msg.params.frame.parentId) || msg.method === "Page.loadEventFired")) nudgeWindow(drive);
+    else if (msg.method === "_closed" || (msg.method === "Target.detachedFromTarget" && msg.params && msg.params.sessionId === drive.sid)) {
+      if (!drive.stopped) lost(drive);
+    }
+  });
+  try {
+    const attached = await call(conn, "Target.attachToTarget", { targetId, flatten: true });
+    drive.sid = attached.sessionId;
+    if (!drive.sid) throw new Error("no DevTools session for the tab");
+    await call(conn, "Target.activateTarget", { targetId }).catch(() => {});
+    const win = await call(conn, "Browser.getWindowForTarget", { targetId });
+    drive.windowId = win.windowId;
+    drive.savedBounds = savedBounds && Number.isFinite(savedBounds.width) && Number.isFinite(savedBounds.height)
+      ? { width: Math.round(savedBounds.width), height: Math.round(savedBounds.height) }
+      : { width: win.bounds.width, height: win.bounds.height };
+    // What the window adds around the page (Chrome's own bars, even headless).
+    const sizes = await call(conn, "Runtime.evaluate", {
+      expression: "JSON.stringify([outerWidth - innerWidth, outerHeight - innerHeight])",
+      returnByValue: true,
+      silent: true,
+    }, drive.sid);
+    try {
+      const [w, h] = JSON.parse(sizes.result.value);
+      if (Number.isFinite(w) && w >= 0 && w < 400) drive.insets.width = Math.round(w);
+      if (Number.isFinite(h) && h >= 0 && h < 400) drive.insets.height = Math.round(h);
+    } catch (_) { /* no insets known: none */ }
+    await call(conn, "Page.enable", {}, drive.sid);
+    await applyViewport(drive, vp);
+  } catch (e) {
+    console.error(`[browser] phone/web view failed: ${e.message}`);
+    await restore(drive);
+    return { ok: false, error: "browser-unavailable" };
+  }
+  current = drive;
+  view.driveFeed.active = drive.session.name;
+  view.driveFeed.emit("start", { session: drive.session.name, fit: vp });
+  scheduleFields(drive, 400);
+  console.log(`[browser] driver's view: ${vp.width}x${vp.height} at ${vp.dpr}x${vp.mobile ? ", phone layout" : ""}${vp.touch ? ", touch" : ""}`);
+  return { ok: true, fit: vp, bounds: drive.savedBounds };
+}
+
+// The driver's screen changed size (a turned phone, a resized panel). -> { ok, fit } | { ok: false, error }
+async function fit(viewport) {
+  const drive = current;
+  if (!drive || drive.stopped) return { ok: false, error: "not-driving" };
+  const vp = checkViewport(viewport);
+  if (!vp) return { ok: false, error: "bad-viewport" };
+  try {
+    await applyViewport(drive, vp);
+  } catch (e) {
+    console.error(`[browser] resizing the driver's view failed: ${e.message}`);
+    return { ok: false, error: "browser-unavailable" };
+  }
+  scheduleFields(drive, 400);
+  view.driveFeed.emit("start", { session: drive.session.name, fit: vp });
+  return { ok: true, fit: vp };
+}
+
+// Undo everything this drive changed. The emulation goes with the DevTools session anyway; the
+// window size does not.
+async function restore(drive) {
+  drive.stopped = true;
+  clearTimeout(drive.fieldsTimer);
+  clearTimeout(drive.nudgeTimer);
+  const { conn, sid } = drive;
+  if (conn && !conn.closed) {
+    if (sid && drive.screencasting) await call(conn, "Page.stopScreencast", {}, sid).catch(() => {});
+    if (sid) await call(conn, "Emulation.clearDeviceMetricsOverride", {}, sid).catch(() => {});
+    if (sid) await call(conn, "Emulation.setTouchEmulationEnabled", { enabled: false }, sid).catch(() => {});
+    if (drive.windowId !== null && drive.savedBounds) {
+      await call(conn, "Browser.setWindowBounds", { windowId: drive.windowId, bounds: { ...drive.savedBounds, windowState: "normal" } }).catch(() => {});
+    }
+  }
+  closeConnection(conn);
+}
+
+// The DevTools connection or the tab went away under a drive.
+function lost(drive) {
+  if (current === drive) {
+    current = null;
+    view.driveFeed.active = null;
+    view.driveFeed.last = null;
+  }
+  drive.stopped = true;
+  clearTimeout(drive.fieldsTimer);
+  clearTimeout(drive.nudgeTimer);
+  closeConnection(drive.conn);
+  view.driveFeed.emit("stop", { session: drive.session.name, reason: "lost" });
+  console.error("[browser] the driver's view lost its connection to Chrome");
+}
+
+async function stop() {
+  const drive = current;
+  if (!drive) return false;
+  current = null;
+  view.driveFeed.active = null;
+  view.driveFeed.last = null;
+  await restore(drive);
+  view.driveFeed.emit("stop", { session: drive.session.name, reason: "handed-back" });
+  return true;
+}
+
+// After a bot restart nothing is live, but the window may still be at the driver's size: put it
+// back from the size the lock remembers. Best effort.
+async function restoreWindow(session, bounds) {
+  if (current || !bounds || !Number.isFinite(bounds.width) || !Number.isFinite(bounds.height)) return false;
+  const targetId = session ? deps.targetOf(session) : null;
+  if (!targetId || !deps.endpoint()) return false;
+  let conn;
+  try {
+    conn = await openConnection();
+    const win = await call(conn, "Browser.getWindowForTarget", { targetId });
+    await call(conn, "Browser.setWindowBounds", { windowId: win.windowId, bounds: { width: Math.round(bounds.width), height: Math.round(bounds.height), windowState: "normal" } });
+    return true;
+  } catch (_) {
+    return false;
+  } finally {
+    closeConnection(conn);
+  }
+}
+
+function active(sessionName) {
+  return !!current && !current.stopped && (sessionName === undefined || current.session.name === sessionName);
+}
+
+function lastFrame() {
+  return current && current.lastFrame ? { data: current.lastFrame.data, metadata: current.lastFrame.metadata, at: current.lastFrame.at } : null;
+}
+
+// ---------------------------------------------------------------------------
+// Input: events already checked by core/browser-control.js checkEvent.
+
+// A touch's time on the driver's device (ms, their clock) moved onto this machine's clock, so a
+// swipe's speed (and its fling) survives the trip, in seconds as Chrome wants. A gesture's first
+// touch lands TOUCH_SLACK_MS in the past, so the touches that arrive with it (a batch spans about
+// one round trip) keep their spacing instead of being cut to "now".
+const TOUCH_SLACK_MS = 150;
+function touchTime(drive, ev) {
+  const now = deps.now();
+  if (!Number.isFinite(ev.t)) return now / 1000;
+  if (!drive.touchBase || (ev.eventType === "touchStart" && !drive.touching)) drive.touchBase = { offset: now - TOUCH_SLACK_MS - ev.t };
+  const at = ev.t + drive.touchBase.offset;
+  // Never in the future, never more than a few seconds behind.
+  return Math.min(now, Math.max(now - 5000, at)) / 1000;
+}
+
+// The driver's points are the page's CSS pixels; with the view drawn at its density (Emulation
+// scale), Chrome reads input in the drawn picture's pixels, so every point and scroll is scaled
+// by it (proven 10 Oct on Chrome 149: a tap at CSS (46,143) with scale 2 landed at (23,72)).
+function toCdp(drive, ev) {
+  const k = drive.viewport && drive.viewport.dpr > 0 ? drive.viewport.dpr : 1;
+  const at = (n) => Math.round(n * k * 100) / 100;
+  if (ev.type === "input_mouse") {
+    const p = { type: ev.eventType, x: at(ev.x), y: at(ev.y), button: ev.button || "none", clickCount: ev.clickCount || 0, modifiers: ev.modifiers || 0, pointerType: "mouse" };
+    if (ev.eventType === "mouseWheel") { p.deltaX = at(ev.deltaX || 0); p.deltaY = at(ev.deltaY || 0); }
+    return ["Input.dispatchMouseEvent", p];
+  }
+  if (ev.type === "input_keyboard") {
+    const p = { type: ev.eventType === "keyDown" && !ev.text ? "rawKeyDown" : ev.eventType, modifiers: ev.modifiers || 0 };
+    if (ev.key !== undefined) p.key = ev.key;
+    if (ev.code !== undefined) p.code = ev.code;
+    if (ev.text !== undefined) { p.text = ev.text; p.unmodifiedText = ev.text; }
+    if (ev.windowsVirtualKeyCode !== undefined) { p.windowsVirtualKeyCode = ev.windowsVirtualKeyCode; p.nativeVirtualKeyCode = ev.windowsVirtualKeyCode; }
+    return ["Input.dispatchKeyEvent", p];
+  }
+  if (ev.type === "input_touch") {
+    const ending = ev.eventType === "touchEnd" || ev.eventType === "touchCancel";
+    const p = {
+      type: ev.eventType,
+      // Chrome wants no points on an end or a cancel.
+      touchPoints: ending ? [] : ev.touchPoints.map((t, i) => ({ x: at(t.x), y: at(t.y), id: t.id !== undefined ? t.id : i, radiusX: 1, radiusY: 1, force: 1 })),
+      modifiers: ev.modifiers || 0,
+      timestamp: touchTime(drive, ev),
+    };
+    drive.touching = !ending;
+    return ["Input.dispatchTouchEvent", p];
+  }
+  if (ev.type === "input_text") return ["Input.insertText", { text: String(ev.text).slice(0, TEXT_MAX) }];
+  return null;
+}
+
+// Send a batch to Chrome in order without waiting for each answer. -> number sent
+function dispatch(events) {
+  const drive = current;
+  if (!drive || drive.stopped || drive.conn.closed) return -1;
+  let sent = 0;
+  let tapped = false;
+  for (const ev of events) {
+    const cdp = toCdp(drive, ev);
+    if (!cdp) continue;
+    fire(drive.conn, cdp[0], cdp[1], drive.sid);
+    sent += 1;
+    if (ev.eventType === "touchEnd" || ev.eventType === "mouseReleased" || ev.type === "input_text" || (ev.type === "input_keyboard" && ev.key === "Tab")) tapped = true;
+  }
+  // A tap or a Tab can move the focus: read the fields again soon.
+  if (tapped) scheduleFields(drive, 200);
+  return sent;
+}
+
+function _setDeps(overrides = {}) {
+  deps = { ...deps, ...overrides };
+}
+
+async function _reset() {
+  if (current) {
+    const drive = current;
+    current = null;
+    drive.stopped = true;
+    clearTimeout(drive.fieldsTimer);
+    closeConnection(drive.conn);
+  }
+  view.driveFeed.active = null;
+  view.driveFeed.last = null;
+  view.driveFeed.removeAllListeners();
+  deps = defaultDeps();
+}
+
+module.exports = {
+  FIELDS_SCRIPT,
+  MAX_FPS,
+  MAX_PIXELS,
+  QUALITY,
+  TEXT_MAX,
+  active,
+  checkViewport,
+  cleanFields,
+  dispatch,
+  fit,
+  lastFrame,
+  restoreWindow,
+  start,
+  stop,
+  supported: () => !!deps.endpoint(),
+  _reset,
+  _setDeps,
+};

package/core/browser-lock.js

@@ -4,7 +4,9 @@
 // hook (core/tool-guard.js, C7), which runs as a separate process for every tool call and
 // re-reads it each time, like the tooling-mode switch (core/tooling-mode.js).
 //
-// { session, userId, name, chatId, since, expiresAt }   (ISO times; written atomically, 0600)
+// { session, userId, name, chatId, since, expiresAt, fit?, bounds? }   (ISO times; atomic, 0600)
+// fit: the driver's view (core/browser-drive.js), bounds: the window's size before it, so a
+// restarted bot can bring the view back, and the hand back can put the window back.
 //
 // An expired lock is no lock: it never blocks the bot, and the sweeper in
 // core/browser-control.js hands the browser back.
@@ -48,4 +50,33 @@ function removeLock() {
   try { fs.rmSync(driverFile(), { force: true }); } catch (_) { /* already gone */ }
 }
 
-module.exports = { LOCK_TTL_MS, driverFile, readLock, lockExpiresAt, liveLock, writeLock, removeLock };
+// The bot was told to wait (core/tool-guard.js refused one of its agent-browser commands while
+// this lock was live), so the hand back must wake it. Keyed by the lock's `since`: a marker from
+// an earlier drive never counts. A separate file, because the tool guard runs in its own process
+// and the lock file is rewritten every second while someone drives.
+function waitingFile() {
+  return process.env.OC_BROWSER_WAITING_FILE || path.join(require("../config-dir"), "browser-waiting.json");
+}
+
+function markWaiting(lock) {
+  if (!lock || !lock.since) return;
+  try {
+    atomicWriteFileSync(waitingFile(), `${JSON.stringify({ since: lock.since, at: new Date().toISOString() })}\n`, { mode: 0o600 });
+  } catch (_) { /* best effort: the bot then just is not woken */ }
+}
+
+function waitedDuring(lock) {
+  if (!lock || !lock.since) return false;
+  try {
+    const mark = JSON.parse(fs.readFileSync(waitingFile(), "utf8"));
+    return !!mark && mark.since === lock.since;
+  } catch (_) {
+    return false;
+  }
+}
+
+function clearWaiting() {
+  try { fs.rmSync(waitingFile(), { force: true }); } catch (_) { /* already gone */ }
+}
+
+module.exports = { LOCK_TTL_MS, driverFile, readLock, lockExpiresAt, liveLock, writeLock, removeLock, markWaiting, waitedDuring, clearWaiting };