@inetafrica/open-claudia 3.5.23 → 3.5.25

package/core/browser-control.js

@@ -7,10 +7,21 @@
 //     the browser. It lasts 120 s; the page's heartbeat and every accepted input extend it.
 //   - one input WebSocket per agent-browser session (ws://127.0.0.1:<port>/?maxFps=2): events
 //     are checked strictly, field by field, and forwarded in order. The same socket keeps the
-//     latest frame, which Teach me saves as a click's picture.
+//     latest frame, which Teach me saves as a click's (or a navigation's) picture.
+//   - the address bar: go to an address, back, forward, reload. The stream socket only takes
+//     input, so these are agent-browser commands on the same session, run in the session's queue
+//     after the input before them, with the bot's own agent-browser environment and never a
+//     launch flag (core/agent-browser-cli.js: other launch settings would restart the browser).
+//     Only public internet addresses (core/browser-address.js).
+//   - the phone and web view (core/browser-drive.js): a take or a "fit" with the driver's screen
+//     makes the bot's tab take its size, phone layout and touch, and then the driver's input goes
+//     straight to Chrome through that view (typed text too: input_text). Only with Chrome in its
+//     own container; elsewhere the browser keeps its size and input goes through agent-browser.
 //   - hand back (release, a closed panel, or a lock found expired by the 30 s sweeper): the lock
-//     goes, the socket closes, any recording stops, and the bot is woken in the person's own chat
-//     to carry on (C6). A recording stopped at the same time folds into one wake-up.
+//     goes, the view and the socket close, any recording stops, and the bot is woken in the
+//     person's own chat (C6), but only when it waits for them: it sent them an Open Browser card,
+//     or one of its browser commands was refused while they drove. A recording stopped at the
+//     same time folds into one wake-up. Otherwise the hand back is only logged.
 //
 // Every route answers { status, body }; web.js writes it. Agent Space reaches these routes with
 // the control token only.
@@ -19,6 +30,9 @@ const WebSocket = require("ws");
 const lockFile = require("./browser-lock");
 const view = require("./browser-view");
 const teach = require("./browser-teach");
+const address = require("./browser-address");
+const drive = require("./browser-drive");
+const { runAgentBrowser } = require("./agent-browser-cli");
 
 const LOCK_TTL_MS = lockFile.LOCK_TTL_MS;
 // Inputs extend the lock; the file (fsynced) is rewritten at most this often, so the lock ends
@@ -31,19 +45,32 @@ const MAX_EVENTS = 50;
 const MAX_PENDING_EVENTS = 1000;
 const COORD_MAX = 10000;
 const ID_RE = /^[A-Za-z0-9._:-]{1,100}$/;
+// An address-bar command still running after this is killed, and the person told it failed.
+const NAVIGATE_TIMEOUT_MS = 20 * 1000;
+const NAVIGATIONS = new Set(["open", "back", "forward", "reload"]);
+
+// An Open Browser card counts as a request for a person this long.
+const REQUEST_MAX_AGE_MS = 24 * 60 * 60 * 1000;
 
 const HAND_BACK_REASONS = {
   closed: " (they closed the Browser)",
   expired: " (their Browser went quiet for 2 minutes, so control came back on its own)",
 };
 
-let deps = {
-  now: () => Date.now(),
-  listSessions: () => view.listSessions(),
-  connect: (url) => new WebSocket(url),
-  scheduler: null, // core/scheduler by default (through core/cards.js wakeOrigin)
-  runFor: null, // the chat's provider, settings and live session; activeRun() by default
-};
+function defaultDeps() {
+  return {
+    now: () => Date.now(),
+    listSessions: () => view.listSessions(),
+    connect: (url) => new WebSocket(url),
+    scheduler: null, // core/scheduler by default (through core/cards.js wakeOrigin)
+    runFor: null, // the chat's provider, settings and live session; activeRun() by default
+    execFile: null, // child_process.execFile by default (core/agent-browser-cli.js)
+    lookupHost: null, // dns.promises.lookup by default (core/browser-address.js)
+    navigateTimeoutMs: NAVIGATE_TIMEOUT_MS,
+  };
+}
+
+let deps = defaultDeps();
 
 function reply(status, body) {
   return { status, body };
@@ -70,6 +97,19 @@ function sameUser(lock, who) {
   return !!lock && !!who && String(lock.userId) === String(who.userId);
 }
 
+// The live lock this person holds on this session, extended (every accepted input or navigation
+// extends it; the file is rewritten at most every LOCK_REWRITE_MS), or null.
+function extendDriverLock(session, who) {
+  const now = deps.now();
+  const lock = lockFile.liveLock(now);
+  if (!lock || lock.session !== session || !sameUser(lock, who)) return null;
+  if (lockFile.lockExpiresAt(lock) - now < LOCK_TTL_MS - LOCK_REWRITE_MS) {
+    lock.expiresAt = iso(now + LOCK_TTL_MS);
+    lockFile.writeLock(lock);
+  }
+  return lock;
+}
+
 // ---------------------------------------------------------------------------
 // C3: input events, checked field by field. Only the listed fields are forwarded.
 
@@ -77,6 +117,7 @@ const MOUSE_EVENTS = new Set(["mousePressed", "mouseReleased", "mouseMoved", "mo
 const MOUSE_BUTTONS = new Set(["none", "left", "middle", "right"]);
 const KEY_EVENTS = new Set(["keyDown", "keyUp", "char"]);
 const TOUCH_EVENTS = new Set(["touchStart", "touchMove", "touchEnd", "touchCancel"]);
+const TEXT_MAX = drive.TEXT_MAX;
 
 const isNumber = (v) => typeof v === "number" && Number.isFinite(v);
 const inRange = (v, lo, hi) => isNumber(v) && v >= lo && v <= hi;
@@ -133,11 +174,34 @@ function checkEvent(ev) {
     }
     const out = { type: "input_touch", eventType: ev.eventType, touchPoints };
     if (!optional(out, "modifiers", modifiers)) return null;
+    // When the finger moved, on the driver's device (ms): keeps a swipe's speed (core/browser-drive.js).
+    if (!optional(out, "t", (v) => inRange(v, 0, 1e13))) return null;
     return out;
   }
+  // Text the driver typed (their phone keyboard, a paste), inserted where the page's caret is.
+  if (ev.type === "input_text") {
+    if (typeof ev.text !== "string" || !ev.text || Array.from(ev.text).length > TEXT_MAX) return null;
+    return { type: "input_text", text: ev.text };
+  }
   return null;
 }
 
+// input_text as key presses, for a bot without the driver's view (agent-browser's socket takes
+// no text) and for Teach me, which counts typing from key presses.
+function textAsKeys(text) {
+  const out = [];
+  for (const ch of Array.from(text)) {
+    if (ch === "\n" || ch === "\r") {
+      out.push({ type: "input_keyboard", eventType: "keyDown", key: "Enter", code: "Enter", text: "\r", windowsVirtualKeyCode: 13 });
+      out.push({ type: "input_keyboard", eventType: "keyUp", key: "Enter", code: "Enter", windowsVirtualKeyCode: 13 });
+    } else {
+      out.push({ type: "input_keyboard", eventType: "keyDown", key: ch, text: ch });
+      out.push({ type: "input_keyboard", eventType: "keyUp", key: ch });
+    }
+  }
+  return out;
+}
+
 // ---------------------------------------------------------------------------
 // Input sockets: one per agent-browser session, opened on take (or on the first input after a
 // restart), closed on hand back.
@@ -295,7 +359,23 @@ function wakeChat({ chatId, userId }, prompt, label) {
 
 function handBackLine(name, reason) {
   const who = teach.cleanName(name) || "Someone";
-  return `[Browser] ${who} handed the browser back${HAND_BACK_REASONS[reason] || ""}. If you were waiting on them, check the page first (it may have changed) and carry on; otherwise just note it.`;
+  return `[Browser] ${who} handed the browser back${HAND_BACK_REASONS[reason] || ""}. Check the page first (it may have changed), then carry on with what you were waiting on them for.`;
+}
+
+// Was the bot waiting for this person when they handed back? It asked for them with an Open
+// Browser card in their chat (since the day before they took over), or one of its own browser
+// commands was refused while they drove.
+function botWasWaiting(lock) {
+  let asked = [];
+  try {
+    const since = Date.parse(lock.since);
+    asked = require("./cards").takeBrowserRequests(lock.chatId, (Number.isFinite(since) ? since : deps.now()) - REQUEST_MAX_AGE_MS, deps.now());
+  } catch (e) {
+    console.error(`[browser] reading the bot's Open Browser cards failed: ${e.message}`);
+  }
+  const refused = lockFile.waitedDuring(lock);
+  lockFile.clearWaiting();
+  return asked.length > 0 || refused;
 }
 
 // ---------------------------------------------------------------------------
@@ -305,7 +385,39 @@ function lockBody(lock) {
   return { driving: true, driver: { name: lock.name } };
 }
 
-async function take({ session, who } = {}) {
+// The driver's view on this lock's session: start it (or move it to a new screen) and remember it
+// in the lock, so a restarted bot can bring it back. -> the fit, or null (the browser keeps its size).
+async function startView(lock, viewport) {
+  if (!viewport || !drive.supported()) return null;
+  const info = sessionInfo(lock.session);
+  if (!info) return null;
+  const r = drive.active(lock.session)
+    ? await drive.fit(viewport)
+    : await drive.start({ session: info, viewport, savedBounds: lock.bounds || null });
+  const now = lockFile.readLock();
+  // The lock changed hands (or was handed back) while the view started: undo it.
+  if (!now || now.since !== lock.since || now.userId !== lock.userId) {
+    if (r.ok && drive.active(lock.session)) await drive.stop();
+    return null;
+  }
+  if (!r.ok) return null;
+  now.fit = r.fit;
+  if (r.bounds) now.bounds = r.bounds;
+  lockFile.writeLock(now);
+  return r.fit;
+}
+
+// After a bot restart the lock remembers a view that is no longer live: bring it back. Once at a
+// time, in the background.
+let reviving = null;
+function reviveView(lock) {
+  if (!lock || !lock.fit || reviving || drive.active(lock.session) || !drive.supported()) return;
+  reviving = startView(lock, lock.fit)
+    .catch((e) => console.error(`[browser] bringing the driver's view back failed: ${e.message}`))
+    .finally(() => { reviving = null; });
+}
+
+async function take({ session, who, viewport } = {}) {
   if (!validSession(session) || !validWho(who, { needChat: true })) return reply(400, { error: "bad-request" });
   // A lock that ran out but has not been swept yet is handed back first, the sweeper's way: its
   // driver's chat is woken and their recording stopped, whoever takes over now.
@@ -324,9 +436,25 @@ async function take({ session, who } = {}) {
     since: current && current.session === session ? current.since : iso(now),
     expiresAt: iso(now + LOCK_TTL_MS),
   };
+  // The same person taking again (a reloaded page) keeps the window's size from before their view.
+  if (current && current.session === session && sameUser(current, who) && current.bounds) lock.bounds = current.bounds;
   lockFile.writeLock(lock);
   openInputSocket(session);
-  return reply(200, lockBody(lock));
+  if (!viewport) return reply(200, lockBody(lock));
+  // fit: the driver's view now live, or null (this bot's browser keeps its own size).
+  const fitted = await startView(lock, viewport);
+  return reply(200, { ...lockBody(lock), fit: fitted });
+}
+
+// The driver's screen changed (a turned phone, a resized panel): { session, who, viewport }.
+async function fitView({ session, who, viewport } = {}) {
+  if (!validSession(session) || !validWho(who)) return reply(400, { error: "bad-request" });
+  if (!drive.checkViewport(viewport)) return reply(400, { error: "bad-viewport" });
+  const lock = extendDriverLock(session, who);
+  if (!lock) return reply(409, { driving: false, error: "not-driving" });
+  if (!drive.supported()) return reply(200, { ...lockBody(lock), fit: null });
+  const fitted = await startView(lock, viewport);
+  return reply(200, { ...lockBody(lock), fit: fitted });
 }
 
 function renew({ session, who } = {}) {
@@ -339,7 +467,8 @@ function renew({ session, who } = {}) {
   current.expiresAt = iso(now + LOCK_TTL_MS);
   lockFile.writeLock(current);
   openInputSocket(current.session); // after a restart the socket comes back here
-  return reply(200, lockBody(current));
+  reviveView(current); // and the driver's view
+  return reply(200, current.fit ? { ...lockBody(current), fit: drive.active(current.session) ? current.fit : null } : lockBody(current));
 }
 
 // Hand the browser back: claim the lock (synchronously, so a release and the sweeper never both
@@ -353,6 +482,13 @@ async function handBack(expected, { reason = null } = {}) {
   let stopped = null;
   await runQueued(current.session, 0, async () => {
     closeInputSocket(current.session);
+    // The bot's tab goes back to its own size before the bot carries on.
+    try {
+      if (drive.active(current.session)) await drive.stop();
+      else if (current.bounds) await drive.restoreWindow(info, current.bounds);
+    } catch (e) {
+      console.error(`[browser] ending the driver's view failed: ${e.message}`);
+    }
     try {
       stopped = await teach.stopRecording(current.userId, { session: info });
     } catch (e) {
@@ -360,9 +496,10 @@ async function handBack(expected, { reason = null } = {}) {
     }
   });
   const line = handBackLine(current.name, reason);
+  const waited = botWasWaiting(current);
   if (stopped && !stopped.discarded) wakeChat(current, teach.teachPrompt(stopped, { handBack: line }), "Teach me");
-  else wakeChat(current, line, "Browser");
-  console.log(`[browser] ${current.userId} handed the browser back${reason ? ` (${reason})` : ""}`);
+  else if (waited) wakeChat(current, line, "Browser");
+  console.log(`[browser] ${current.userId} handed the browser back${reason ? ` (${reason})` : ""}${waited || (stopped && !stopped.discarded) ? "" : "; the bot was not waiting for them, so it is not woken"}`);
   return true;
 }
 
@@ -379,15 +516,20 @@ async function control(body = {}) {
   if (body.action === "take") return take(body);
   if (body.action === "renew") return renew(body);
   if (body.action === "release") return release(body);
+  if (body.action === "fit") return fitView(body);
   return reply(400, { error: "bad-action" });
 }
 
+// navigate: true tells the front door this bot has the address bar (POST /api/browser/navigate);
+// fit: true that a take can bring the driver's view (core/browser-drive.js) and input_text works.
 function status() {
   const lock = lockFile.liveLock(deps.now());
   return reply(200, {
     driving: !!lock,
     driver: lock ? { name: lock.name, userId: lock.userId } : null,
     recording: teach.activeSummary(),
+    navigate: true,
+    fit: drive.supported(),
   });
 }
 
@@ -403,31 +545,42 @@ async function input({ session, events, who } = {}) {
     if (!msg) return reply(400, { error: "bad-event", index: i });
     messages.push(msg);
   }
-  const now = deps.now();
-  const lock = lockFile.liveLock(now);
-  if (!lock || lock.session !== session || !sameUser(lock, who)) return reply(409, { error: "not-driving" });
-  // Every accepted input extends the lock; the file is rewritten only every few seconds.
-  if (lockFile.lockExpiresAt(lock) - now < LOCK_TTL_MS - LOCK_REWRITE_MS) {
-    lock.expiresAt = iso(now + LOCK_TTL_MS);
-    lockFile.writeLock(lock);
-  }
+  // Every accepted input extends the lock.
+  const lock = extendDriverLock(session, who);
+  if (!lock) return reply(409, { error: "not-driving" });
   if (!messages.length) return reply(200, { ok: true, accepted: 0 });
+  if (lock.fit && !drive.active(session)) reviveView(lock);
   const entry = openInputSocket(session);
-  if (!entry) return reply(404, { error: "not-running" });
+  if (!entry && !drive.active(session)) return reply(404, { error: "not-running" });
   const job = runQueued(session, messages.length, async () => {
-    if (!(await socketReady(entry))) return reply(503, { error: "browser-unavailable", accepted: 0 });
     const recorder = teach.recorderFor(session, who.userId);
-    let sent = 0;
-    for (const msg of messages) {
-      if (recorder) {
+    const record = async (msg, frame, info) => {
+      if (!recorder) return;
+      // Teach me counts typing from key presses: typed text is shown to it as keys.
+      for (const ev of msg.type === "input_text" ? textAsKeys(msg.text) : [msg]) {
         try {
-          await recorder.onEvent(msg, { session: { name: entry.name, dir: entry.dir }, frame: entry.frame });
+          await recorder.onEvent(ev, { session: info, frame });
         } catch (e) {
           console.error(`[browser] recording a step failed: ${e.message}`);
         }
       }
+    };
+    // The driver's view: straight to Chrome, in order, without waiting for each answer.
+    if (drive.active(session)) {
+      const info = sessionInfo(session) || { name: session, dir: entry ? entry.dir : null };
+      for (const msg of messages) {
+        await record(msg, drive.lastFrame(), { name: info.name, dir: info.dir });
+        if (drive.dispatch([msg]) < 0) return reply(503, { error: "browser-unavailable" });
+      }
+      return reply(200, { ok: true, accepted: messages.length });
+    }
+    if (!entry || !(await socketReady(entry))) return reply(503, { error: "browser-unavailable", accepted: 0 });
+    let sent = 0;
+    for (const msg of messages) {
+      await record(msg, entry.frame, { name: entry.name, dir: entry.dir });
       if (!entry.open || entry.closed) return reply(503, { error: "browser-unavailable", accepted: sent });
-      entry.ws.send(JSON.stringify(msg));
+      // agent-browser's socket takes no text: typed text goes as key presses.
+      for (const ev of msg.type === "input_text" ? textAsKeys(msg.text) : [msg]) entry.ws.send(JSON.stringify(ev));
       sent += 1;
     }
     return reply(200, { ok: true, accepted: sent });
@@ -436,6 +589,72 @@ async function input({ session, events, who } = {}) {
   return job;
 }
 
+// ---------------------------------------------------------------------------
+// The address bar: open an address, back, forward, reload
+
+// Log words. An address is cut to origin + path: query strings and fragments can carry tokens.
+function navigationWords(action, href) {
+  if (action === "open") {
+    const where = view.safePageAddress(href) || "an address";
+    return { done: `went to ${where}`, failed: `go to ${where}` };
+  }
+  if (action === "reload") return { done: "reloaded the page", failed: "reload the page" };
+  return { done: `went ${action}`, failed: `go ${action}` };
+}
+
+// { session, action: "open" | "back" | "forward" | "reload", url (open), who: { userId } }
+async function navigate({ session, action, url, who } = {}) {
+  if (!validSession(session) || !validWho(who)) return reply(400, { error: "bad-request" });
+  if (!NAVIGATIONS.has(action)) return reply(400, { error: "bad-action" });
+  let target = null;
+  if (action === "open") {
+    target = address.parseAddress(url);
+    if (target.error) return reply(target.error === "bad-url" ? 400 : 403, { error: target.error });
+  }
+  // Like an input, a navigation needs the live lock and extends it.
+  if (!extendDriverLock(session, who)) return reply(409, { error: "not-driving" });
+  if (!openInputSocket(session)) return reply(404, { error: "not-running" });
+  // The name is resolved from now on, while the input queued before this still runs.
+  const resolving = target && target.lookupName
+    ? address.resolveAddress(target.lookupName, { lookup: deps.lookupHost || undefined })
+    : null;
+  const href = target ? target.url.href : null;
+  const job = runQueued(session, 1, async () => {
+    if (resolving) {
+      const found = await resolving;
+      if (found.error) return reply(found.error === "unknown-host" ? 400 : 403, { error: found.error });
+    }
+    // agent-browser starts a new browser for a session it does not know: one that ended while
+    // this waited is not asked.
+    const info = sessionInfo(session);
+    if (!info || !info.dir) return reply(404, { error: "not-running" });
+    const recorder = teach.recorderFor(session, who.userId);
+    if (recorder) {
+      const live = sockets.get(session);
+      try {
+        await recorder.onNavigate({ how: action, to: href }, { session: { name: info.name, dir: info.dir }, frame: live && !live.closed ? live.frame : null });
+      } catch (e) {
+        console.error(`[browser] recording a step failed: ${e.message}`);
+      }
+    }
+    const words = navigationWords(action, href);
+    const run = await runAgentBrowser(info, action === "open" ? ["open", href] : [action], {
+      timeoutMs: deps.navigateTimeoutMs,
+      execFile: deps.execFile || undefined,
+    });
+    if (!run.ok) {
+      // Never the command's own output or error: both can repeat the whole address.
+      const why = run.timedOut ? `agent-browser took longer than ${Math.ceil(deps.navigateTimeoutMs / 1000)} s and was stopped` : "agent-browser failed";
+      console.error(`[browser] ${who.userId} could not ${words.failed}: ${why}`);
+      return reply(503, { error: "browser-unavailable" });
+    }
+    console.log(`[browser] ${who.userId} ${words.done}`);
+    return reply(200, { ok: true });
+  });
+  if (!job) return reply(429, { error: "busy" });
+  return job;
+}
+
 // ---------------------------------------------------------------------------
 // C4: Teach me, start / stop / cancel
 
@@ -501,7 +720,8 @@ function startSweeper({ intervalMs = SWEEP_INTERVAL_MS } = {}) {
   return sweepTimer;
 }
 
-// Tests swap the clock, the session list, the socket factory, the scheduler and the run lookup.
+// Tests swap the clock, the session list, the socket factory, the scheduler, the run lookup, the
+// agent-browser call (execFile), the DNS lookup and the address-bar time limit.
 function _setDeps(overrides = {}) {
   deps = { ...deps, ...overrides };
 }
@@ -509,24 +729,23 @@ function _reset() {
   for (const name of [...sockets.keys()]) closeInputSocket(name);
   queues.clear();
   if (sweepTimer) { clearInterval(sweepTimer); sweepTimer = null; }
-  deps = {
-    now: () => Date.now(),
-    listSessions: () => view.listSessions(),
-    connect: (url) => new WebSocket(url),
-    scheduler: null,
-    runFor: null,
-  };
+  reviving = null;
+  drive._reset();
+  deps = defaultDeps();
 }
 
 module.exports = {
   LOCK_TTL_MS,
   MAX_EVENTS,
+  NAVIGATE_TIMEOUT_MS,
   SWEEP_INTERVAL_MS,
   checkEvent,
   chatOrigin,
   control,
+  fitView,
   handBackLine,
   input,
+  navigate,
   release,
   renew,
   startSweeper,