@inetafrica/open-claudia 3.5.27 → 3.5.28

package/core/browser-control.js

@@ -13,18 +13,30 @@
 //     after the input before them, with the bot's own agent-browser environment and never a
 //     launch flag (core/agent-browser-cli.js: other launch settings would restart the browser).
 //     Only public internet addresses (core/browser-address.js).
+//   - the phone and web view (core/browser-drive.js): a take or a "fit" with the driver's screen
+//     makes the bot's tab take its size, phone layout and touch, and then the driver's input goes
+//     straight to Chrome through that view (typed text too: input_text). Only with Chrome in its
+//     own container; elsewhere the browser keeps its size and input goes through agent-browser.
+//   - always on: a person who opens the Browser app while the bot's browser is not open gets it
+//     opened on a start page, in the bot's own default session (the one its `agent-browser open`
+//     uses), with the bot's own agent-browser environment, so the bot carries on in that tab.
 //   - hand back (release, a closed panel, or a lock found expired by the 30 s sweeper): the lock
-//     goes, the socket closes, any recording stops, and the bot is woken in the person's own chat
-//     to carry on (C6). A recording stopped at the same time folds into one wake-up.
+//     goes, the view and the socket close, any recording stops, and the bot is woken in the
+//     person's own chat (C6), but only when it waits for them: it sent them an Open Browser card,
+//     or one of its browser commands was refused while they drove. A recording stopped at the
+//     same time folds into one wake-up. Otherwise the hand back is only logged.
 //
 // Every route answers { status, body }; web.js writes it. Agent Space reaches these routes with
 // the control token only.
 
+const os = require("os");
+const path = require("path");
 const WebSocket = require("ws");
 const lockFile = require("./browser-lock");
 const view = require("./browser-view");
 const teach = require("./browser-teach");
 const address = require("./browser-address");
+const drive = require("./browser-drive");
 const { runAgentBrowser } = require("./agent-browser-cli");
 
 const LOCK_TTL_MS = lockFile.LOCK_TTL_MS;
@@ -41,6 +53,13 @@ const ID_RE = /^[A-Za-z0-9._:-]{1,100}$/;
 // An address-bar command still running after this is killed, and the person told it failed.
 const NAVIGATE_TIMEOUT_MS = 20 * 1000;
 const NAVIGATIONS = new Set(["open", "back", "forward", "reload"]);
+// Always on: the page a browser opened for a person starts on, and the session it opens (the one
+// the bot's own `agent-browser` commands use when they name none).
+const START_PAGE = "https://www.google.com/";
+const START_SESSION = "default";
+
+// An Open Browser card counts as a request for a person this long.
+const REQUEST_MAX_AGE_MS = 24 * 60 * 60 * 1000;
 
 const HAND_BACK_REASONS = {
   closed: " (they closed the Browser)",
@@ -107,6 +126,7 @@ const MOUSE_EVENTS = new Set(["mousePressed", "mouseReleased", "mouseMoved", "mo
 const MOUSE_BUTTONS = new Set(["none", "left", "middle", "right"]);
 const KEY_EVENTS = new Set(["keyDown", "keyUp", "char"]);
 const TOUCH_EVENTS = new Set(["touchStart", "touchMove", "touchEnd", "touchCancel"]);
+const TEXT_MAX = drive.TEXT_MAX;
 
 const isNumber = (v) => typeof v === "number" && Number.isFinite(v);
 const inRange = (v, lo, hi) => isNumber(v) && v >= lo && v <= hi;
@@ -163,11 +183,34 @@ function checkEvent(ev) {
     }
     const out = { type: "input_touch", eventType: ev.eventType, touchPoints };
     if (!optional(out, "modifiers", modifiers)) return null;
+    // When the finger moved, on the driver's device (ms): keeps a swipe's speed (core/browser-drive.js).
+    if (!optional(out, "t", (v) => inRange(v, 0, 1e13))) return null;
     return out;
   }
+  // Text the driver typed (their phone keyboard, a paste), inserted where the page's caret is.
+  if (ev.type === "input_text") {
+    if (typeof ev.text !== "string" || !ev.text || Array.from(ev.text).length > TEXT_MAX) return null;
+    return { type: "input_text", text: ev.text };
+  }
   return null;
 }
 
+// input_text as key presses, for a bot without the driver's view (agent-browser's socket takes
+// no text) and for Teach me, which counts typing from key presses.
+function textAsKeys(text) {
+  const out = [];
+  for (const ch of Array.from(text)) {
+    if (ch === "\n" || ch === "\r") {
+      out.push({ type: "input_keyboard", eventType: "keyDown", key: "Enter", code: "Enter", text: "\r", windowsVirtualKeyCode: 13 });
+      out.push({ type: "input_keyboard", eventType: "keyUp", key: "Enter", code: "Enter", windowsVirtualKeyCode: 13 });
+    } else {
+      out.push({ type: "input_keyboard", eventType: "keyDown", key: ch, text: ch });
+      out.push({ type: "input_keyboard", eventType: "keyUp", key: ch });
+    }
+  }
+  return out;
+}
+
 // ---------------------------------------------------------------------------
 // Input sockets: one per agent-browser session, opened on take (or on the first input after a
 // restart), closed on hand back.
@@ -325,7 +368,23 @@ function wakeChat({ chatId, userId }, prompt, label) {
 
 function handBackLine(name, reason) {
   const who = teach.cleanName(name) || "Someone";
-  return `[Browser] ${who} handed the browser back${HAND_BACK_REASONS[reason] || ""}. If you were waiting on them, check the page first (it may have changed) and carry on; otherwise just note it.`;
+  return `[Browser] ${who} handed the browser back${HAND_BACK_REASONS[reason] || ""}. Check the page first (it may have changed), then carry on with what you were waiting on them for.`;
+}
+
+// Was the bot waiting for this person when they handed back? It asked for them with an Open
+// Browser card in their chat (since the day before they took over), or one of its own browser
+// commands was refused while they drove.
+function botWasWaiting(lock) {
+  let asked = [];
+  try {
+    const since = Date.parse(lock.since);
+    asked = require("./cards").takeBrowserRequests(lock.chatId, (Number.isFinite(since) ? since : deps.now()) - REQUEST_MAX_AGE_MS, deps.now());
+  } catch (e) {
+    console.error(`[browser] reading the bot's Open Browser cards failed: ${e.message}`);
+  }
+  const refused = lockFile.waitedDuring(lock);
+  lockFile.clearWaiting();
+  return asked.length > 0 || refused;
 }
 
 // ---------------------------------------------------------------------------
@@ -335,7 +394,39 @@ function lockBody(lock) {
   return { driving: true, driver: { name: lock.name } };
 }
 
-async function take({ session, who } = {}) {
+// The driver's view on this lock's session: start it (or move it to a new screen) and remember it
+// in the lock, so a restarted bot can bring it back. -> the fit, or null (the browser keeps its size).
+async function startView(lock, viewport) {
+  if (!viewport || !drive.supported()) return null;
+  const info = sessionInfo(lock.session);
+  if (!info) return null;
+  const r = drive.active(lock.session)
+    ? await drive.fit(viewport)
+    : await drive.start({ session: info, viewport, savedBounds: lock.bounds || null });
+  const now = lockFile.readLock();
+  // The lock changed hands (or was handed back) while the view started: undo it.
+  if (!now || now.since !== lock.since || now.userId !== lock.userId) {
+    if (r.ok && drive.active(lock.session)) await drive.stop();
+    return null;
+  }
+  if (!r.ok) return null;
+  now.fit = r.fit;
+  if (r.bounds) now.bounds = r.bounds;
+  lockFile.writeLock(now);
+  return r.fit;
+}
+
+// After a bot restart the lock remembers a view that is no longer live: bring it back. Once at a
+// time, in the background.
+let reviving = null;
+function reviveView(lock) {
+  if (!lock || !lock.fit || reviving || drive.active(lock.session) || !drive.supported()) return;
+  reviving = startView(lock, lock.fit)
+    .catch((e) => console.error(`[browser] bringing the driver's view back failed: ${e.message}`))
+    .finally(() => { reviving = null; });
+}
+
+async function take({ session, who, viewport } = {}) {
   if (!validSession(session) || !validWho(who, { needChat: true })) return reply(400, { error: "bad-request" });
   // A lock that ran out but has not been swept yet is handed back first, the sweeper's way: its
   // driver's chat is woken and their recording stopped, whoever takes over now.
@@ -354,9 +445,25 @@ async function take({ session, who } = {}) {
     since: current && current.session === session ? current.since : iso(now),
     expiresAt: iso(now + LOCK_TTL_MS),
   };
+  // The same person taking again (a reloaded page) keeps the window's size from before their view.
+  if (current && current.session === session && sameUser(current, who) && current.bounds) lock.bounds = current.bounds;
   lockFile.writeLock(lock);
   openInputSocket(session);
-  return reply(200, lockBody(lock));
+  if (!viewport) return reply(200, lockBody(lock));
+  // fit: the driver's view now live, or null (this bot's browser keeps its own size).
+  const fitted = await startView(lock, viewport);
+  return reply(200, { ...lockBody(lock), fit: fitted });
+}
+
+// The driver's screen changed (a turned phone, a resized panel): { session, who, viewport }.
+async function fitView({ session, who, viewport } = {}) {
+  if (!validSession(session) || !validWho(who)) return reply(400, { error: "bad-request" });
+  if (!drive.checkViewport(viewport)) return reply(400, { error: "bad-viewport" });
+  const lock = extendDriverLock(session, who);
+  if (!lock) return reply(409, { driving: false, error: "not-driving" });
+  if (!drive.supported()) return reply(200, { ...lockBody(lock), fit: null });
+  const fitted = await startView(lock, viewport);
+  return reply(200, { ...lockBody(lock), fit: fitted });
 }
 
 function renew({ session, who } = {}) {
@@ -369,7 +476,8 @@ function renew({ session, who } = {}) {
   current.expiresAt = iso(now + LOCK_TTL_MS);
   lockFile.writeLock(current);
   openInputSocket(current.session); // after a restart the socket comes back here
-  return reply(200, lockBody(current));
+  reviveView(current); // and the driver's view
+  return reply(200, current.fit ? { ...lockBody(current), fit: drive.active(current.session) ? current.fit : null } : lockBody(current));
 }
 
 // Hand the browser back: claim the lock (synchronously, so a release and the sweeper never both
@@ -383,6 +491,13 @@ async function handBack(expected, { reason = null } = {}) {
   let stopped = null;
   await runQueued(current.session, 0, async () => {
     closeInputSocket(current.session);
+    // The bot's tab goes back to its own size before the bot carries on.
+    try {
+      if (drive.active(current.session)) await drive.stop();
+      else if (current.bounds) await drive.restoreWindow(info, current.bounds);
+    } catch (e) {
+      console.error(`[browser] ending the driver's view failed: ${e.message}`);
+    }
     try {
       stopped = await teach.stopRecording(current.userId, { session: info });
     } catch (e) {
@@ -390,9 +505,10 @@ async function handBack(expected, { reason = null } = {}) {
     }
   });
   const line = handBackLine(current.name, reason);
+  const waited = botWasWaiting(current);
   if (stopped && !stopped.discarded) wakeChat(current, teach.teachPrompt(stopped, { handBack: line }), "Teach me");
-  else wakeChat(current, line, "Browser");
-  console.log(`[browser] ${current.userId} handed the browser back${reason ? ` (${reason})` : ""}`);
+  else if (waited) wakeChat(current, line, "Browser");
+  console.log(`[browser] ${current.userId} handed the browser back${reason ? ` (${reason})` : ""}${waited || (stopped && !stopped.discarded) ? "" : "; the bot was not waiting for them, so it is not woken"}`);
   return true;
 }
 
@@ -409,10 +525,13 @@ async function control(body = {}) {
   if (body.action === "take") return take(body);
   if (body.action === "renew") return renew(body);
   if (body.action === "release") return release(body);
+  if (body.action === "fit") return fitView(body);
   return reply(400, { error: "bad-action" });
 }
 
-// navigate: true tells the front door this bot has the address bar (POST /api/browser/navigate).
+// navigate: true tells the front door this bot has the address bar (POST /api/browser/navigate);
+// fit: true that a take can bring the driver's view (core/browser-drive.js) and input_text works;
+// start: true that it opens its browser on a start page when there is none (POST /api/browser/start).
 function status() {
   const lock = lockFile.liveLock(deps.now());
   return reply(200, {
@@ -420,9 +539,50 @@ function status() {
     driver: lock ? { name: lock.name, userId: lock.userId } : null,
     recording: teach.activeSummary(),
     navigate: true,
+    fit: drive.supported(),
+    start: true,
   });
 }
 
+// ---------------------------------------------------------------------------
+// Always on: POST /api/browser/start
+
+// The socket folder the bot's own agent-browser commands use (its agent runs without
+// XDG_RUNTIME_DIR): AGENT_BROWSER_SOCKET_DIR, else ~/.agent-browser.
+function agentSocketDir(env = process.env, home = os.homedir()) {
+  if (env.AGENT_BROWSER_SOCKET_DIR) return env.AGENT_BROWSER_SOCKET_DIR;
+  return home ? path.join(home, ".agent-browser") : null;
+}
+
+let starting = null;
+
+// { who: { userId } }. A browser already open is answered as it is (the newest session); otherwise
+// the bot's default session opens on the start page, as the bot's own `agent-browser open` would,
+// and two people opening the app at once start one browser. { session, started }.
+async function start({ who } = {}) {
+  if (!validWho(who)) return reply(400, { error: "bad-request" });
+  const open = deps.listSessions();
+  if (open.length) return reply(200, { session: open[0].name, started: false });
+  if (!starting) {
+    starting = (async () => {
+      const dir = agentSocketDir();
+      const run = await runAgentBrowser({ name: START_SESSION, dir }, ["open", START_PAGE], {
+        timeoutMs: deps.navigateTimeoutMs,
+        execFile: deps.execFile || undefined,
+      });
+      const listed = run.ok ? deps.listSessions().find((s) => s.name === START_SESSION) : null;
+      if (!listed) {
+        const why = run.timedOut ? `agent-browser took longer than ${Math.ceil(deps.navigateTimeoutMs / 1000)} s and was stopped` : run.ok ? "the browser did not come up" : "agent-browser failed";
+        console.error(`[browser] could not open the browser for ${who.userId}: ${why}`);
+        return reply(503, { error: "browser-unavailable" });
+      }
+      console.log(`[browser] opened the browser on the start page for ${who.userId}`);
+      return reply(200, { session: listed.name, started: true });
+    })().finally(() => { starting = null; });
+  }
+  return starting;
+}
+
 // ---------------------------------------------------------------------------
 // C3: input
 
@@ -436,24 +596,41 @@ async function input({ session, events, who } = {}) {
     messages.push(msg);
   }
   // Every accepted input extends the lock.
-  if (!extendDriverLock(session, who)) return reply(409, { error: "not-driving" });
+  const lock = extendDriverLock(session, who);
+  if (!lock) return reply(409, { error: "not-driving" });
   if (!messages.length) return reply(200, { ok: true, accepted: 0 });
+  if (lock.fit && !drive.active(session)) reviveView(lock);
   const entry = openInputSocket(session);
-  if (!entry) return reply(404, { error: "not-running" });
+  if (!entry && !drive.active(session)) return reply(404, { error: "not-running" });
   const job = runQueued(session, messages.length, async () => {
-    if (!(await socketReady(entry))) return reply(503, { error: "browser-unavailable", accepted: 0 });
     const recorder = teach.recorderFor(session, who.userId);
-    let sent = 0;
-    for (const msg of messages) {
-      if (recorder) {
+    const record = async (msg, frame, info) => {
+      if (!recorder) return;
+      // Teach me counts typing from key presses: typed text is shown to it as keys.
+      for (const ev of msg.type === "input_text" ? textAsKeys(msg.text) : [msg]) {
         try {
-          await recorder.onEvent(msg, { session: { name: entry.name, dir: entry.dir }, frame: entry.frame });
+          await recorder.onEvent(ev, { session: info, frame });
         } catch (e) {
           console.error(`[browser] recording a step failed: ${e.message}`);
         }
       }
+    };
+    // The driver's view: straight to Chrome, in order, without waiting for each answer.
+    if (drive.active(session)) {
+      const info = sessionInfo(session) || { name: session, dir: entry ? entry.dir : null };
+      for (const msg of messages) {
+        await record(msg, drive.lastFrame(), { name: info.name, dir: info.dir });
+        if (drive.dispatch([msg]) < 0) return reply(503, { error: "browser-unavailable" });
+      }
+      return reply(200, { ok: true, accepted: messages.length });
+    }
+    if (!entry || !(await socketReady(entry))) return reply(503, { error: "browser-unavailable", accepted: 0 });
+    let sent = 0;
+    for (const msg of messages) {
+      await record(msg, entry.frame, { name: entry.name, dir: entry.dir });
       if (!entry.open || entry.closed) return reply(503, { error: "browser-unavailable", accepted: sent });
-      entry.ws.send(JSON.stringify(msg));
+      // agent-browser's socket takes no text: typed text goes as key presses.
+      for (const ev of msg.type === "input_text" ? textAsKeys(msg.text) : [msg]) entry.ws.send(JSON.stringify(ev));
       sent += 1;
     }
     return reply(200, { ok: true, accepted: sent });
@@ -602,6 +779,8 @@ function _reset() {
   for (const name of [...sockets.keys()]) closeInputSocket(name);
   queues.clear();
   if (sweepTimer) { clearInterval(sweepTimer); sweepTimer = null; }
+  reviving = null;
+  drive._reset();
   deps = defaultDeps();
 }
 
@@ -609,15 +788,20 @@ module.exports = {
   LOCK_TTL_MS,
   MAX_EVENTS,
   NAVIGATE_TIMEOUT_MS,
+  START_PAGE,
+  START_SESSION,
   SWEEP_INTERVAL_MS,
+  agentSocketDir,
   checkEvent,
   chatOrigin,
   control,
+  fitView,
   handBackLine,
   input,
   navigate,
   release,
   renew,
+  start,
   startSweeper,
   status,
   sweepLock,