@tiens.nguyen/gu-cli 1.0.834 → 1.0.836

package/browser-extension/actions.js

@@ -0,0 +1,319 @@
+// What every browser command DOES (task #363/#369, GitHub #144/#150), shared by the two ways gu
+// reaches Chrome:
+//   • the gu Chrome extension (background.js) — chrome.debugger in the user's own Chrome;
+//   • gu's own Chrome window (browser-chrome.mjs, in the bridge) — the DevTools protocol over a
+//     WebSocket, no extension, nothing to install.
+// Both speak the same DevTools protocol, so one implementation serves both. A "host" supplies the
+// tabs and the protocol:
+//   host.cdp(tabId, method, params) → result        host.attach(tabId)
+//   host.open(url, newTab) → tabId                   host.current() → tabId (throws when none)
+//   host.tabs() → [{id, title, url, current}]        host.switchTo(tabId) → boolean
+//   host.info(tabId) → {url, title, status} | null   host.activate(tabId)
+//   host.close(tabId)
+// No chrome.* and no Node API in here: it runs in an extension service worker and in Node.
+
+const consoleBuf = new Map(); // tabId → [{level, text}] since the last "console" read
+const dialogs = new Map(); // tabId → [text] of alerts/confirms gu accepted
+const freshErrors = new Map(); // tabId → errors not yet reported after a command
+
+export function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+export function forgetTab(tabId) {
+  consoleBuf.delete(tabId);
+  freshErrors.delete(tabId);
+  dialogs.delete(tabId);
+}
+
+function pushConsole(tabId, level, text) {
+  const buf = consoleBuf.get(tabId) || [];
+  buf.push({ level, text: String(text).slice(0, 500) });
+  if (buf.length > 100) buf.splice(0, buf.length - 100);
+  consoleBuf.set(tabId, buf);
+  if (level === "error") freshErrors.set(tabId, [...(freshErrors.get(tabId) || []), String(text).slice(0, 300)].slice(-5));
+}
+
+/** Every DevTools event of a tab goes through here (both hosts call it). */
+export function onCdpEvent(tabId, method, params, cdp) {
+  if (method === "Runtime.consoleAPICalled") {
+    const text = (params.args || []).map((a) => (a.value !== undefined ? String(a.value) : a.description || a.type)).join(" ");
+    pushConsole(tabId, params.type === "warning" ? "warn" : params.type, text);
+  } else if (method === "Runtime.exceptionThrown") {
+    const d = params.exceptionDetails || {};
+    pushConsole(tabId, "error", `Uncaught ${d.exception?.description || d.text || "exception"}`);
+  } else if (method === "Log.entryAdded") {
+    const e = params.entry || {};
+    if (e.level === "error" || e.level === "warning") pushConsole(tabId, e.level === "warning" ? "warn" : "error", `${e.text}${e.url ? ` (${e.url})` : ""}`);
+  } else if (method === "Page.javascriptDialogOpening") {
+    // An alert/confirm would freeze the page and every later command; accept it and report it.
+    const list = dialogs.get(tabId) || [];
+    list.push(`${params.type}: ${params.message}`);
+    dialogs.set(tabId, list);
+    cdp(tabId, "Page.handleJavaScriptDialog", { accept: true, promptText: params.defaultPrompt || "" }).catch(() => {});
+  }
+}
+
+/** The domains a freshly attached tab needs, for the events above. */
+export async function enableDomains(cdp, tabId) {
+  await cdp(tabId, "Runtime.enable");
+  await cdp(tabId, "Page.enable");
+  await cdp(tabId, "Log.enable").catch(() => {});
+}
+
+async function evaluate(host, tabId, expression) {
+  const r = await host.cdp(tabId, "Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true, userGesture: true });
+  if (r.exceptionDetails) {
+    const d = r.exceptionDetails;
+    throw new Error(`page script error: ${d.exception?.description || d.text}`);
+  }
+  return r.result?.value;
+}
+
+export async function waitLoaded(host, tabId, ms = 20000) {
+  const end = Date.now() + ms;
+  await sleep(150);
+  while (Date.now() < end) {
+    const t = await host.info(tabId);
+    if (!t) throw new Error("the tab was closed");
+    if (t.status === "complete") return t;
+    await sleep(150);
+  }
+  return host.info(tabId);
+}
+
+// ── reading the page ────────────────────────────────────────────────────────────────────────
+// Runs IN the page. Numbers the visible interactive elements (data-gu-ref) so the agent can say
+// "click 7"; the numbers are good until the next read.
+function readPage(opts) {
+  const sel = 'a[href],button,input:not([type=hidden]),select,textarea,summary,[role=button],[role=link],[role=checkbox],[role=radio],[role=tab],[role=menuitem],[role=option],[role=switch],[role=combobox],[role=textbox],[contenteditable=""],[contenteditable=true],[onclick],[tabindex]:not([tabindex="-1"])';
+  const vw = innerWidth;
+  const vh = innerHeight;
+  for (const e of document.querySelectorAll("[data-gu-ref]")) e.removeAttribute("data-gu-ref");
+  const clean = (s) => String(s || "").replace(/\s+/g, " ").trim();
+  const lines = [];
+  let n = 0;
+  let below = 0;
+  for (const el of document.querySelectorAll(sel)) {
+    const r = el.getBoundingClientRect();
+    if (r.width < 1 || r.height < 1) continue;
+    const cs = getComputedStyle(el);
+    if (cs.visibility === "hidden" || cs.display === "none" || Number(cs.opacity) === 0) continue;
+    if (!(r.bottom > 0 && r.right > 0 && r.top < vh && r.left < vw)) {
+      below++;
+      continue;
+    }
+    if (lines.length >= (opts.max || 150)) {
+      below++;
+      continue;
+    }
+    const ref = ++n;
+    el.setAttribute("data-gu-ref", String(ref));
+    const tag = el.tagName.toLowerCase();
+    const type = tag === "input" ? (el.getAttribute("type") || "text").toLowerCase() : "";
+    const role = el.getAttribute("role") || (tag === "a" ? "link" : tag === "input" ? (["checkbox", "radio", "submit", "button"].includes(type) ? type : `textbox(${type})`) : tag);
+    const label = el.labels && el.labels[0] ? el.labels[0].innerText : "";
+    const name = clean(el.getAttribute("aria-label") || label || (tag === "input" || tag === "textarea" || tag === "select" ? "" : el.innerText) || el.getAttribute("placeholder") || el.getAttribute("title") || el.getAttribute("alt") || el.getAttribute("name") || "").slice(0, 80);
+    const bits = [];
+    if ((tag === "input" && !["checkbox", "radio", "submit", "button"].includes(type)) || tag === "textarea") bits.push(`value=${JSON.stringify(String(el.value).slice(0, 60))}`);
+    if (tag === "select") bits.push(`selected=${JSON.stringify(clean(el.options[el.selectedIndex]?.text))}`);
+    if (type === "checkbox" || type === "radio") bits.push(el.checked ? "checked" : "unchecked");
+    if (el.disabled || el.getAttribute("aria-disabled") === "true") bits.push("disabled");
+    if (tag === "a") bits.push(`→ ${clean(el.getAttribute("href")).slice(0, 60)}`);
+    if (el === document.activeElement) bits.push("focused");
+    lines.push(`[${ref}] ${role} ${JSON.stringify(name)}${bits.length ? ` ${bits.join(" ")}` : ""} @${Math.round(r.left + r.width / 2)},${Math.round(r.top + r.height / 2)}`);
+  }
+  const de = document.documentElement;
+  return {
+    url: location.href,
+    title: document.title,
+    viewport: `${vw}x${vh}`,
+    scroll: `${Math.round(scrollY)} of ${Math.max(0, de.scrollHeight - vh)}`,
+    elements: lines,
+    offscreen: below,
+    text: String((document.body && document.body.innerText) || "").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim().slice(0, opts.maxText || 6000),
+  };
+}
+
+async function refPoint(host, tabId, ref) {
+  const p = await evaluate(host, tabId, `(() => {
+    const el = document.querySelector('[data-gu-ref="${Number(ref)}"]');
+    if (!el) return null;
+    el.scrollIntoView({ block: "center", inline: "center", behavior: "instant" });
+    const r = el.getBoundingClientRect();
+    return { x: r.left + r.width / 2, y: r.top + r.height / 2 };
+  })()`);
+  if (!p) throw new Error(`no element [${ref}] on the page now — read the page again for fresh numbers`);
+  return p;
+}
+
+// What is really under the point — an overlay or a cookie banner takes the click, not the button.
+async function hitAt(host, tabId, x, y) {
+  return evaluate(host, tabId, `(() => {
+    const el = document.elementFromPoint(${x}, ${y});
+    if (!el) return "nothing";
+    const t = (el.innerText || el.value || el.getAttribute("aria-label") || "").replace(/\\s+/g, " ").trim().slice(0, 50);
+    return el.tagName.toLowerCase() + (el.id ? "#" + el.id : "") + (t ? ' "' + t + '"' : "");
+  })()`);
+}
+
+const hasRef = (cmd) => cmd.ref !== undefined && cmd.ref !== null && cmd.ref !== "";
+
+async function point(host, tabId, cmd) {
+  if (hasRef(cmd)) return refPoint(host, tabId, cmd.ref);
+  if (Number.isFinite(cmd.x) && Number.isFinite(cmd.y)) return { x: cmd.x, y: cmd.y };
+  throw new Error("say where: ref (a number from read) or x and y");
+}
+
+async function mouse(host, tabId, x, y, button = "left", clicks = 1) {
+  await host.cdp(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x, y });
+  for (let i = 1; i <= clicks; i++) {
+    await host.cdp(tabId, "Input.dispatchMouseEvent", { type: "mousePressed", x, y, button, clickCount: i });
+    await host.cdp(tabId, "Input.dispatchMouseEvent", { type: "mouseReleased", x, y, button, clickCount: i });
+  }
+}
+
+// ── keys ────────────────────────────────────────────────────────────────────────────────────
+const KEYS = {
+  enter: ["Enter", "Enter", 13, "\r"], tab: ["Tab", "Tab", 9], escape: ["Escape", "Escape", 27], esc: ["Escape", "Escape", 27],
+  backspace: ["Backspace", "Backspace", 8], delete: ["Delete", "Delete", 46], space: [" ", "Space", 32, " "],
+  arrowup: ["ArrowUp", "ArrowUp", 38], arrowdown: ["ArrowDown", "ArrowDown", 40], arrowleft: ["ArrowLeft", "ArrowLeft", 37],
+  arrowright: ["ArrowRight", "ArrowRight", 39], up: ["ArrowUp", "ArrowUp", 38], down: ["ArrowDown", "ArrowDown", 40],
+  left: ["ArrowLeft", "ArrowLeft", 37], right: ["ArrowRight", "ArrowRight", 39], home: ["Home", "Home", 36], end: ["End", "End", 35],
+  pageup: ["PageUp", "PageUp", 33], pagedown: ["PageDown", "PageDown", 34],
+};
+const MODS = { alt: 1, option: 1, ctrl: 2, control: 2, meta: 4, cmd: 4, command: 4, shift: 8 };
+
+async function pressKey(host, tabId, combo) {
+  const parts = String(combo).split("+").map((s) => s.trim()).filter(Boolean);
+  let mods = 0;
+  for (const p of parts.slice(0, -1)) mods |= MODS[p.toLowerCase()] || 0;
+  const k = parts[parts.length - 1] || "";
+  // Select-all and friends are editor COMMANDS in Chrome: a bare ctrl+a does not select on a Mac.
+  const cmdName = (mods & 6) && /^[acvxz]$/i.test(k) ? { a: "selectAll", c: "copy", v: "paste", x: "cut", z: "undo" }[k.toLowerCase()] : null;
+  let key, code, keyCode, text;
+  if (KEYS[k.toLowerCase()]) [key, code, keyCode, text] = KEYS[k.toLowerCase()];
+  else if (k.length === 1) {
+    key = mods & 8 ? k.toUpperCase() : k;
+    code = /[a-z]/i.test(k) ? `Key${k.toUpperCase()}` : /\d/.test(k) ? `Digit${k}` : "";
+    keyCode = k.toUpperCase().charCodeAt(0);
+    text = key;
+  } else throw new Error(`unknown key "${k}" — use Enter, Tab, Escape, Backspace, Delete, ArrowUp/Down/Left/Right, Home, End, PageUp, PageDown, Space or one character, with ctrl+ / shift+ / alt+ / meta+`);
+  if (mods & 7) text = undefined; // a shortcut types nothing
+  const base = { key, code, windowsVirtualKeyCode: keyCode, modifiers: mods };
+  await host.cdp(tabId, "Input.dispatchKeyEvent", { ...base, type: text ? "keyDown" : "rawKeyDown", text, ...(cmdName ? { commands: [cmdName] } : {}) });
+  await host.cdp(tabId, "Input.dispatchKeyEvent", { ...base, type: "keyUp" });
+}
+
+// ── the commands ────────────────────────────────────────────────────────────────────────────
+async function settle(host, tabId) {
+  await sleep(350);
+  const t = await host.info(tabId);
+  if (t && t.status === "loading") await waitLoaded(host, tabId, 15000);
+}
+
+async function after(host, tabId, extra = {}) {
+  const t = await host.info(tabId);
+  const errs = freshErrors.get(tabId) || [];
+  const dl = dialogs.get(tabId) || [];
+  freshErrors.delete(tabId);
+  dialogs.delete(tabId);
+  return { tab: tabId, url: t?.url || "", title: t?.title || "", errors: errs, dialogs: dl, ...extra };
+}
+
+export async function runAction(cmd, host) {
+  const a = String(cmd.action || "").toLowerCase();
+  if (a === "open") {
+    const url = String(cmd.url || "");
+    if (!/^(https?|file):\/\//i.test(url) && !/^about:blank$/i.test(url)) throw new Error("open needs a full http(s):// or file:// url");
+    const id = await host.open(url, Boolean(cmd.new_tab));
+    await host.attach(id);
+    await waitLoaded(host, id);
+    return after(host, id);
+  }
+  if (a === "tabs") return { tabs: await host.tabs() };
+  if (a === "switch") {
+    const id = cmd.tab;
+    if (!(await host.switchTo(id))) throw new Error(`tab ${cmd.tab} is not one of gu's tabs — see action "tabs"`);
+    return after(host, id);
+  }
+  const id = await host.current();
+  if (a === "close") {
+    await host.close(id);
+    forgetTab(id);
+    return { closed: id };
+  }
+  await host.attach(id);
+  switch (a) {
+    case "read":
+      return { ...(await evaluate(host, id, `(${readPage})(${JSON.stringify({ maxText: cmd.max_text || 6000 })})`)), ...(await after(host, id)) };
+    case "screenshot": {
+      await host.activate(id); // a background tab is not painted
+      const m = await host.cdp(id, "Page.getLayoutMetrics");
+      const vp = m.cssVisualViewport || m.visualViewport || {};
+      const shot = await host.cdp(id, "Page.captureScreenshot", { format: "jpeg", quality: 80 });
+      return { ...(await after(host, id)), image: shot.data, mime: "image/jpeg", width: Math.round(vp.clientWidth), height: Math.round(vp.clientHeight) };
+    }
+    case "click":
+    case "double_click":
+    case "right_click":
+    case "hover": {
+      const p = await point(host, id, cmd);
+      const hit = await hitAt(host, id, p.x, p.y);
+      if (a === "hover") await host.cdp(id, "Input.dispatchMouseEvent", { type: "mouseMoved", x: p.x, y: p.y });
+      else await mouse(host, id, p.x, p.y, a === "right_click" ? "right" : "left", a === "double_click" ? 2 : 1);
+      await settle(host, id);
+      return after(host, id, { at: `${Math.round(p.x)},${Math.round(p.y)}`, hit });
+    }
+    case "type": {
+      if (hasRef(cmd)) {
+        const p = await refPoint(host, id, cmd.ref);
+        await mouse(host, id, p.x, p.y);
+        if (cmd.clear) await pressKey(host, id, "ctrl+a");
+      }
+      await host.cdp(id, "Input.insertText", { text: String(cmd.text ?? "") });
+      if (cmd.submit) await pressKey(host, id, "Enter");
+      await settle(host, id);
+      return after(host, id);
+    }
+    case "key":
+      for (const k of String(cmd.text || "").split(/\s+/).filter(Boolean)) await pressKey(host, id, k);
+      await settle(host, id);
+      return after(host, id);
+    case "scroll": {
+      const m = await host.cdp(id, "Page.getLayoutMetrics");
+      const vp = m.cssVisualViewport || {};
+      const x = Number.isFinite(cmd.x) ? cmd.x : vp.clientWidth / 2;
+      const y = Number.isFinite(cmd.y) ? cmd.y : vp.clientHeight / 2;
+      const amount = Number(cmd.amount) || Math.round((vp.clientHeight || 800) * 0.8);
+      const dir = String(cmd.direction || "down").toLowerCase();
+      const dx = dir === "left" ? -amount : dir === "right" ? amount : 0;
+      const dy = dir === "up" ? -amount : dir === "down" ? amount : 0;
+      await host.cdp(id, "Input.dispatchMouseEvent", { type: "mouseWheel", x, y, deltaX: dx, deltaY: dy });
+      await sleep(300);
+      return after(host, id, { scroll: await evaluate(host, id, "Math.round(scrollY) + ' of ' + Math.max(0, document.documentElement.scrollHeight - innerHeight)") });
+    }
+    case "back":
+      await evaluate(host, id, "history.back()");
+      await settle(host, id);
+      return after(host, id);
+    case "reload":
+      await host.cdp(id, "Page.reload", { ignoreCache: true });
+      await settle(host, id);
+      return after(host, id);
+    case "wait":
+      await sleep(Math.max(0, Math.min(Number(cmd.seconds) || 1, 30)) * 1000);
+      return after(host, id);
+    case "js": {
+      const value = await evaluate(host, id, String(cmd.text || ""));
+      return after(host, id, { value: value === undefined ? "undefined" : JSON.stringify(value)?.slice(0, 8000) });
+    }
+    case "console": {
+      const all = consoleBuf.get(id) || [];
+      consoleBuf.set(id, []);
+      return after(host, id, { console: all });
+    }
+    default:
+      throw new Error(`unknown action "${cmd.action}"`);
+  }
+}