@xnng/browser-relay 1.6.0

package/extension/snapshot.js

@@ -0,0 +1,161 @@
+/**
+ * DOM-to-text extraction function.
+ * This JS string is sent to the browser via Runtime.evaluate and returns
+ * a text representation of the visible page suitable for an LLM.
+ */
+export const SNAPSHOT_JS = `
+(function() {
+  const INTERACTIVE_TAGS = new Set([
+    'A', 'BUTTON', 'INPUT', 'SELECT', 'TEXTAREA', 'DETAILS', 'SUMMARY',
+  ]);
+  const SKIP_TAGS = new Set([
+    'SCRIPT', 'STYLE', 'NOSCRIPT', 'SVG', 'PATH', 'TEMPLATE',
+    'IFRAME', 'OBJECT', 'EMBED',
+  ]);
+  const BLOCK_TAGS = new Set([
+    'DIV', 'P', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6',
+    'UL', 'OL', 'LI', 'TABLE', 'TR', 'SECTION', 'ARTICLE',
+    'NAV', 'HEADER', 'FOOTER', 'MAIN', 'ASIDE', 'FORM',
+    'BLOCKQUOTE', 'PRE', 'HR', 'BR', 'DL', 'DT', 'DD',
+    'FIGURE', 'FIGCAPTION', 'ADDRESS',
+  ]);
+
+  function isVisible(el) {
+    if (!el.offsetParent && el.tagName !== 'BODY' && el.tagName !== 'HTML') {
+      var style = getComputedStyle(el);
+      if (style.display === 'none' || style.visibility === 'hidden') return false;
+      if (style.display !== 'contents' && style.position !== 'fixed' && style.position !== 'sticky') return false;
+    }
+    if (el.getAttribute('aria-hidden') === 'true') return false;
+    if (el.hasAttribute('hidden')) return false;
+    return true;
+  }
+
+  function describeElement(el) {
+    var tag = el.tagName;
+    if (tag === 'A') {
+      var href = el.getAttribute('href') || '';
+      var text = (el.innerText || el.getAttribute('aria-label') || '').trim();
+      return '[link' + (text ? ' "' + text + '"' : '') + (href ? ' href="' + href + '"' : '') + ']';
+    }
+    if (tag === 'BUTTON' || (el.getAttribute('role') === 'button')) {
+      var text = (el.getAttribute('aria-label') || el.innerText || '').trim();
+      return '[button' + (text ? ' "' + text + '"' : '') + ']';
+    }
+    if (tag === 'INPUT') {
+      var type = el.getAttribute('type') || 'text';
+      var name = el.getAttribute('name') || '';
+      var placeholder = el.getAttribute('placeholder') || '';
+      var label = el.getAttribute('aria-label') || '';
+      var val = type === 'password' ? '[redacted]' : (el.value || '').slice(0, 80);
+      var desc = '[input type=' + type;
+      if (name) desc += ' name="' + name + '"';
+      if (placeholder) desc += ' placeholder="' + placeholder + '"';
+      if (label) desc += ' label="' + label + '"';
+      if (val) desc += ' value="' + val + '"';
+      desc += ']';
+      return desc;
+    }
+    if (tag === 'TEXTAREA') {
+      var name = el.getAttribute('name') || '';
+      var placeholder = el.getAttribute('placeholder') || '';
+      var val = (el.value || '').slice(0, 200);
+      var desc = '[textarea';
+      if (name) desc += ' name="' + name + '"';
+      if (placeholder) desc += ' placeholder="' + placeholder + '"';
+      if (val) desc += ' value="' + val + '"';
+      desc += ']';
+      return desc;
+    }
+    if (tag === 'SELECT') {
+      var name = el.getAttribute('name') || '';
+      var selected = el.options && el.selectedIndex >= 0 ? (el.options[el.selectedIndex].text || '').slice(0, 60) : '';
+      var desc = '[select';
+      if (name) desc += ' name="' + name + '"';
+      if (selected) desc += ' selected="' + selected + '"';
+      desc += ']';
+      return desc;
+    }
+    if (tag === 'IMG') {
+      var alt = el.getAttribute('alt') || '';
+      return alt ? '[image "' + alt.slice(0, 100) + '"]' : '[image]';
+    }
+    return null;
+  }
+
+  var lines = [];
+  var maxLen = (typeof __maxLength !== 'undefined') ? __maxLength : 100000;
+  var totalLen = 0;
+  var truncated = false;
+
+  function addLine(text, indent) {
+    if (truncated) return;
+    var line = '  '.repeat(indent) + text;
+    totalLen += line.length + 1;
+    if (totalLen > maxLen) { truncated = true; return; }
+    lines.push(line);
+  }
+
+  function walk(node, depth) {
+    if (truncated) return;
+    if (node.nodeType === 3) {
+      var text = node.textContent.trim();
+      if (text) addLine(text, depth);
+      return;
+    }
+    if (node.nodeType !== 1) return;
+    var el = node;
+    var tag = el.tagName;
+    if (SKIP_TAGS.has(tag)) return;
+    if (!isVisible(el)) return;
+
+    var headingMatch = tag.match(/^H([1-6])$/);
+    if (headingMatch) {
+      var text = (el.innerText || '').trim();
+      if (text) addLine('#'.repeat(parseInt(headingMatch[1])) + ' ' + text, depth);
+      return;
+    }
+
+    var desc = describeElement(el);
+    if (desc) {
+      addLine(desc, depth);
+      if (tag === 'A' || tag === 'BUTTON') return;
+    }
+
+    var isBlock = BLOCK_TAGS.has(tag);
+    var childDepth = desc ? depth + 1 : (isBlock ? depth : depth);
+
+    if (tag === 'LI') {
+      var text = '';
+      for (var i = 0; i < el.childNodes.length; i++) {
+        var child = el.childNodes[i];
+        if (child.nodeType === 3) text += child.textContent;
+      }
+      text = text.trim();
+      if (text) addLine('- ' + text, depth);
+      for (var i = 0; i < el.children.length; i++) {
+        walk(el.children[i], depth + 1);
+      }
+      return;
+    }
+
+    if (tag === 'TR') {
+      var cells = [];
+      for (var i = 0; i < el.children.length; i++) {
+        cells.push((el.children[i].innerText || '').trim());
+      }
+      if (cells.some(function(c) { return c; })) {
+        addLine('| ' + cells.join(' | ') + ' |', depth);
+      }
+      return;
+    }
+
+    for (var i = 0; i < el.childNodes.length; i++) {
+      walk(el.childNodes[i], childDepth);
+    }
+  }
+
+  walk(document.body, 0);
+  return JSON.stringify({ snapshot: lines.join('\\n'), truncated: truncated });
+})()
+`;

package/extension/task-groups.js

@@ -0,0 +1,108 @@
+import { TaskError } from './tasks.js';
+
+// 页面资源独立于两分钟的控制租约，断线/租约过期不能代表用户任务完成。
+// storage.session 允许 service worker 重启后清理，浏览器重启后自动失效，避免复用旧 tab/group ID。
+export function createTaskGroups(chrome) {
+  const key = 'automationTaskGroupsV1';
+  let names = new Map(), records = new Map();
+  const ready = chrome.storage.session.get(key).then(data => {
+    names = new Map(data[key]?.names || []);
+    records = new Map(data[key]?.records || []);
+  });
+  const save = () => chrome.storage.session.set({ [key]: { names: [...names], records: [...records] } });
+  const valid = id => {
+    if (typeof id !== 'string' || !/^[\w.-]{1,128}$/.test(id))
+      throw new TaskError('invalid_session', 'A named session is required');
+  };
+  async function current(id) {
+    try { return await chrome.tabs.get(id); }
+    catch (error) {
+      if (/No tab with id|Invalid tab ID/i.test(error.message)) return null;
+      throw error;
+    }
+  }
+  async function list() {
+    await ready;
+    const result = new Map();
+    for (const [tabId, record] of records) {
+      const tab = await current(tabId);
+      if (!tab || tab.groupId !== record.groupId) { records.delete(tabId); continue; }
+      if (!result.has(record.sessionId)) result.set(record.sessionId, {
+        sessionId: record.sessionId, label: names.get(record.sessionId) || record.sessionId, groups: [],
+      });
+      const item = result.get(record.sessionId);
+      let group = item.groups.find(g => g.groupId === tab.groupId);
+      if (!group) {
+        const details = await chrome.tabGroups.get(tab.groupId);
+        item.groups.push(group = { groupId: tab.groupId, windowId: tab.windowId, title: details.title, tabIds: [] });
+      }
+      group.tabIds.push(tabId);
+    }
+    await save();
+    return [...result.values()];
+  }
+  async function start(id, label) {
+    valid(id); await ready;
+    if (label !== undefined && (typeof label !== 'string' || !label.trim() || label.length > 100))
+      throw new TaskError('invalid_label', 'Task label must contain 1–100 characters');
+    names.set(id, label || names.get(id) || id);
+    for (const item of await list()) if (item.sessionId === id)
+      for (const group of item.groups) await chrome.tabGroups.update(group.groupId, { title: names.get(id) });
+    await save();
+  }
+  async function add(tabId, id) {
+    valid(id); await ready;
+    const tab = await chrome.tabs.get(tabId);
+    const tasks = await list();
+    const existing = tasks.find(s => s.sessionId === id)?.groups.find(g => g.windowId === tab.windowId);
+    const groupId = await chrome.tabs.group({ tabIds: [tabId], ...(existing ? { groupId: existing.groupId } : {}) });
+    // 先保存创建来源；命名失败仍能识别并清理这一页，不把组内其他用户页面当成任务页面。
+    records.set(tabId, { sessionId: id, groupId });
+    await save();
+    await chrome.tabGroups.update(groupId, { title: names.get(id) || id, color: 'blue' });
+    return groupId;
+  }
+  async function claim(tabId, id) {
+    await ready;
+    const record = records.get(tabId);
+    if (!record) return false;
+    const tab = await current(tabId);
+    if (!tab || tab.groupId !== record.groupId) {
+      records.delete(tabId); await save(); return false;
+    }
+    // 新任务接管已释放租约的任务页时，资源归属也一起转移，旧任务不能在稍后误关。
+    if (record.sessionId !== id) await add(tabId, id);
+    return true;
+  }
+  async function release(tabId, id) {
+    await ready;
+    const record = records.get(tabId);
+    if (record?.sessionId !== id) return;
+    const tab = await current(tabId);
+    if (tab?.groupId === record.groupId) await chrome.tabs.ungroup([tabId]);
+    records.delete(tabId); await save();
+  }
+  async function complete(id, ownerOf, publicId = id => id) {
+    valid(id); await ready;
+    const closed = [], preserved = [], errors = [];
+    for (const [tabId, record] of records) {
+      if (record.sessionId !== id) continue;
+      const resultId = publicId(tabId);
+      try {
+        const tab = await current(tabId);
+        const owner = ownerOf(tabId);
+        // 手动移出/改组、交给其他 session 的页面视为用户接管，不再自动关闭。
+        if (tab && tab.groupId === record.groupId && (!owner || owner === id)) {
+          await chrome.tabs.remove(tabId);
+          closed.push(resultId);
+        } else if (tab) preserved.push(resultId);
+        records.delete(tabId);
+        await save();
+      } catch (error) { errors.push({ tabId: resultId, message: error.message }); }
+    }
+    if (!errors.length) names.delete(id);
+    await save();
+    return { closed, preserved, errors };
+  }
+  return { start, add, claim, release, complete, list };
+}

package/extension/tasks.js

@@ -0,0 +1,186 @@
+// Shared by the extension and tests. A queue owns a whole operation, not one
+// mouse event, so independently submitted scripts cannot interleave keystrokes.
+export class TaskError extends Error {
+  constructor(code, message, status = 400) {
+    super(message);
+    this.code = code;
+    this.status = status;
+  }
+}
+export function checkCancelled(signal) {
+  if (signal?.aborted)
+    throw new TaskError(
+      "task_cancelled",
+      "Task cancelled; completed actions were not undone",
+      409,
+    );
+}
+export function pause(ms, signal) {
+  checkCancelled(signal);
+  return new Promise((resolve, reject) => {
+    const abort = () => {
+      clearTimeout(timer);
+      reject(new TaskError("task_cancelled", "Task cancelled", 409));
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", abort);
+      resolve();
+    }, ms);
+    signal?.addEventListener("abort", abort, { once: true });
+  });
+}
+export function createTaskQueue({ limit = 100 } = {}) {
+  const tails = new Map(),
+    jobs = new Map(),
+    cancelledRequests = new Map();
+  const publicJob = (job) => ({
+    id: job.id,
+    tabId: job.tabId,
+    sessionId: job.sessionId,
+    status: job.status,
+    createdAt: job.createdAt,
+    startedAt: job.startedAt,
+    finishedAt: job.finishedAt,
+    results: job.results,
+    completedActions: job.results.length,
+    observation: job.observation,
+    error: job.error,
+    interruptedAction:
+      job.finishedAt && job.currentAction ? job.currentAction : undefined,
+  });
+  function get(id) {
+    const job = jobs.get(id);
+    if (!job && cancelledRequests.has(id)) return cancelledRequests.get(id);
+    if (!job)
+      throw new TaskError(
+        "task_not_found",
+        "Task not found (the extension may have restarted)",
+        404,
+      );
+    return publicJob(job);
+  }
+  function start(tabId, run, timeoutMs = 20000, requestedId, sessionId) {
+    for (const [id, job] of jobs) {
+      if (jobs.size < limit) break;
+      if (job.finishedAt) jobs.delete(id);
+    }
+    if (jobs.size >= limit)
+      throw new TaskError("task_limit", "Too many pending tasks", 429);
+    const id = requestedId || `job_${crypto.randomUUID()}`;
+    if (!/^job_[\w-]{36}$/.test(id))
+      throw new TaskError("invalid_task_id", "Invalid task id");
+    if (cancelledRequests.has(id))
+      throw new TaskError(
+        "task_cancelled",
+        "This request was cancelled before execution",
+        409,
+      );
+    if (jobs.has(id))
+      throw new TaskError(
+        "duplicate_task",
+        "Task id already exists; inspect its result instead of replaying actions",
+        409,
+      );
+    const job = {
+      id,
+      tabId,
+      sessionId,
+      status: "queued",
+      createdAt: Date.now(),
+      results: [],
+      controller: new AbortController(),
+    };
+    jobs.set(id, job);
+    const previous = tails.get(tabId) || Promise.resolve();
+    const done = previous
+      .catch(() => {})
+      .then(async () => {
+        let timer;
+        try {
+          checkCancelled(job.controller.signal);
+          job.status = "running";
+          job.startedAt = Date.now();
+          timer = setTimeout(() => {
+            job.timedOut = true;
+            job.controller.abort();
+          }, timeoutMs);
+          const observation = await run(job, job.controller.signal);
+          checkCancelled(job.controller.signal);
+          job.observation = observation;
+          job.status = "completed";
+        } catch (error) {
+          job.status = job.controller.signal.aborted ? "cancelled" : "failed";
+          job.error = {
+            code: job.timedOut
+              ? "task_timeout"
+              : job.cancelReason || error.code || "action_failed",
+            message: error.message,
+            status: error.status || 500,
+          };
+        } finally {
+          clearTimeout(timer);
+          job.finishedAt = Date.now();
+        }
+        return publicJob(job);
+      });
+    tails.set(tabId, done);
+    done.finally(() => {
+      if (tails.get(tabId) === done) tails.delete(tabId);
+    });
+    return { id, done };
+  }
+  function cancel(id, reason = "task_cancelled", sessionId) {
+    if (!jobs.has(id)) {
+      if (!/^job_[\w-]{36}$/.test(id))
+        throw new TaskError("invalid_task_id", "Invalid task id");
+      const result = {
+        id,
+        sessionId,
+        status: "cancelled",
+        results: [],
+        error: {
+          code: reason,
+          message: "Request cancelled before execution",
+          status: 409,
+        },
+        cancellationRequested: true,
+      };
+      cancelledRequests.set(id, result);
+      if (cancelledRequests.size > 500)
+        cancelledRequests.delete(cancelledRequests.keys().next().value);
+      return result;
+    }
+    get(id);
+    const job = jobs.get(id);
+    if (!job.finishedAt) {
+      job.cancelReason = reason;
+      job.controller.abort();
+    }
+    return { ...publicJob(job), cancellationRequested: !job.finishedAt };
+  }
+  function cancelTab(tabId, reason = "task_cancelled") {
+    for (const job of jobs.values())
+      if (job.tabId === tabId && !job.finishedAt) cancel(job.id, reason);
+  }
+  function cancelSession(sessionId, reason = "session_stopped") {
+    for (const job of jobs.values())
+      if (job.sessionId === sessionId && !job.finishedAt)
+        cancel(job.id, reason);
+  }
+  const active = () =>
+    [...jobs.values()]
+      .filter((j) => !j.finishedAt)
+      .map((j) => ({
+        id: j.id,
+        tabId: j.tabId,
+        sessionId: j.sessionId,
+        status: j.status,
+        completedActions: j.results.length,
+      }));
+  const cancelAll = () => {
+    const pending = active();
+    for (const job of pending) cancel(job.id);
+    return pending.length;
+  };
+  return { start, get, cancel, cancelTab, cancelSession, active, cancelAll };
+}

package/extension/wait.js

@@ -0,0 +1,89 @@
+export const WAIT_STATES = ["attached", "visible"];
+export const WAIT_DEFAULT_TIMEOUT_MS = 5_000;
+export const WAIT_DEFAULT_POLL_MS = 100;
+export const WAIT_MIN_TIMEOUT_MS = 1;
+export const WAIT_MAX_TIMEOUT_MS = 20_000;
+export const WAIT_MIN_POLL_MS = 50;
+export const WAIT_MAX_POLL_MS = 1_000;
+
+function boundedInteger(value, fallback, min, max, field) {
+  if (value === undefined) return { ok: true, value: fallback };
+  if (!Number.isInteger(value) || value < min || value > max) {
+    return { ok: false, field, message: `${field} must be an integer between ${min} and ${max}` };
+  }
+  return { ok: true, value };
+}
+
+export function normalizeWaitOptions(input = {}) {
+  const selector = input.selector;
+  if (typeof selector !== "string" || !selector.trim()) {
+    return { ok: false, field: "selector", message: "selector is required" };
+  }
+
+  const state = input.state ?? "visible";
+  if (!WAIT_STATES.includes(state)) {
+    return {
+      ok: false,
+      field: "state",
+      message: `state must be one of: ${WAIT_STATES.join(", ")}`,
+    };
+  }
+
+  const timeout = boundedInteger(
+    input.timeoutMs,
+    WAIT_DEFAULT_TIMEOUT_MS,
+    WAIT_MIN_TIMEOUT_MS,
+    WAIT_MAX_TIMEOUT_MS,
+    "timeoutMs",
+  );
+  if (!timeout.ok) return timeout;
+
+  const poll = boundedInteger(
+    input.pollMs,
+    WAIT_DEFAULT_POLL_MS,
+    WAIT_MIN_POLL_MS,
+    WAIT_MAX_POLL_MS,
+    "pollMs",
+  );
+  if (!poll.ok) return poll;
+
+  return {
+    ok: true,
+    selector,
+    state,
+    timeoutMs: timeout.value,
+    pollMs: poll.value,
+    tabId: input.tabId,
+  };
+}
+
+export function buildWaitExpression(selector, state) {
+  return `(() => {
+    const __browserRelayWait = true;
+    try {
+      const nodes = Array.from(document.querySelectorAll(${JSON.stringify(selector)}));
+      const isVisible = (element) => {
+        if (!element || !element.isConnected) return false;
+        const rect = element.getBoundingClientRect();
+        if (!(rect.width > 0 || rect.height > 0 || element.getClientRects().length > 0)) return false;
+        for (let node = element; node && node.nodeType === 1; node = node.parentElement) {
+          const style = getComputedStyle(node);
+          if (style.display === 'none' || style.visibility === 'hidden' || style.visibility === 'collapse' || Number.parseFloat(style.opacity) === 0) return false;
+        }
+        return true;
+      };
+      const visibleCount = nodes.filter(isVisible).length;
+      return {
+        matched: ${JSON.stringify(state)} === 'attached' ? nodes.length > 0 : visibleCount > 0,
+        matchCount: nodes.length,
+        visibleCount,
+      };
+    } catch (error) {
+      return {
+        matched: false,
+        invalidSelector: true,
+        message: error instanceof Error ? error.message : String(error),
+      };
+    }
+  })()`;
+}

package/hub/README.md

@@ -0,0 +1,42 @@
+# Browser Relay Hub on Cloudflare
+
+The public relay is a small Cloudflare Worker + Durable Object. Run your own so remote control never touches a third-party service.
+
+## Deploy your own
+
+One click — deploys this Worker to your own Cloudflare account:
+
+[![Deploy to Cloudflare](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https://github.com/reliefeai/browser-relay/tree/main/hub)
+
+The button clones this repo into your account and deploys via Workers Builds, so the first time it asks you to connect Cloudflare to GitHub. If you hit `no associated Git installation`, reconnect Git in the Cloudflare dashboard (**Workers & Pages → connect Git**) and make sure this repo is in the allowed list, then click again.
+
+Prefer the command line?
+
+```bash
+git clone https://github.com/reliefeai/browser-relay
+cd browser-relay/hub && npx wrangler deploy
+```
+
+Either way you get a URL like `https://browser-relay-hub.<your-subdomain>.workers.dev`. Put it in the extension's **Public relay** field (Options → Remote Relay). No custom domain required; to use your own, add a `routes` entry to `wrangler.toml`.
+
+## Smoke check
+
+```bash
+curl https://browser-relay-hub.<your-subdomain>.workers.dev/v1/health
+# { "ok": true, "service": "browser-relay-hub" }
+```
+
+## Endpoints
+
+- `GET  /v1/device/connect?routeId=...` — the browser extension's outbound WebSocket. The device sends its secret in the first `device.auth` frame, never in the URL.
+- `POST /v1/rpc` — remote CLI/MCP command entrypoint.
+- `GET  /v1/status/:routeId` — device online status.
+
+The `remote-device-id` is a capability secret generated by the extension. The hub keeps only an in-memory hash of it while the device is connected; nothing is persisted.
+
+For rollout compatibility, the Hub temporarily accepts older extensions that send
+`?token=...`. New extension code intentionally has no query-token fallback. Deploy the
+new Hub before updating extensions, then remove legacy acceptance after old clients have
+aged out.
+
+For the npm distribution, copy `wrangler.example.toml` to `wrangler.toml` before deploying. The package excludes machine-specific deployment configuration and local dependencies.