@otterware/openclaw-otter 0.1.2 → 0.2.0

package/control.js

@@ -0,0 +1,161 @@
+/** A held tab is released this long after Otter last sent input or a heartbeat (it quit, say). */
+export const CONTROL_TTL_MS = 30_000;
+/** How long a batch waits for one sent before it, before giving up on that one. */
+const GAP_MS = 150;
+
+const MOUSE_TYPES = { move: "mouseMoved", down: "mousePressed", up: "mouseReleased" };
+const BUTTONS = new Set(["none", "left", "middle", "right"]);
+const num = (value) => (Number.isFinite(value) ? value : 0);
+
+/** The DevTools call that replays one of Otter's input events, as a person's input. */
+export function cdpInput(event) {
+  const modifiers = num(event?.modifiers);
+  switch (event?.type) {
+    case "mouse":
+      if (!MOUSE_TYPES[event.action]) return null;
+      return [
+        "Input.dispatchMouseEvent",
+        {
+          type: MOUSE_TYPES[event.action],
+          x: num(event.x),
+          y: num(event.y),
+          button: BUTTONS.has(event.button) ? event.button : "none",
+          buttons: num(event.buttons),
+          clickCount: num(event.clickCount),
+          modifiers,
+        },
+      ];
+    case "wheel":
+      return [
+        "Input.dispatchMouseEvent",
+        {
+          type: "mouseWheel",
+          x: num(event.x),
+          y: num(event.y),
+          deltaX: num(event.deltaX),
+          deltaY: num(event.deltaY),
+          modifiers,
+        },
+      ];
+    case "key": {
+      if (typeof event.key !== "string" || typeof event.code !== "string") return null;
+      const text = typeof event.text === "string" ? event.text : undefined;
+      return [
+        "Input.dispatchKeyEvent",
+        {
+          type: event.action === "up" ? "keyUp" : text ? "keyDown" : "rawKeyDown",
+          key: event.key,
+          code: event.code,
+          windowsVirtualKeyCode: num(event.keyCode),
+          nativeVirtualKeyCode: num(event.keyCode),
+          ...(text ? { text, unmodifiedText: text } : {}),
+          modifiers,
+        },
+      ];
+    }
+    case "text":
+      return typeof event.text === "string"
+        ? ["Input.insertText", { text: event.text.slice(0, 10_000) }]
+        : null;
+    default:
+      return null;
+  }
+}
+
+/** A page's own DevTools connection: input is sent without waiting for Chromium's replies. */
+export async function connectTarget(cdpUrl, targetId) {
+  const list = await fetch(new URL("/json/list", cdpUrl), { signal: AbortSignal.timeout(5000) })
+    .then((response) => response.json())
+    .catch(() => {
+      throw new Error("Couldn't reach the agent's browser.");
+    });
+  const target = Array.isArray(list) ? list.find((t) => t?.id === targetId) : undefined;
+  if (typeof target?.webSocketDebuggerUrl !== "string") throw new Error("That tab has closed.");
+  const socket = new WebSocket(target.webSocketDebuggerUrl);
+  await new Promise((resolve, reject) => {
+    socket.addEventListener("open", resolve, { once: true });
+    socket.addEventListener(
+      "error",
+      () => reject(new Error("Couldn't reach the agent's browser.")),
+      {
+        once: true,
+      },
+    );
+  });
+  let id = 0;
+  return {
+    open: () => socket.readyState === WebSocket.OPEN,
+    send: (method, params) => socket.send(JSON.stringify({ id: ++id, method, params })),
+    close: () => socket.close(),
+  };
+}
+
+/**
+ * Who holds which of the agent's tabs, and their input. Batches carry a sequence number, so
+ * they're replayed in the order Otter sent them however the gateway runs its requests.
+ */
+export function createBrowserControl({ cdpUrlFor, connect = connectTarget, now = Date.now }) {
+  const held = new Map();
+
+  const holding = (targetId) => {
+    const tab = held.get(targetId);
+    if (tab && tab.until > now()) return tab;
+    if (tab) drop(targetId);
+    return undefined;
+  };
+  const drop = (targetId) => {
+    const tab = held.get(targetId);
+    held.delete(targetId);
+    clearTimeout(tab?.gap);
+    void tab?.page?.then((page) => page.close()).catch(() => {});
+  };
+  const open = (targetId, profile) => cdpUrlFor(profile).then((url) => connect(url, targetId));
+  const pageOf = async (tab, targetId, profile) => {
+    tab.page ??= open(targetId, profile);
+    const page = await tab.page.catch(() => null);
+    if (page?.open()) return page;
+    tab.page = open(targetId, profile);
+    return tab.page;
+  };
+  const flush = (tab, page) => {
+    const missing = tab.next;
+    while (tab.waiting.has(tab.next)) {
+      for (const event of tab.waiting.get(tab.next)) {
+        const call = cdpInput(event);
+        if (call) page.send(...call);
+      }
+      tab.waiting.delete(tab.next);
+      tab.next += 1;
+    }
+    // A batch is missing: let it arrive, then go on without it. Batches arriving behind it
+    // don't push that back.
+    if (tab.gap && tab.next === missing && tab.waiting.size > 0) return;
+    clearTimeout(tab.gap);
+    tab.gap = undefined;
+    if (tab.waiting.size === 0) return;
+    tab.gap = setTimeout(() => {
+      tab.gap = undefined;
+      tab.next = Math.min(...tab.waiting.keys());
+      flush(tab, page);
+    }, GAP_MS);
+  };
+
+  return {
+    anyHeld: () => [...held.keys()].some((targetId) => holding(targetId)),
+    take(targetId) {
+      const tab = holding(targetId);
+      if (tab) tab.until = now() + CONTROL_TTL_MS;
+      else held.set(targetId, { until: now() + CONTROL_TTL_MS, next: 1, waiting: new Map() });
+    },
+    release: drop,
+    async input(targetId, profile, seq, events) {
+      const tab = holding(targetId);
+      if (!tab) throw new Error("Take control of this tab first.");
+      tab.until = now() + CONTROL_TTL_MS;
+      if (!Number.isInteger(seq) || seq < tab.next || !Array.isArray(events)) return;
+      tab.waiting.set(seq, events.slice(0, 200));
+      const page = await pageOf(tab, targetId, profile);
+      if (held.get(targetId) === tab) flush(tab, page);
+    },
+  };
+}

package/index.js

@@ -1,17 +1,46 @@
+import { AsyncResource } from "node:async_hooks";
 import { randomUUID } from "node:crypto";
 import {
   createChannelPluginBase,
   createChatChannelPlugin,
   defineChannelPluginEntry,
 } from "openclaw/plugin-sdk/channel-core";
+import { toInboundMediaFacts } from "openclaw/plugin-sdk/channel-inbound";
+import { defineStableChannelIngressIdentity } from "openclaw/plugin-sdk/channel-ingress-runtime";
 import { resolveAgentWorkspaceDir } from "openclaw/plugin-sdk/agent-runtime";
+import { dispatchGatewayMethod } from "openclaw/plugin-sdk/gateway-method-runtime";
 import { resolveAgentIdFromSessionKey } from "openclaw/plugin-sdk/routing";
+import { createBrowserControl } from "./control.js";
+import { runDetachedWebhookWork } from "openclaw/plugin-sdk/webhook-request-guards";
 import { readWorkspaceFile } from "./files.js";
-import { accountOf, inboxKey, OTTER_TARGET, pointerOf, tell } from "./otter.js";
+import {
+  accountOf,
+  chatOf,
+  chatTarget,
+  isOwnerConfigured,
+  messageOf,
+  OTTER_TARGET,
+  OWNER,
+  pointerOf,
+  sessionOf,
+  tell,
+} from "./otter.js";
+
+/**
+ * Where a message sent from Otter starts its run: outside the request that carried it,
+ * so the run is the channel's owner's and not that connection's, and outlives it.
+ */
+const receiving = new AsyncResource("otter-inbound");
+
+const owner = defineStableChannelIngressIdentity({
+  key: "otter-sender",
+  normalize: (value) => String(value).trim(),
+  sensitivity: "pii",
+});
 
 async function deliver(ctx, payload) {
   const account = accountOf(ctx.cfg);
-  const pointer = pointerOf({ sessionKey: ctx.sessionKey, payload });
+  const pointer = pointerOf({ sessionKey: ctx.sessionKey, to: ctx.to, payload });
   await tell(account, pointer, { signal: ctx.signal });
   return { channel: "otter", messageId: randomUUID(), timestamp: pointer.at };
 }
@@ -36,33 +65,120 @@ export const otterChannel = createChatChannelPlugin({
       },
     }),
     messaging: {
-      targetResolver: { looksLikeId: () => true, hint: `"${OTTER_TARGET}"` },
-      // Every delivery to Otter lands in the agent's Otter session, whatever the gateway's DM scope.
-      resolveOutboundSessionRoute: ({ agentId }) => ({
-        sessionKey: inboxKey(agentId),
-        baseSessionKey: inboxKey(agentId),
-        recipientSessionExact: true,
-        peer: { kind: "direct", id: OTTER_TARGET },
-        chatType: "direct",
-        from: "otter",
-        to: OTTER_TARGET,
-      }),
+      targetResolver: {
+        looksLikeId: () => true,
+        hint: `"${OTTER_TARGET}", or "chat:<session key>" for one of Otter's chats`,
+      },
+      // A delivery to a chat is kept in its session; any other, in the agent's Otter session.
+      resolveOutboundSessionRoute: ({ agentId, target }) => {
+        const chat = chatOf(target);
+        const sessionKey = sessionOf(target, agentId);
+        const to = chat ? chatTarget(chat) : OTTER_TARGET;
+        return {
+          sessionKey,
+          baseSessionKey: sessionKey,
+          recipientSessionExact: true,
+          peer: { kind: "direct", id: to },
+          chatType: "direct",
+          from: "otter",
+          to,
+        };
+      },
     },
   },
   outbound: {
     deliveryMode: "direct",
-    resolveTarget: () => ({ ok: true, to: OTTER_TARGET }),
+    resolveTarget: (params) => {
+      const chat = chatOf(params?.to);
+      return { ok: true, to: chat ? chatTarget(chat) : OTTER_TARGET };
+    },
     sendText: (ctx) => deliver(ctx),
     sendPayload: (ctx) => deliver(ctx, ctx.payload),
   },
 });
 
+/** Runs what you sent from Otter as the channel's owner: an inbound message, as from any chat app. */
+async function receive(api, cfg, message) {
+  const { sessionKey, agentId, text, messageId, files } = message;
+  const inbound = api.runtime.channel.inbound;
+  const saved = await Promise.all(
+    files.map((file) =>
+      api.runtime.channel.media.saveMediaBuffer(
+        Buffer.from(file.content, "base64"),
+        file.mimeType,
+        "inbound",
+        undefined,
+        typeof file.fileName === "string" ? file.fileName : undefined,
+      ),
+    ),
+  );
+  const channelIngress = await inbound.ingress.resolve({
+    channelId: "otter",
+    accountId: "default",
+    identity: owner,
+    subject: { stableId: OWNER },
+    conversation: { kind: "direct", id: sessionKey },
+    contextBinding: { agentId, sessionKey, messageId, inboundEventKind: "user_request" },
+    event: { kind: "message", authMode: "inbound", mayPair: false },
+    policy: { dmPolicy: "allowlist" },
+    allowFrom: [OWNER],
+    readStoreAllowFrom: async () => [],
+  });
+  const to = chatTarget(sessionKey);
+  const ctxPayload = await inbound.buildContext({
+    channelIngress,
+    channel: "otter",
+    accountId: "default",
+    messageId,
+    timestamp: Date.now(),
+    from: `otter:${OWNER}`,
+    sender: { id: OWNER, name: "Otter" },
+    conversation: { kind: "direct", id: sessionKey },
+    route: { agentId, accountId: "default", routeSessionKey: sessionKey },
+    reply: { to, originatingTo: to },
+    message: { body: text, rawBody: text, commandBody: text, bodyForAgent: text },
+    ...(saved.length > 0 && {
+      media: toInboundMediaFacts(
+        saved.map((file) => ({ path: file.path, contentType: file.contentType, messageId })),
+      ),
+    }),
+  });
+  await inbound.dispatch({
+    cfg,
+    channel: "otter",
+    accountId: "default",
+    route: { agentId, sessionKey },
+    ctxPayload,
+    replyOptions: { runId: messageId, sourceReplyDeliveryMode: "automatic" },
+    // The reply is kept in the chat's session, which Otter is reading as it's written.
+    delivery: { deliver: async () => ({ visibleReplySent: true }) },
+  });
+}
+
 export default defineChannelPluginEntry({
   id: "otter",
   name: "Otter",
   description: "Send what your agents want you to see to Otter.",
   plugin: otterChannel,
   registerFull(api) {
+    // What you send in an Otter chat, as the gateway's owner: `chat.send`'s run, with the authority
+    // a configured channel owner has (an automation another chat made, for one).
+    api.registerGatewayMethod(
+      "otter.chat.send",
+      async ({ params, context }) => {
+        const cfg = context?.getRuntimeConfig?.() ?? api.config;
+        if (!isOwnerConfigured(cfg)) throw new Error("otter_not_owner");
+        const message = messageOf(params);
+        receiving.runInAsyncScope(() =>
+          runDetachedWebhookWork(() => receive(api, cfg, message)).catch((error) =>
+            api.logger?.warn?.(`otter: a message from Otter failed: ${String(error)}`),
+          ),
+        );
+        return { runId: message.messageId, status: "started" };
+      },
+      { scope: "operator.admin" },
+    );
+
     // Otter's preview of a file an agent linked: PDFs and other files sessions.files.get won't send.
     api.registerGatewayMethod(
       "otter.files.get",
@@ -76,5 +192,56 @@ export default defineChannelPluginEntry({
       },
       { scope: "operator.read" },
     );
+
+    // Take control of the agent's browser from Otter: your input goes to the tab as a person's
+    // would, and the agent's browser waits until you hand it back.
+    const control = createBrowserControl({
+      async cdpUrlFor(profile) {
+        const result = await dispatchGatewayMethod("browser.request", {
+          method: "GET",
+          path: "/profiles",
+        });
+        if (!result?.ok)
+          throw new Error(result?.error?.message ?? "Couldn't find the agent's browser.");
+        const profiles = result.payload?.profiles ?? [];
+        const found = profiles.find((p) => (profile ? p.name === profile : p.isDefault));
+        const url = typeof found?.cdpUrl === "string" ? URL.parse(found.cdpUrl) : null;
+        if (!url || !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname))
+          throw new Error("This browser can't be controlled from Otter.");
+        return url.href;
+      },
+    });
+    const targetOf = (params) => {
+      if (typeof params?.targetId !== "string" || !params.targetId)
+        throw new Error("targetId is required");
+      return params.targetId;
+    };
+    api.registerGatewayMethod(
+      "otter.browser.control",
+      ({ params }) => {
+        if (params?.take === true) control.take(targetOf(params));
+        else control.release(targetOf(params));
+        return { controlled: params?.take === true };
+      },
+      { scope: "operator.admin" },
+    );
+    api.registerGatewayMethod(
+      "otter.browser.input",
+      async ({ params }) => {
+        const profile = typeof params?.profile === "string" ? params.profile : undefined;
+        await control.input(targetOf(params), profile, params?.seq, params?.events);
+        return {};
+      },
+      { scope: "operator.admin" },
+    );
+    api.on("before_tool_call", (event) =>
+      /(^|__)browser$/.test(event.toolName) && control.anyHeld()
+        ? {
+            block: true,
+            blockReason:
+              "The user has taken control of the browser. Wait until they hand it back, then try again.",
+          }
+        : undefined,
+    );
   },
 });

package/openclaw.plugin.json

@@ -3,6 +3,7 @@
   "name": "Otter",
   "description": "Send what your agents want you to see to Otter.",
   "activation": { "onStartup": true },
+  "contracts": { "gatewayMethodDispatch": ["authenticated-request"] },
   "channels": ["otter"],
   "configSchema": { "type": "object", "additionalProperties": false, "properties": {} },
   "channelConfigs": {

package/otter.js

@@ -5,25 +5,48 @@ export const inboxKey = (agentId) => `agent:${agentId}:otter:inbox`;
 
 export const OTTER_TARGET = "inbox";
 
+/** Who sends what you type in Otter: the gateway's owner once `commands.ownerAllowFrom` lists `otter:owner`. */
+export const OWNER = "owner";
+export const OWNER_ENTRY = `otter:${OWNER}`;
+
+/** A chat in Otter is a target of its own: what's sent to it is kept in its session. */
+export const chatTarget = (sessionKey) => `chat:${sessionKey}`;
+
 export const agentOf = (sessionKey) =>
   typeof sessionKey === "string" && sessionKey.startsWith("agent:")
     ? sessionKey.split(":")[1]
     : undefined;
 
+/** The chat a target names, if it names one. */
+export function chatOf(to) {
+  const target = typeof to === "string" ? to.trim().replace(/^otter:/, "") : "";
+  if (!target.startsWith("chat:")) return undefined;
+  const sessionKey = target.slice("chat:".length);
+  return agentOf(sessionKey) ? sessionKey : undefined;
+}
+
+/** The session a delivery to this target is kept in. */
+export const sessionOf = (to, agentId) => chatOf(to) ?? inboxKey(agentId);
+
+export const isOwnerConfigured = (cfg) =>
+  Array.isArray(cfg?.commands?.ownerAllowFrom) &&
+  cfg.commands.ownerAllowFrom.some((entry) => String(entry).trim() === OWNER_ENTRY);
+
 /**
  * What the relay is told of a delivery: never its text, only where to read it.
  * Otter's apps read it from the gateway themselves.
  */
-export function pointerOf({ sessionKey, payload, at = Date.now() }) {
+export function pointerOf({ sessionKey, to, payload, at = Date.now() }) {
   const data = payload?.channelData ?? {};
   const approval = data.execApproval;
   const question = data.askUser;
   const session = approval?.sessionKey ?? sessionKey;
-  const agentId = approval?.agentId ?? agentOf(session) ?? "main";
+  const chat = chatOf(to);
+  const agentId = approval?.agentId ?? agentOf(chat ?? session) ?? "main";
   return {
     kind: approval ? "approval" : question ? "question" : "message",
     agentId,
-    sessionKey: approval || question ? (session ?? null) : inboxKey(agentId),
+    sessionKey: approval || question ? (session ?? null) : (chat ?? inboxKey(agentId)),
     ...(approval && {
       approvalId: String(approval.approvalId),
       approvalKind: approval.approvalKind === "plugin" ? "plugin" : "exec",
@@ -56,3 +79,19 @@ export async function tell(account, pointer, { fetch = globalThis.fetch, signal
   if (!response.ok) throw new Error(`Otter's relay answered ${response.status}`);
   return true;
 }
+
+/** A message sent from Otter, checked: where it goes and what it carries. */
+export function messageOf(params) {
+  const { sessionKey, message, idempotencyKey, attachments } = params ?? {};
+  if (typeof sessionKey !== "string" || !agentOf(sessionKey))
+    throw new Error("sessionKey must be an agent's session");
+  if (typeof idempotencyKey !== "string" || !idempotencyKey.trim())
+    throw new Error("idempotencyKey is required");
+  const files = Array.isArray(attachments) ? attachments : [];
+  for (const file of files)
+    if (typeof file?.content !== "string" || typeof file?.mimeType !== "string")
+      throw new Error("each attachment needs its content (base64) and mimeType");
+  const text = typeof message === "string" ? message : "";
+  if (!text.trim() && files.length === 0) throw new Error("message or attachments are required");
+  return { sessionKey, agentId: agentOf(sessionKey), text, messageId: idempotencyKey, files };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@otterware/openclaw-otter",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "Otter as an OpenClaw channel: what your agents send you waits in Otter, and Otter tells your devices.",
   "license": "MIT",
   "repository": {
@@ -10,6 +10,7 @@
   },
   "files": [
     "index.js",
+    "control.js",
     "files.js",
     "otter.js",
     "openclaw.plugin.json"