@integrity-labs/agt-cli 0.28.833 → 0.28.835

package/dist/mcp/computer-use-proxy.js

@@ -0,0 +1,285 @@
+#!/usr/bin/env node
+import { createRequire as __augmentedCreateRequire } from 'module';
+globalThis.require ??= __augmentedCreateRequire(import.meta.url);
+
+// src/computer-use-proxy.ts
+import { createInterface } from "readline";
+import { spawn } from "child_process";
+import { fileURLToPath } from "url";
+import { resolve as resolvePath } from "path";
+var INCLUDE_SCREENSHOT_PARAM = "include_screenshot";
+var STRIPPED_IMAGE_STUB = `[screenshot omitted to save tokens \u2014 pass ${INCLUDE_SCREENSHOT_PARAM}: true to receive it. Element indices in the accessibility tree above are usually enough to act.]`;
+var RELIABILITY_NOTE = `
+
+IMPORTANT: isError:false does NOT mean this action took effect. Some key combinations are silently ignored (super+n opens a window; super+a returns the same isError:false and does nothing). Verify by comparing the accessibility tree in this response against what you expected \u2014 it reflects the state after the action. set_value with an empty string is a no-op.`;
+function isImageBlock(block) {
+  return typeof block === "object" && block !== null && block.type === "image";
+}
+var WINDOW_FURNITURE_ROLES = [
+  "container",
+  "standard window",
+  "close button",
+  "minimise button",
+  "minimize button",
+  "zoom button",
+  "full screen button",
+  "menu bar"
+];
+function treeHasActionableContent(text) {
+  if (typeof text !== "string") return false;
+  let skipDeeperThan = null;
+  for (const raw of text.split("\n")) {
+    const m = /^(\s*)(\d+) (.+)$/.exec(raw);
+    if (!m) continue;
+    const indent = m[1].length;
+    const rest = m[3].toLowerCase();
+    if (skipDeeperThan !== null) {
+      if (indent > skipDeeperThan) continue;
+      skipDeeperThan = null;
+    }
+    if (rest.startsWith("menu bar")) {
+      skipDeeperThan = indent;
+      continue;
+    }
+    if (WINDOW_FURNITURE_ROLES.some((role) => rest.startsWith(role))) continue;
+    return true;
+  }
+  return false;
+}
+function stripImages(result) {
+  if (typeof result !== "object" || result === null) return { result, stripped: 0 };
+  const content = result.content;
+  if (!Array.isArray(content)) return { result, stripped: 0 };
+  if (!content.some(isImageBlock)) return { result, stripped: 0 };
+  const next = [];
+  let stripped = 0;
+  for (const block of content) {
+    if (isImageBlock(block)) {
+      stripped += 1;
+      next.push({ type: "text", text: STRIPPED_IMAGE_STUB });
+    } else {
+      next.push(block);
+    }
+  }
+  return { result: { ...result, content: next }, stripped };
+}
+function decorateToolsList(result) {
+  if (typeof result !== "object" || result === null) return result;
+  const tools = result.tools;
+  if (!Array.isArray(tools)) return result;
+  const decorated = tools.map((tool) => {
+    if (typeof tool !== "object" || tool === null) return tool;
+    const t = tool;
+    const schema = t["inputSchema"] ?? {};
+    const props = schema["properties"] ?? {};
+    return {
+      ...t,
+      description: `${typeof t["description"] === "string" ? t["description"] : ""}${RELIABILITY_NOTE}`,
+      inputSchema: {
+        ...schema,
+        type: schema["type"] ?? "object",
+        properties: {
+          ...props,
+          [INCLUDE_SCREENSHOT_PARAM]: {
+            type: "boolean",
+            description: "Return the screenshot for this call. Omitted by default: it is ~99% of the response and element indices in the accessibility tree are usually enough to act on."
+          }
+        }
+      }
+    };
+  });
+  return { ...result, tools: decorated };
+}
+function takeIncludeScreenshot(params) {
+  if (typeof params !== "object" || params === null) return { include: false, params };
+  const p = params;
+  if (typeof p.arguments !== "object" || p.arguments === null) {
+    return { include: false, params };
+  }
+  const args = p.arguments;
+  if (!(INCLUDE_SCREENSHOT_PARAM in args)) return { include: false, params };
+  const { [INCLUDE_SCREENSHOT_PARAM]: raw, ...rest } = args;
+  return { include: raw === true, params: { ...params, arguments: rest } };
+}
+function logErr(msg) {
+  process.stderr.write(`[computer-use-proxy] ${msg}
+`);
+}
+var allowAll = async () => null;
+function createProxy(deps) {
+  const beforeForward = deps.beforeForward ?? allowAll;
+  const pending = /* @__PURE__ */ new Map();
+  async function handleDownstream(line) {
+    const text = line.trim();
+    if (!text) return [];
+    let frame;
+    try {
+      frame = JSON.parse(text);
+    } catch {
+      logErr("dropping non-JSON frame from client");
+      return [];
+    }
+    if (frame.method !== "tools/call") {
+      return [text];
+    }
+    const isRequest = "id" in frame;
+    const { include, params } = takeIncludeScreenshot(frame.params);
+    const toolName = typeof params?.name === "string" ? params.name : "";
+    const args = params?.arguments;
+    const short = await beforeForward(toolName, args);
+    if (short) {
+      if (!isRequest) {
+        logErr(`short-circuited notification tools/call ${toolName} (no reply sent)`);
+        return [];
+      }
+      return [JSON.stringify({ jsonrpc: "2.0", id: frame.id, result: short })];
+    }
+    if (isRequest) {
+      pending.set(String(frame.id), { method: "tools/call", includeScreenshot: include });
+    }
+    return [JSON.stringify({ ...frame, params })];
+  }
+  async function handleUpstream(line) {
+    const text = line.trim();
+    if (!text) return [];
+    let frame;
+    try {
+      frame = JSON.parse(text);
+    } catch {
+      logErr("dropping non-JSON frame from upstream");
+      return [];
+    }
+    if (!("id" in frame)) return [text];
+    const key = String(frame.id);
+    if (frame.result === void 0) {
+      pending.delete(key);
+      return [text];
+    }
+    const p = pending.get(key);
+    if (!p) {
+      const decorated = decorateToolsList(frame.result);
+      if (decorated !== frame.result) {
+        return [JSON.stringify({ ...frame, result: decorated })];
+      }
+      return [text];
+    }
+    pending.delete(key);
+    if (p.includeScreenshot) return [text];
+    const firstText = frame.result?.content;
+    if (Array.isArray(firstText)) {
+      const treeBlock = firstText.find(
+        (b) => typeof b === "object" && b !== null && b.type === "text"
+      );
+      if (treeBlock && !treeHasActionableContent(treeBlock.text)) {
+        logErr("tree carries no actionable content; keeping screenshot");
+        return [text];
+      }
+    }
+    const { result, stripped } = stripImages(frame.result);
+    if (stripped === 0) return [text];
+    return [JSON.stringify({ ...frame, result })];
+  }
+  return { handleDownstream, handleUpstream };
+}
+function createWriter(sink) {
+  let lock = Promise.resolve();
+  return (lines) => {
+    lock = lock.then(async () => {
+      try {
+        for (const line of lines) {
+          if (!sink.write(line + "\n")) {
+            await new Promise((res) => sink.once("drain", res));
+          }
+        }
+      } catch (err) {
+        logErr(`write failed: ${err.message}`);
+      }
+    });
+    return lock;
+  };
+}
+async function wirePump(deps) {
+  const { child, clientIn, clientOut, proxy } = deps;
+  const exit = deps.onExit ?? ((code) => process.exit(code));
+  const toUpstream = createWriter(child.stdin);
+  const toClient = createWriter(clientOut);
+  child.stdin.on("error", (err) => {
+    logErr(`upstream stdin: ${err.message}`);
+  });
+  return new Promise((resolve) => {
+    let settled = false;
+    const finish = async (code) => {
+      if (settled) return;
+      settled = true;
+      await toClient([]);
+      exit(code);
+      resolve();
+    };
+    child.on("error", (err) => {
+      logErr(`upstream error: ${err.message}`);
+      void finish(1);
+    });
+    child.on("exit", (code) => {
+      logErr(`upstream exited with code ${code ?? "null"}`);
+      void finish(code ?? 1);
+    });
+    createInterface({ input: clientIn, crlfDelay: Infinity }).on("line", (line) => {
+      void proxy.handleDownstream(line).then(toUpstream).catch((err) => logErr(`downstream handler: ${err.message}`));
+    }).on("close", () => {
+      logErr("client disconnected; terminating upstream");
+      child.kill("SIGTERM");
+      void finish(0);
+    });
+    createInterface({ input: child.stdout, crlfDelay: Infinity }).on("line", (line) => {
+      void proxy.handleUpstream(line).then(toClient).catch((err) => logErr(`upstream handler: ${err.message}`));
+    });
+  });
+}
+async function main(argv = process.argv) {
+  const sep = argv.indexOf("--");
+  const upstream = sep >= 0 ? argv.slice(sep + 1) : [];
+  if (upstream.length === 0) {
+    logErr("no upstream command; usage: computer-use-proxy -- <cmd> [args...]");
+    process.exit(1);
+  }
+  let child;
+  try {
+    child = spawn(upstream[0], upstream.slice(1), { stdio: ["pipe", "pipe", "inherit"] });
+  } catch (err) {
+    logErr(`failed to spawn upstream: ${err.message}`);
+    process.exit(1);
+  }
+  await wirePump({
+    child,
+    clientIn: process.stdin,
+    clientOut: process.stdout,
+    proxy: createProxy({ upstream })
+  });
+}
+var invokedDirectly = (() => {
+  try {
+    const entry = process.argv[1];
+    if (!entry) return false;
+    return fileURLToPath(import.meta.url) === resolvePath(entry);
+  } catch {
+    return false;
+  }
+})();
+if (invokedDirectly) {
+  void main();
+}
+export {
+  INCLUDE_SCREENSHOT_PARAM,
+  RELIABILITY_NOTE,
+  STRIPPED_IMAGE_STUB,
+  allowAll,
+  createProxy,
+  createWriter,
+  decorateToolsList,
+  isImageBlock,
+  main,
+  stripImages,
+  takeIncludeScreenshot,
+  treeHasActionableContent,
+  wirePump
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@integrity-labs/agt-cli",
-  "version": "0.28.833",
+  "version": "0.28.835",
   "description": "Augmented Team CLI — agent provisioning and management",
   "type": "module",
   "engines": {
@@ -23,8 +23,8 @@
   },
   "scripts": {
     "build": "tsup && npm run build:mcp-assets && npm run build:cli-assets",
-    "build:mcp-assets": "mkdir -p dist/mcp && cp ../../packages/mcp/dist/index.js dist/mcp/index.js && cp ../../packages/mcp/dist/slack-channel.js dist/mcp/slack-channel.js && cp ../../packages/mcp/dist/direct-chat-channel.js dist/mcp/direct-chat-channel.js && cp ../../packages/mcp/dist/telegram-channel.js dist/mcp/telegram-channel.js && cp ../../packages/mcp/dist/teams-channel.js dist/mcp/teams-channel.js && cp ../../packages/mcp/dist/whatsapp-channel.js dist/mcp/whatsapp-channel.js && cp ../../packages/mcp/dist/whatsapp-link.js dist/mcp/whatsapp-link.js && cp ../../packages/mcp/dist/remote-oauth-proxy.js dist/mcp/remote-oauth-proxy.js && cp ../../packages/augmented-admin-mcp/dist/index.js dist/mcp/augmented-admin.js && cp ../../packages/augmented-support-mcp/dist/index.js dist/mcp/augmented-support.js && cp ../../packages/augmented-help-kb-mcp/dist/index.js dist/mcp/augmented-help-kb.js && cp ../../packages/origami-mcp-server/dist/index.js dist/mcp/origami.js && cp ../../packages/xero-mcp-server/dist/index.js dist/mcp/xero.js",
-    "build:cli-assets": "mkdir -p dist/assets && cp assets/impersonate-statusline.sh dist/assets/impersonate-statusline.sh && chmod +x dist/assets/impersonate-statusline.sh",
+    "build:mcp-assets": "mkdir -p dist/mcp && cp ../../packages/mcp/dist/index.js dist/mcp/index.js && cp ../../packages/mcp/dist/slack-channel.js dist/mcp/slack-channel.js && cp ../../packages/mcp/dist/direct-chat-channel.js dist/mcp/direct-chat-channel.js && cp ../../packages/mcp/dist/telegram-channel.js dist/mcp/telegram-channel.js && cp ../../packages/mcp/dist/teams-channel.js dist/mcp/teams-channel.js && cp ../../packages/mcp/dist/whatsapp-channel.js dist/mcp/whatsapp-channel.js && cp ../../packages/mcp/dist/whatsapp-link.js dist/mcp/whatsapp-link.js && cp ../../packages/mcp/dist/remote-oauth-proxy.js dist/mcp/remote-oauth-proxy.js && cp ../../packages/mcp/dist/computer-use-proxy.js dist/mcp/computer-use-proxy.js && cp ../../packages/augmented-admin-mcp/dist/index.js dist/mcp/augmented-admin.js && cp ../../packages/augmented-support-mcp/dist/index.js dist/mcp/augmented-support.js && cp ../../packages/augmented-help-kb-mcp/dist/index.js dist/mcp/augmented-help-kb.js && cp ../../packages/origami-mcp-server/dist/index.js dist/mcp/origami.js && cp ../../packages/xero-mcp-server/dist/index.js dist/mcp/xero.js",
+    "build:cli-assets": "mkdir -p dist/assets && cp assets/impersonate-statusline.sh dist/assets/impersonate-statusline.sh && chmod +x dist/assets/impersonate-statusline.sh && mkdir -p dist/assets/review && cp assets/review/post-review-findings.mjs dist/assets/review/post-review-findings.mjs && cp assets/review/obiwan-auth.mjs dist/assets/review/obiwan-auth.mjs && chmod +x dist/assets/review/post-review-findings.mjs",
     "dev": "tsx watch src/bin/agt.ts",
     "test": "vitest run",
     "typecheck": "bash ../../scripts/typecheck-guard.sh tsc --noEmit",