openzoo 0.50.39 → 0.50.40

package/lib/cursorbackend.js

@@ -1129,12 +1129,99 @@ function appendLine(agentId, role, text, extra = {}) {
       timestampMs: ts,
       requestId,
       ...(extra.author && typeof extra.author === 'object' ? { author: extra.author } : {}),
+      ...(extra.ephemeral ? { ephemeral: true } : {}),
     };
   }
   t.entries.push(e);
   saveTranscripts();
   return e;
 }
+
+/** One zooComplete per agent. A new sendPrompt (or Stop) aborts the previous
+ *  loop so "try again" does not stack 32-step exec storms with an empty canvas. */
+export function createZooTurnQueue() {
+  const inflight = new Map();
+  return {
+    begin(agentId, nonce) {
+      const id = String(agentId || '');
+      const prev = inflight.get(id);
+      if (prev && prev.nonce !== nonce) {
+        try { prev.abort.abort(new Error('superseded')); } catch { /* */ }
+      }
+      const abort = new AbortController();
+      inflight.set(id, { nonce, abort });
+      return abort;
+    },
+    isCurrent(agentId, nonce) {
+      return inflight.get(String(agentId || ''))?.nonce === nonce;
+    },
+    end(agentId, nonce) {
+      const id = String(agentId || '');
+      const cur = inflight.get(id);
+      if (cur && cur.nonce === nonce) inflight.delete(id);
+    },
+    abort(agentId) {
+      const id = String(agentId || '');
+      const prev = inflight.get(id);
+      if (!prev) return 0;
+      try { prev.abort.abort(new Error('interrupted')); } catch { /* */ }
+      inflight.delete(id);
+      return 1;
+    },
+  };
+}
+const zooTurns = createZooTurnQueue();
+
+export function formatZooProgress({ step, maxSteps, names, command } = {}) {
+  const tools = Array.isArray(names) ? names.filter(Boolean).join(', ') : String(names || 'tools');
+  const cmd = command ? ` ${JSON.stringify(String(command).slice(0, 80))}` : '';
+  return `Working on your Mac (step ${Number(step) + 1}/${maxSteps || '?'}): ${tools}${cmd}`;
+}
+
+/** One canvas line per tool. Asar ingest is append-only — mutating a working
+ *  bubble is still silence. Keep it short; the model history skips ephemeral. */
+export function formatZooToolLine({ name, args = {}, result } = {}) {
+  const detail = args.command || args.path || args.name || args.window || '';
+  const head = detail
+    ? `${name} ${JSON.stringify(String(detail).slice(0, 90))}`
+    : String(name || 'tool');
+  let tail = String(result || '').replace(/\s+/g, ' ').trim();
+  if (tail.startsWith('{')) {
+    try {
+      const o = JSON.parse(String(result));
+      if (o && typeof o === 'object') {
+        tail = [o.path, o.id, o.name, o.bytes != null ? `${o.bytes}b` : '', o.ok === true ? 'ok' : '']
+          .filter(Boolean).join(' ') || tail;
+      }
+    } catch { /* raw */ }
+  }
+  tail = tail.slice(0, 160);
+  return `→ ${head}${tail ? `\n${tail}` : ''}`;
+}
+
+export function combinedAbortSignal(a, b) {
+  if (!a) return b;
+  if (!b) return a;
+  if (typeof AbortSignal.any === 'function') return AbortSignal.any([a, b]);
+  const c = new AbortController();
+  const on = () => { try { c.abort(); } catch { /* */ } };
+  if (a.aborted || b.aborted) { on(); return c.signal; }
+  a.addEventListener('abort', on, { once: true });
+  b.addEventListener('abort', on, { once: true });
+  return c.signal;
+}
+
+export function isSupersededError(e) {
+  const m = String(e?.message || e || '');
+  return /superseded|interrupted/i.test(m);
+}
+
+function paintChatUpdate(agentId, note, extra = {}) {
+  const line = fanoutLine(agentId, 'assistant', note, { ...extra, ephemeral: true });
+  ssePush('transcript', { ...gatewayEntry(line), agentId });
+  bumpAgent(agentId, { preview: note, notify: false });
+  return line;
+}
 function fanoutLine(primaryId, role, text, extra = {}) {
   // Only the addressed agent. Writing to every tailedAgents id mixed canvases
   // so a new/empty chat showed someone else's thread.
@@ -1254,7 +1341,7 @@ function mergeAgentLists(remote) {
   return sortAgentsByActivity(out);
 }
 function gatewayEntry(e) {
-  const { seq, pulledRemote, promptRaw, ...rest } = e;
+  const { seq, pulledRemote, promptRaw, ephemeral, ...rest } = e;
   return rest;
 }
 
@@ -1286,6 +1373,7 @@ function historyMessages(agentId, currentPrompt) {
     if (!e || typeof e !== 'object') continue;
     let role = null;
     let text = '';
+    if (e.ephemeral) continue;
     if (e.kind === 'send-message' || e.role === 'assistant' || e.kind === 'assistant') {
       text = entryPlainText(e.message ?? e.content ?? e.text);
       role = 'assistant';
@@ -1622,9 +1710,10 @@ async function execLocal(command, cwd, log) {
     return `${out}${err}`.trim() || `(exit ${got.exitCode ?? 0})`;
   }
   log(`cursor-backend:      exec cwd=${dir} ${JSON.stringify(String(command).slice(0, 80))}`);
+  const execMs = Math.min(180000, Math.max(15000, Number(process.env.OPENZOO_EXEC_TIMEOUT_MS || 90000)));
   const { stdout, stderr } = await execFileAsync('/bin/zsh', ['-lc', command], {
     cwd: dir,
-    timeout: 30000,
+    timeout: execMs,
     maxBuffer: 2 * 1024 * 1024,
     env: process.env,
   });
@@ -1672,8 +1761,62 @@ const LOCAL_TOOLS = [
       parameters: { type: 'object', properties: { path: { type: 'string' } }, required: ['path'] },
     },
   },
+  {
+    type: 'function',
+    function: {
+      name: 'screenshot',
+      description: 'Capture the Mac screen (or the front window) and attach the image so you can SEE the UI. Use this before filling forms or clicking. Do not guess at a dashboard you have not screenshotted this turn.',
+      parameters: {
+        type: 'object',
+        properties: {
+          window: { type: 'string', description: 'Optional window title substring; default is the full display.' },
+        },
+      },
+    },
+  },
+  {
+    type: 'function',
+    function: {
+      name: 'create_agent',
+      description: 'Mint another Grok Bot in the sidebar. Use this when asked to spawn bots — talking about spawning is not enough.',
+      parameters: {
+        type: 'object',
+        properties: {
+          name: { type: 'string', description: 'Sidebar label for the new bot.' },
+          select: { type: 'boolean', description: 'If true (default), switch the UI to the new bot.' },
+        },
+        required: ['name'],
+      },
+    },
+  },
 ];
 
+export const LOCAL_TOOL_NAMES = LOCAL_TOOLS.map((t) => t.function.name);
+
+async function captureScreenshot(log) {
+  const dir = path.join(os.tmpdir(), 'openzoo-screens');
+  fs.mkdirSync(dir, { recursive: true });
+  const stamp = Date.now();
+  const raw = path.join(dir, `oz-${stamp}.png`);
+  const jpg = path.join(dir, `oz-${stamp}.jpg`);
+  await execFileAsync('screencapture', ['-x', '-C', raw], { timeout: 15000 });
+  if (!fs.existsSync(raw) || fs.statSync(raw).size < 80) {
+    throw new Error('screencapture wrote nothing — Screen Recording permission may be off for Grok Bot / Terminal');
+  }
+  try {
+    await execFileAsync('sips', [
+      '-Z', '1400', '-s', 'format', 'jpeg', '-s', 'formatOptions', '70',
+      raw, '--out', jpg,
+    ], { timeout: 15000 });
+    if (fs.existsSync(jpg) && fs.statSync(jpg).size > 80) {
+      return { path: jpg, mime: 'image/jpeg', buf: fs.readFileSync(jpg) };
+    }
+  } catch (e) {
+    log(`cursor-backend:      sips screenshot resize failed: ${e.message}`);
+  }
+  return { path: raw, mime: 'image/png', buf: fs.readFileSync(raw) };
+}
+
 async function runLocalTool(name, args, log) {
   try {
     if (name === 'read_file') {
@@ -1697,6 +1840,24 @@ async function runLocalTool(name, args, log) {
     if (name === 'exec') {
       return await execLocal(String(args.command || ''), args.cwd, log);
     }
+    if (name === 'screenshot') {
+      const shot = await captureScreenshot(log);
+      return JSON.stringify({
+        ok: true,
+        path: shot.path,
+        mime: shot.mime,
+        bytes: shot.buf.length,
+        dataUrl: `data:${shot.mime};base64,${shot.buf.toString('base64')}`,
+      });
+    }
+    if (name === 'create_agent') {
+      const agent = mintLocalAgent({
+        name: String(args.name || args.title || 'New Bot').slice(0, 80),
+      });
+      pushCreatedAgent(agent, { select: args.select !== false });
+      log(`cursor-backend:      create_agent tool id=${agent.id} name=${JSON.stringify(agent.name)}`);
+      return JSON.stringify({ ok: true, id: agent.id, name: agent.name });
+    }
     return `unknown tool ${name}`;
   } catch (e) {
     return `ERROR ${e.message}`;
@@ -1718,7 +1879,7 @@ function zooTextFromMessage(msg, data) {
   return '';
 }
 
-async function zooComplete(prompt, log, agentId, parsed = {}) {
+async function zooComplete(prompt, log, agentId, parsed = {}, opts = {}) {
   const model = currentModel(agentId);
   const helper = localExecSse.size > 0;
   const visitor = visitorFromSend(parsed);
@@ -1726,8 +1887,12 @@ async function zooComplete(prompt, log, agentId, parsed = {}) {
   const spoken = visitor && !/^You are /.test(String(prompt || ''))
     ? labeledVisitorPrompt(visitor, prompt)
     : prompt;
+  const throwIfAborted = () => {
+    if (opts.signal?.aborted) throw new Error('superseded');
+  };
   await ensureTranscriptHydrated(agentId, log);
   log(`cursor-backend:      zoo POST :8402 model=${model} helper=${helper ? localExecSse.size : 0} hist=${historyMessages(agentId, spoken).length}${chatOnly ? ` visitor=${visitor.shortname} chat-only` : ''} ${JSON.stringify((spoken || '').slice(0, 60))}`);
+  if (typeof opts.onProgress === 'function') opts.onProgress('Working on your Mac…');
 
   const images = [];
   const textFiles = [];
@@ -1783,10 +1948,12 @@ async function zooComplete(prompt, log, agentId, parsed = {}) {
         : [
           `You are ${model} served through openzoo inside Grok Bot.`,
           `You HAVE local tools on the user's computer via ${via}.`,
-          'Tools: read_file, write_file, exec, list_dir. USE THEM. Write files to disk instead of pasting giant HTML into chat.',
+          'Tools: read_file, write_file, exec, list_dir, screenshot, create_agent. USE THEM. Write files to disk instead of pasting giant HTML into chat.',
+          'create_agent mints a new Grok Bot in the sidebar. When asked to spawn bots, call it — do not only describe spawning.',
+          'screenshot captures the display and attaches the image. For any on-screen form, dashboard, or click target, screenshot first this turn. Do not guess at UI you have not seen.',
           'Pasted images arrive as attachments — you can see them when present. Do not claim you cannot see images if they are in this turn.',
           'Never claim you lack filesystem access or local-exec. If a tool errors, report the error.',
-          'After using tools you MUST still write a normal chat reply: what you did, file paths written, and what to open. Empty content is a bug.',
+          'After using tools you MUST still write a normal chat reply: what you did, file paths written, and what to open. Empty content is a bug. A new user message cancels this turn — leave a visible reply before that happens.',
           'Do not stop mid-task. Do not write "Stopped on research", "nothing to open yet", "no app files written this turn", or "say go again". Keep using tools until the files the user asked for exist on disk THIS turn. Summarize only after those writes succeed.',
           'A spend footer is appended after your reply by the host — ignore it.',
           'Prior turns of THIS Grok Bot chat are in the messages below. Do not claim the thread starts blank or that earlier questions did not arrive.',
@@ -1823,12 +1990,13 @@ async function zooComplete(prompt, log, agentId, parsed = {}) {
   const usedTools = [];
   const KEEP_GOING = 'Do not stop. Do not wait for another message. Write the files to disk now with write_file / exec. Keep going until the requested paths exist. A summary is only allowed after the files are written.';
   const zooPost = async (payload) => {
+    throwIfAborted();
     const ms = Number(process.env.OPENZOO_ASK_TIMEOUT_MS || 10 * 60_000);
     const post = () => fetch('http://127.0.0.1:8402/v1/chat/completions', {
       method: 'POST',
       headers: { 'content-type': 'application/json', authorization: 'Bearer sk-openzoo' },
       body: JSON.stringify(payload),
-      signal: AbortSignal.timeout(ms),
+      signal: combinedAbortSignal(opts.signal, AbortSignal.timeout(ms)),
     });
     let r;
     let lastErr;
@@ -1872,7 +2040,9 @@ async function zooComplete(prompt, log, agentId, parsed = {}) {
     return { r, data };
   };
   let keepGoingNudge = 0;
+  const pendingVision = [];
   for (let step = 0; step < maxSteps; step++) {
+    throwIfAborted();
     const payload = { model, messages, max_tokens: maxTok };
     if (!chatOnly) {
       payload.tools = LOCAL_TOOLS;
@@ -1896,25 +2066,49 @@ async function zooComplete(prompt, log, agentId, parsed = {}) {
         }
         continue;
       }
-      log(`cursor-backend:      zoo tools step=${step} n=${calls.length} ${calls.map((c) => c.function?.name || c.name).join(',')}`);
+      const names = calls.map((c) => c.function?.name || c.name);
+      log(`cursor-backend:      zoo tools step=${step} n=${calls.length} ${names.join(',')}`);
       messages.push(msg);
       for (const c of calls) {
+        throwIfAborted();
         let args = {};
         try { args = JSON.parse(c.function?.arguments || c.arguments || '{}'); } catch { args = {}; }
         const name = c.function?.name || c.name || '';
-        const result = await runLocalTool(name, args, log);
+        let result = await runLocalTool(name, args, log);
+        if (name === 'screenshot') {
+          try {
+            const shot = JSON.parse(result);
+            if (shot?.dataUrl) {
+              pendingVision.push({ path: shot.path, dataUrl: shot.dataUrl });
+              result = `screenshot ${shot.path} ${shot.bytes}b — image attached, look at it before acting`;
+            }
+          } catch { /* keep raw */ }
+        }
         usedTools.push({
           name,
           path: args.path,
           command: args.command ? String(args.command).slice(0, 120) : undefined,
           note: String(result).slice(0, 200),
         });
+        if (typeof opts.onProgress === 'function') {
+          opts.onProgress(formatZooToolLine({ name, args, result }));
+        }
         messages.push({
           role: 'tool',
           tool_call_id: c.id,
           content: String(result).slice(0, 120000),
         });
       }
+      if (pendingVision.length) {
+        messages.push({
+          role: 'user',
+          content: [
+            ...pendingVision.map((img) => ({ type: 'image_url', image_url: { url: img.dataUrl } })),
+            { type: 'text', text: pendingVision.map((img) => `screenshot: ${img.path}`).join('\n') },
+          ],
+        });
+        pendingVision.length = 0;
+      }
       continue;
     }
     text = zooTextFromMessage(msg, data);
@@ -2081,7 +2275,7 @@ async function handleHijackedPodHttp(req, res, full, body, log) {
     'getBotTemplateExportPolicy', 'getTeachRecordingStatus', 'isGlobalSearchEnabled',
     'isEgressTunnelAvailable', 'listBoxMcpServers',
     'setWindowFocused', 'getAgentAutomations',
-    'interruptAgentRun', 'requestDiskSaverAudit',
+    'requestDiskSaverAudit',
     'setAgentUnread', 'setAgentHiddenFromSidebar', 'setAgentNotificationsEnabled',
     'setAgentNotifyOnUpdates', 'setAgentAvatarBytes', 'getAgentAvatar',
   ]);
@@ -2100,6 +2294,15 @@ async function handleHijackedPodHttp(req, res, full, body, log) {
     log(`cursor-backend:      createAgent local id=${agent.id} name=${JSON.stringify(agent.name)}`);
     return true;
   }
+  if (name === 'interruptAgentRun') {
+    let parsed = {};
+    try { parsed = JSON.parse(String(body || '{}')); } catch { parsed = {}; }
+    const id = String(parsed.id || parsed.agentId || focusedAgentId || '');
+    const n = id ? zooTurns.abort(id) : 0;
+    jsonSend(res, { ok: true, interrupted: n, id });
+    log(`cursor-backend:      interruptAgentRun id=${id || '?'} n=${n}`);
+    return true;
+  }
   if (name === 'kickstartAgent') {
     let parsed = {};
     try { parsed = JSON.parse(String(body || '{}')); } catch { parsed = {}; }
@@ -2262,6 +2465,7 @@ async function handleHijackedPodHttp(req, res, full, body, log) {
     log(`cursor-backend:      >> sendPrompt agent=${agentId} keys=${Object.keys(parsed).join(',')} attachments=${attN}${visitor ? ` visitor=${visitor.shortname}` : ''} prompt=${JSON.stringify((prompt || '').slice(0, 80))}`);
     lastSendEchoId = String(nonce);
     jsonSend(res, { accepted: true });
+    const turn = zooTurns.begin(agentId, nonce);
     const userLine = fanoutLine(agentId, 'user', uiText, {
       clientNonce: nonce,
       requestId: nonce,
@@ -2275,6 +2479,7 @@ async function handleHijackedPodHttp(req, res, full, body, log) {
     setImmediate(async () => {
       let text = '';
       let grouped = false;
+      const stillCurrent = () => zooTurns.isCurrent(agentId, nonce) && !turn.signal.aborted;
       try {
         if (modelCmd) {
           const want = modelCmd[1];
@@ -2298,15 +2503,31 @@ async function handleHijackedPodHttp(req, res, full, body, log) {
             grouped = true;
             await runGroupQueue({ agentId, humanPrompt: prompt, parsed, nonce, log });
           } else {
-            const z = await zooComplete(prompt, log, agentId, parsed);
+            const z = await zooComplete(prompt, log, agentId, parsed, {
+              signal: turn.signal,
+              onProgress: (note) => {
+                if (!stillCurrent()) return;
+                paintChatUpdate(agentId, note, { clientNonce: nonce, requestId: nonce });
+              },
+            });
             text = z.text;
           }
         }
       } catch (e) {
+        if (!stillCurrent() || isSupersededError(e)) {
+          log(`cursor-backend:      sendPrompt superseded agent=${agentId} nonce=${nonce} ${e.message}`);
+          zooTurns.end(agentId, nonce);
+          return;
+        }
         grouped = false;
         text = `openzoo error: ${e.message}`;
         log(`cursor-backend:      sendPrompt zoo failed: ${e.message}`);
       }
+      if (!stillCurrent()) {
+        zooTurns.end(agentId, nonce);
+        log(`cursor-backend:      sendPrompt dropped stale agent=${agentId} nonce=${nonce}`);
+        return;
+      }
       if (!grouped) {
         const line = fanoutLine(agentId, 'assistant', text, { clientNonce: nonce, requestId: nonce });
         ssePush('transcript', { ...gatewayEntry(line), agentId });
@@ -2315,6 +2536,7 @@ async function handleHijackedPodHttp(req, res, full, body, log) {
       } else {
         log(`cursor-backend:      sendPrompt group done agent=${agentId}`);
       }
+      zooTurns.end(agentId, nonce);
     });
     return true;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openzoo",
-  "version": "0.50.39",
+  "version": "0.50.40",
   "description": "Local x402-paying proxy + MCP server for openzoo.fun — point any OpenAI-compatible harness (Cursor, Claude Code, aider, SDKs) at localhost and it pays per call from a local burner wallet. Solana and Base rails live; Robinhood experimental.",
   "license": "MIT",
   "type": "module",