openzoo 0.48.4 → 0.48.6

package/lib/grokui.mjs

@@ -8,7 +8,7 @@
 import { exec } from 'node:child_process';
 import http from 'node:http';
 import { randomUUID } from 'node:crypto';
-import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
 import { homedir } from 'node:os';
 import path from 'node:path';
 import { brain, brainStream, PROXY } from './podagent.mjs';
@@ -112,6 +112,34 @@ workspace folder, not their real project. Same one-line-no-prose reply format:
                                                        text — web search only gives you short
                                                        snippets; use FETCH when asked to
                                                        "read" or quote something specific
+  MCP: <url>                                          list the tools an MCP server exposes
+  MCP: <url> | <tool> | {"arg": "value"}              CALL one of them, for real
+                                                       An MCP endpoint speaks JSON-RPC over
+                                                       POST. FETCH does a GET, so it will
+                                                       always come back 405 Method Not
+                                                       Allowed — that is NOT a broken URL and
+                                                       NOT a reason to curl it or write your
+                                                       own client. Use this directive; it
+                                                       does the initialize handshake, holds
+                                                       the session, and calls the tool.
+  LS: <path, or blank for the root>                   list a directory
+  GLOB: <pattern>                                     find files — *.js, **/*.test.ts, src/**
+  GREP: <regex> | <optional path or glob>             search file CONTENTS, with line numbers
+  EDIT: <path> | <exact old text> ||| <new text>      change PART of a file. Prefer this over
+                                                       WRITE for edits — WRITE replaces the
+                                                       whole file, so anything you don't
+                                                       reproduce from memory is destroyed.
+                                                       The old text must match byte for byte
+                                                       and appear exactly once; READ first.
+  TODO: <one item per line>                           set a visible checklist
+  TODO: done <n>                                      tick an item   (TODO: alone = show it)
+
+WORK IN PARALLEL. READ, LS, GLOB, GREP, FETCH, PEEK and MCP are read-only, and if you emit
+several of them in ONE reply the harness runs them CONCURRENTLY and returns every result
+together. Four files in one turn costs one round trip; four turns costs four, and you pay per
+call. Ask for everything you know you need at once instead of discovering it one file at a
+time. Mutating directives (RUN, WRITE, EDIT, SPAWN, SEND) stay sequential on purpose — racing
+them against each other corrupts the tree.
   RUN: <shell command>                                run a REAL shell command in this
                                                        thread's directory — by default this
                                                        pauses and waits for the user to
@@ -463,7 +491,59 @@ if (!loadThreads()) newThread('openzoo', null);
 // Parses a SPAWN/SEND/PING directive out of a reply, performs its side effect
 // (creating or messaging another thread), and returns the ack text to show in
 // place of the raw directive line — or null if the reply wasn't a directive.
+// Directives that only READ. These are safe to run at the same time, so a
+// reply carrying several of them costs one round trip instead of N — a model
+// that wants four files currently spends four full turns (and four payments)
+// fetching them one at a time.
+//
+// Deliberately excludes RUN / WRITE / EDIT / SPAWN / SEND: those mutate, and
+// concurrent mutation of the same tree is a race the model cannot reason
+// about. Reads fan out, writes stay sequential.
+const PARALLEL_DIRECTIVE = /^[ \t>*-]*(READ|LS|GLOB|GREP|FETCH|PEEK|MCP):[ \t]*(.+)$/gm;
+
+// Walk a thread dir once, cheaply, skipping the things nobody means to search.
+const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'build', '.next', '__pycache__', '.venv', 'venv']);
+function walkDir(base, rel = '', out = [], depth = 0) {
+  if (depth > 12 || out.length > 5000) return out;
+  let entries = [];
+  try { entries = readdirSync(path.join(base, rel), { withFileTypes: true }); } catch { return out; }
+  for (const e of entries) {
+    const r = rel ? path.join(rel, e.name) : e.name;
+    if (e.isDirectory()) {
+      if (SKIP_DIRS.has(e.name)) continue;
+      walkDir(base, r, out, depth + 1);
+    } else out.push(r);
+  }
+  return out;
+}
+
+// Glob -> RegExp. `**` crosses separators, `*` and `?` do not.
+function globToRe(glob) {
+  let re = '';
+  for (let i = 0; i < glob.length; i++) {
+    const c = glob[i];
+    if (c === '*') {
+      if (glob[i + 1] === '*') { re += '.*'; i++; if (glob[i + 1] === '/') i++; }
+      else re += '[^/]*';
+    } else if (c === '?') re += '[^/]';
+    else if ('.+^${}()|[]\\'.includes(c)) re += '\\' + c;
+    else re += c;
+  }
+  return new RegExp('^' + re + '$');
+}
+
 async function tryDirective(reply, originId) {
+  // FAN OUT FIRST. Each line is re-entered on its own, so every branch below
+  // stays single-directive and none of them had to learn about batching.
+  const batch = [...reply.matchAll(PARALLEL_DIRECTIVE)];
+  if (batch.length > 1) {
+    const results = await Promise.all(
+      batch.map((m) => tryDirective(m[0].replace(/^[ \t>*-]*/, ''), originId)
+        .catch((e) => `${m[1]}: ${e.message}`)),
+    );
+    return results.filter(Boolean).join('\n\n');
+  }
+
   const spawn = /^SPAWN:\s*([^|]+)\|\s*([\s\S]+)/.exec(reply);
   if (spawn) {
     const name = spawn[1].trim();
@@ -517,6 +597,109 @@ async function tryDirective(reply, originId) {
       return `${rel}:\n${data.slice(0, 4000)}${data.length > 4000 ? '\n…(truncated)' : ''}`;
     } catch (e) { return `Couldn't read ${rel}: ${e.message}`; }
   }
+  // EDIT beats WRITE for changing part of a file: WRITE overwrites the whole
+  // thing, so a model that wants a one-line change has to reproduce the entire
+  // file from memory and silently drops whatever it forgot.
+  const edit = /^EDIT:\s*([^|]+)\|([\s\S]*?)\|\|\|([\s\S]*)$/.exec(reply);
+  if (edit) {
+    const rel = edit[1].trim();
+    const oldStr = edit[2].replace(/^\n/, '').replace(/\n$/, '');
+    const newStr = edit[3].replace(/^\n/, '').replace(/\n$/, '');
+    try {
+      const full = safeResolveIn(dirFor(originId), rel);
+      const before = readFileSync(full, 'utf8');
+      const hits = before.split(oldStr).length - 1;
+      if (hits === 0) return `EDIT ${rel}: that exact text isn't in the file — READ it first, the copy must match byte for byte.`;
+      if (hits > 1) return `EDIT ${rel}: that text appears ${hits} times — include more surrounding context so it matches exactly once.`;
+      writeFileSync(full, before.replace(oldStr, newStr));
+      return `Edited ${rel} (${before.length} -> ${before.replace(oldStr, newStr).length} bytes).`;
+    } catch (e) { return `Couldn't edit ${rel}: ${e.message}`; }
+  }
+
+  const ls = /^LS:\s*(.*)$/.exec(reply);
+  if (ls) {
+    const rel = ls[1].trim() || '.';
+    try {
+      const full = safeResolveIn(dirFor(originId), rel);
+      const entries = readdirSync(full, { withFileTypes: true });
+      if (!entries.length) return `${rel}: (empty)`;
+      const lines = entries.slice(0, 300).map((e) => {
+        if (e.isDirectory()) return `  ${e.name}/`;
+        let size = '';
+        try { size = ` (${statSync(path.join(full, e.name)).size}b)`; } catch { /* raced */ }
+        return `  ${e.name}${size}`;
+      });
+      return `${rel}:\n${lines.join('\n')}${entries.length > 300 ? `\n  …${entries.length - 300} more` : ''}`;
+    } catch (e) { return `Couldn't list ${rel}: ${e.message}`; }
+  }
+
+  const glob = /^GLOB:\s*(.+)$/.exec(reply);
+  if (glob) {
+    const pattern = glob[1].trim();
+    try {
+      const base = dirFor(originId);
+      const re = globToRe(pattern.startsWith('./') ? pattern.slice(2) : pattern);
+      const hits = walkDir(base).filter((f) => re.test(f) || re.test(path.basename(f)));
+      if (!hits.length) return `GLOB ${pattern}: no matches`;
+      return `GLOB ${pattern} — ${hits.length} match(es):\n${hits.slice(0, 200).map((h) => '  ' + h).join('\n')}`
+        + (hits.length > 200 ? `\n  …${hits.length - 200} more` : '');
+    } catch (e) { return `GLOB ${pattern}: ${e.message}`; }
+  }
+
+  const grep = /^GREP:\s*([^|]+?)(?:\s*\|\s*(.+))?$/.exec(reply);
+  if (grep) {
+    const pattern = grep[1].trim();
+    const scope = (grep[2] || '').trim();
+    try {
+      const base = dirFor(originId);
+      let re;
+      try { re = new RegExp(pattern, 'i'); }
+      catch { return `GREP: ${pattern} isn't a valid regex.`; }
+      let files = walkDir(base);
+      if (scope) { const sre = globToRe(scope); files = files.filter((f) => sre.test(f) || f.startsWith(scope)); }
+      const out = [];
+      for (const f of files) {
+        if (out.length > 200) break;
+        let text;
+        try { text = readFileSync(path.join(base, f), 'utf8'); } catch { continue; }
+        if (text.indexOf(String.fromCharCode(0)) !== -1) continue; // binary — NUL, not whitespace
+        text.split('\n').forEach((line, i) => {
+          if (out.length <= 200 && re.test(line)) out.push(`  ${f}:${i + 1}: ${line.trim().slice(0, 200)}`);
+        });
+      }
+      if (!out.length) return `GREP ${pattern}: no matches`;
+      return `GREP ${pattern} — ${out.length} hit(s):\n${out.join('\n')}`;
+    } catch (e) { return `GREP: ${e.message}`; }
+  }
+
+  // A real, persisted checklist. Bots were already narrating plans; this makes
+  // the plan a thing the user can see and the model can be held to.
+  const todo = /^TODO:\s*([\s\S]*)$/.exec(reply);
+  if (todo) {
+    const t = threads.get(originId);
+    if (!t) return 'TODO: no such thread.';
+    t.todos = t.todos || [];
+    const body = todo[1].trim();
+    if (!body || /^(list|show)$/i.test(body)) {
+      if (!t.todos.length) return 'TODO: (empty)';
+      return 'TODO:\n' + t.todos.map((x, i) => `  ${i + 1}. [${x.done ? 'x' : ' '}] ${x.text}`).join('\n');
+    }
+    const done = /^done\s+(\d+)/i.exec(body);
+    if (done) {
+      const idx = Number(done[1]) - 1;
+      if (!t.todos[idx]) return `TODO: no item ${done[1]}.`;
+      t.todos[idx].done = true;
+      saveThreads();
+      return 'TODO:\n' + t.todos.map((x, i) => `  ${i + 1}. [${x.done ? 'x' : ' '}] ${x.text}`).join('\n');
+    }
+    if (/^clear$/i.test(body)) { t.todos = []; saveThreads(); return 'TODO: cleared.'; }
+    // Otherwise: replace the list with the lines given.
+    t.todos = body.split('\n').map((l) => l.replace(/^[-*\d.)\]\s]+/, '').trim())
+      .filter(Boolean).map((text) => ({ text, done: false }));
+    saveThreads();
+    return 'TODO:\n' + t.todos.map((x, i) => `  ${i + 1}. [ ] ${x.text}`).join('\n');
+  }
+
   const serve = /^SERVE:\s*(.*)$/.exec(reply);
   if (serve) {
     const rel = serve[1].trim();
@@ -536,12 +719,127 @@ async function tryDirective(reply, originId) {
           .replace(/<[^>]+>/g, ' ').replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&')
           .replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/\s+/g, ' ').trim();
       }
+      // A 405 on an MCP endpoint is not a broken URL, and a model cannot tell
+      // the difference — it retries the same GET, then gives up and hand-rolls
+      // a client. Say what actually happened and point at the directive that
+      // works. MEASURED: three wasted turns curling /api/mcp before the user
+      // had to intervene with "stop hitting /api/mcp as a curl".
+      if (r.status === 405 && /\/mcp\/?$/.test(url)) {
+        return `${url} (405): that is an MCP endpoint, not a web page — it speaks `
+          + `JSON-RPC over POST and rejects the GET that FETCH does. Use the MCP `
+          + `directive instead:\n  MCP: ${url}\nto list its tools, then\n  `
+          + `MCP: ${url} | <tool> | {"arg": "value"}\nto call one.`;
+      }
       return `${url} (${r.status}):\n${text.slice(0, 8000)}${text.length > 8000 ? '\n…(truncated)' : ''}`;
     } catch (e) { return `Couldn't fetch ${url}: ${e.message}`; }
   }
+
+  const mcpD = /^MCP:\s*(\S+)\s*(?:\|\s*([^|]+?)\s*(?:\|\s*([\s\S]+))?)?$/m.exec(reply);
+  if (mcpD) {
+    const [, url, tool, argsRaw] = mcpD;
+    let args = {};
+    if (argsRaw) {
+      const fenced = /```[\w-]*\n([\s\S]*?)```/.exec(argsRaw);
+      try { args = JSON.parse((fenced ? fenced[1] : argsRaw).trim()); }
+      catch (e) { return `MCP: couldn't parse the arguments as JSON — ${e.message}`; }
+    }
+    return await mcpDirective(url.trim(), tool?.trim(), args);
+  }
   return null;
 }
 
+// ---------------------------------------------------------------------------
+// MCP client — streamable-http, written against fetch on purpose.
+//
+// grokui runs STANDALONE from /opt/grokui/grokui.mjs, which has no
+// node_modules beside it, so importing @modelcontextprotocol/sdk would break
+// the copy that boxes actually execute. The wire protocol is small: POST
+// JSON-RPC, accept both JSON and SSE, carry the session id the server hands
+// back on initialize.
+//
+// This exists because bots were told to "install an MCP" and had no way to
+// speak to one. FETCH does a GET; every MCP endpoint answers GET with 405, so
+// the model saw a dead URL, retried, then wrote its own Python client.
+let mcpId = 0;
+
+async function mcpRpc(url, method, params, session, notify = false) {
+  const id = notify ? undefined : ++mcpId;
+  const r = await fetch(url, {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      // BOTH are required. Servers negotiate between a plain JSON reply and an
+      // SSE stream, and offering only one gets a 406 from spec-strict servers.
+      accept: 'application/json, text/event-stream',
+      'mcp-protocol-version': '2025-06-18',
+      ...(session ? { 'mcp-session-id': session } : {}),
+    },
+    body: JSON.stringify(notify ? { jsonrpc: '2.0', method, params } : { jsonrpc: '2.0', id, method, params }),
+  });
+  const sid = r.headers.get('mcp-session-id') || session;
+  const body = await r.text();
+  if (notify) return { status: r.status, session: sid, json: null };
+
+  let json = null;
+  if ((r.headers.get('content-type') || '').includes('text/event-stream')) {
+    // SSE frames: take the last `data:` payload that carries a result/error.
+    for (const line of body.split(/\r?\n/)) {
+      if (!line.startsWith('data:')) continue;
+      try {
+        const j = JSON.parse(line.slice(5).trim());
+        if (j && (j.result !== undefined || j.error !== undefined)) json = j;
+      } catch { /* keep-alive or partial frame */ }
+    }
+  } else {
+    try { json = JSON.parse(body); } catch { /* non-JSON error page */ }
+  }
+  return { status: r.status, session: sid, json, raw: body };
+}
+
+async function mcpDirective(url, tool, args) {
+  try {
+    const init = await mcpRpc(url, 'initialize', {
+      protocolVersion: '2025-06-18',
+      capabilities: {},
+      clientInfo: { name: 'openzoo-grokui', version: '1' },
+    });
+    if (init.json?.error) return `MCP ${url}: initialize failed — ${JSON.stringify(init.json.error)}`;
+    if (!init.json) return `MCP ${url}: no JSON-RPC reply (HTTP ${init.status})\n${(init.raw || '').slice(0, 600)}`;
+    const session = init.session;
+    // Required by spec before any other request; skipping it makes some
+    // servers reject everything after initialize.
+    await mcpRpc(url, 'notifications/initialized', {}, session, true).catch(() => {});
+
+    const server = init.json.result?.serverInfo;
+    const banner = `MCP ${url}${server ? ` — ${server.name} ${server.version || ''}`.trimEnd() : ''}`;
+
+    if (!tool) {
+      const list = await mcpRpc(url, 'tools/list', {}, session);
+      if (list.json?.error) return `${banner}\ntools/list failed — ${JSON.stringify(list.json.error)}`;
+      const tools = list.json?.result?.tools || [];
+      if (!tools.length) return `${banner}\n(no tools)`;
+      const lines = tools.map((t) => {
+        const req = t.inputSchema?.required || [];
+        const props = Object.keys(t.inputSchema?.properties || {});
+        const sig = props.map((p) => (req.includes(p) ? p : `${p}?`)).join(', ');
+        return `  ${t.name}(${sig})\n      ${(t.description || '').split('\n')[0].slice(0, 160)}`;
+      });
+      return `${banner}\n${tools.length} tools:\n${lines.join('\n')}\n\n`
+        + `Call one with:  MCP: ${url} | <tool> | {"arg": "value"}`;
+    }
+
+    const call = await mcpRpc(url, 'tools/call', { name: tool, arguments: args }, session);
+    if (call.json?.error) return `${banner}\n${tool} failed — ${JSON.stringify(call.json.error)}`;
+    const res = call.json?.result;
+    const out = (res?.content || [])
+      .map((c) => (c.type === 'text' ? c.text : `[${c.type}]`))
+      .join('\n') || JSON.stringify(res ?? call.raw);
+    return `${banner}\n$ ${tool}\n${out.slice(0, 6000)}${out.length > 6000 ? '\n…(truncated)' : ''}`;
+  } catch (e) {
+    return `MCP ${url}: ${e.message}`;
+  }
+}
+
 // onEvent (optional) gets live progress for whoever's actually watching this
 // call: {type:'start',name,color} when a bot begins its turn, {type:'delta',
 // name,color,delta} per streamed token, {type:'final',name,color,text} once

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openzoo",
-  "version": "0.48.4",
+  "version": "0.48.6",
   "description": "Local x402-paying proxy + MCP server for openzoo.fun — point any OpenAI-compatible harness (Cursor, Claude Code, aider, SDKs) at localhost and it pays per call from a local burner wallet. Solana and Base rails live; Robinhood experimental.",
   "license": "MIT",
   "type": "module",