openzoo 0.10.0 → 0.11.0

package/bin/openzoo.js

@@ -77,6 +77,55 @@ async function main() {
       console.log('\nthese corpora never ship again — asks against them send only the question + X-HRR-Context.');
       break;
     }
+    case 'bind': {
+      const target = process.argv[3];
+      if (!target) throw new Error('usage: openzoo bind <file-or-directory> [--ext .txt,.md] [--force]');
+      const { bindPath } = await import('../lib/bindpath.js');
+      const ei = process.argv.indexOf('--ext');
+      const exts = ei !== -1 && process.argv[ei + 1]
+        ? process.argv[ei + 1].split(',').map((e) => (e.startsWith('.') ? e : `.${e}`))
+        : undefined;
+      const mb = (n) => (n / 1048576).toFixed(1);
+      const out = await bindPath(target, {
+        exts,
+        force: process.argv.includes('--force'),
+        onProgress: (p) => {
+          if (p.stage === 'reused') console.log(`already bound — ${mb(p.bytes)}MB across ${p.files} file(s) reused, nothing uploaded`);
+          if (p.stage === 'start') console.log(`binding ${mb(p.bytes)}MB from ${p.files} file(s) in ${p.parts} part(s)...`);
+          if (p.stage === 'part') console.log(`  part ${p.index}/${p.of} bound (${mb(p.bytes)}MB)`);
+        },
+      });
+      console.log('');
+      console.log(`context: ${out.contextId}`);
+      console.log(`ask it:  npx openzoo ask "your question" --context ${out.contextId}`);
+      console.log('or send X-HRR-Context: <id> with a small body to /v1/chat/completions');
+      break;
+    }
+    case 'ask': {
+      const question = process.argv[3];
+      if (!question) throw new Error('usage: openzoo ask "<question>" [--context <id>] [--model <id>]');
+      const ci = process.argv.indexOf('--context');
+      const mi = process.argv.indexOf('--model');
+      const { PayClient } = await import('../lib/pay.js');
+      const { config } = await import('../lib/config.js');
+      const client = new PayClient();
+      const headers = { 'content-type': 'application/json' };
+      if (ci !== -1 && process.argv[ci + 1]) headers['x-hrr-context'] = process.argv[ci + 1];
+      const { response, receipt } = await client.fetch(`${config.apiBase}/v1/chat/completions`, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify({
+          model: (mi !== -1 && process.argv[mi + 1]) || process.env.OPENZOO_DEFAULT_MODEL || 'deepseek/deepseek-v4-pro-0813',
+          messages: [{ role: 'user', content: question }],
+          max_tokens: Number(process.env.OPENZOO_ASK_MAX_TOKENS || 1024),
+        }),
+      });
+      if (!response.ok) throw new Error(`zoo returned HTTP ${response.status}: ${(await response.text()).slice(0, 300)}`);
+      const data = await response.json();
+      console.log(data.choices?.[0]?.message?.content ?? '(no content)');
+      if (receipt) console.error(`\n${receipt.line}`);
+      break;
+    }
     case 'balance':
       await (await import('../lib/info.js')).printBalance();
       break;

package/lib/bindpath.js

@@ -0,0 +1,143 @@
+/**
+ * Chunk-and-bind a FILE or DIRECTORY from local disk.
+ *
+ * WHY THIS EXISTS: harnesses cap what they can inline. Cursor truncates every
+ * file read at 100,000 characters, so an agent asked to "consume this 8.7MB
+ * export" physically cannot put it in a request body no matter how large the
+ * zoo's context is — it never gets the bytes in the first place. The CLI and
+ * the local MCP server both run ON the machine that holds the file, so they
+ * can read it directly and hand the zoo the whole thing.
+ *
+ * A single request also cannot carry an arbitrary corpus: the network hop in
+ * front of the gateway drops bodies past ~8MB (an opaque 413 or a dead
+ * connection). So the corpus is split into parts under that ceiling and bound
+ * SEQUENTIALLY — part 1 creates the context, every later part is appended to
+ * the same context_id. The result is one context holding everything.
+ *
+ * Splitting happens on PARAGRAPH boundaries where possible. leCore's own
+ * chunker documents why ("a fact split across two chunks is retrievable from
+ * neither"), and the same reasoning applies one level up: cutting mid-sentence
+ * at an arbitrary byte offset can strand a fact across two bind calls.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { config } from './config.js';
+import { corpusHash, rememberContext, lookupContext } from './contexts.js';
+
+/** Bodies over this go in their own part. Under the ~8MB transport ceiling
+ *  with room for JSON escaping, which can inflate text substantially. */
+export const MAX_PART_BYTES = Number(process.env.OPENZOO_BIND_PART_BYTES || 4_000_000);
+
+/** Text-ish files worth binding. Anything else is skipped rather than
+ *  silently binding a binary as mojibake. */
+const DEFAULT_EXTS = [
+  '.txt', '.md', '.json', '.jsonl', '.csv', '.tsv', '.log', '.html', '.htm', '.xml',
+  '.py', '.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.rs', '.go', '.java', '.c',
+  '.h', '.cpp', '.hpp', '.rb', '.php', '.sh', '.sql', '.yaml', '.yml', '.toml', '.ini',
+];
+
+const SKIP_DIRS = new Set(['node_modules', '.git', 'target', 'dist', 'build', '.next', '__pycache__', '.venv']);
+
+/** Every bindable file under `root` (or just `root` if it is a file). */
+export function collectFiles(root, { exts = DEFAULT_EXTS, maxFiles = 5000 } = {}) {
+  const st = fs.statSync(root);
+  if (st.isFile()) return [root];
+  const out = [];
+  const walk = (dir) => {
+    if (out.length >= maxFiles) return;
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      if (out.length >= maxFiles) return;
+      if (entry.name.startsWith('.') && entry.name !== '.env.example') continue;
+      const p = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (!SKIP_DIRS.has(entry.name)) walk(p);
+      } else if (exts.includes(path.extname(entry.name).toLowerCase())) {
+        out.push(p);
+      }
+    }
+  };
+  walk(root);
+  return out.sort();
+}
+
+/**
+ * Split text into parts at or under `maxBytes`, preferring paragraph breaks
+ * and falling back to line breaks, then to a hard cut. Byte-aware, because a
+ * character count lies for non-ASCII — and chat exports are full of it.
+ */
+export function splitIntoParts(text, maxBytes = MAX_PART_BYTES) {
+  const parts = [];
+  let rest = text;
+  while (Buffer.byteLength(rest) > maxBytes) {
+    // Walk back from the byte ceiling to a boundary. Slice generously by
+    // chars first (bytes >= chars), then trim to fit.
+    let cut = rest.length;
+    while (Buffer.byteLength(rest.slice(0, cut)) > maxBytes) cut = Math.floor(cut * 0.9);
+    const window = rest.slice(0, cut);
+    const at = Math.max(window.lastIndexOf('\n\n'), window.lastIndexOf('\n'));
+    const end = at > cut * 0.5 ? at : cut; // only honour a boundary that is not absurdly early
+    parts.push(rest.slice(0, end));
+    rest = rest.slice(end).replace(/^\n+/, '');
+  }
+  if (rest.trim()) parts.push(rest);
+  return parts;
+}
+
+async function postBind(payload) {
+  const r = await fetch(`${config.apiBase}/v1/hrr/bind`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify(payload),
+  });
+  if (r.status !== 200) throw new Error(`bind failed: HTTP ${r.status}: ${(await r.text()).slice(0, 300)}`);
+  const j = await r.json();
+  if (!j?.context_id) throw new Error('bind returned no context_id');
+  return j;
+}
+
+/**
+ * Read `target` (file or directory), split it, and bind every part into ONE
+ * context. Returns { contextId, files, parts, bytes, reused }.
+ *
+ * Binding is free, but it is not instant on a large corpus, so `onProgress`
+ * reports each part as it lands rather than going quiet for minutes.
+ */
+export async function bindPath(target, { exts, onProgress, force = false } = {}) {
+  const resolved = path.resolve(target);
+  if (!fs.existsSync(resolved)) throw new Error(`no such file or directory: ${resolved}`);
+
+  const files = collectFiles(resolved, exts ? { exts } : {});
+  if (!files.length) throw new Error(`nothing bindable under ${resolved} (looked for text-like files)`);
+
+  // Each file is prefixed with its path so retrieval can cite where a passage
+  // came from — a corpus of concatenated files with no provenance is much
+  // less useful to answer from.
+  const text = files
+    .map((f) => `===== ${path.relative(path.dirname(resolved), f) || path.basename(f)} =====\n${fs.readFileSync(f, 'utf8')}`)
+    .join('\n\n');
+
+  const bytes = Buffer.byteLength(text);
+  const hash = corpusHash(text);
+  if (!force) {
+    const hit = lookupContext(config.apiBase, hash);
+    if (hit) {
+      onProgress?.({ stage: 'reused', contextId: hit.context_id, bytes, files: files.length });
+      return { contextId: hit.context_id, files, parts: 0, bytes, reused: true };
+    }
+  }
+
+  const parts = splitIntoParts(text);
+  onProgress?.({ stage: 'start', files: files.length, parts: parts.length, bytes });
+
+  let contextId = null;
+  for (let i = 0; i < parts.length; i++) {
+    // Part 1 creates the context; every later part APPENDS by passing the id
+    // back. This is what lets a corpus exceed the per-request ceiling.
+    const j = await postBind(contextId ? { corpus: parts[i], context_id: contextId } : { corpus: parts[i] });
+    contextId = j.context_id;
+    onProgress?.({ stage: 'part', index: i + 1, of: parts.length, bytes: Buffer.byteLength(parts[i]), contextId });
+  }
+
+  rememberContext(config.apiBase, hash, contextId);
+  return { contextId, files, parts: parts.length, bytes, reused: false };
+}

package/lib/brief.js

@@ -11,9 +11,20 @@
  * OPENZOO_NO_BRIEF=1 turns it off.
  */
 
-export const BRIEF = [
+/**
+ * `selfUrl` is the base URL THIS request arrived on — the tunnel URL for a
+ * remote harness, localhost otherwise. Without it an agent writing a helper
+ * script has to guess its own endpoint, and it guesses wrong: observed in the
+ * wild, a Cursor agent hardcoded `https://openzoo.fun/v1` (a marketing site,
+ * not an API) because nothing in the conversation named the real one.
+ */
+export const briefFor = (selfUrl) => [
   'You are connected through an openzoo proxy (openzoo.fun), not a stock OpenAI endpoint. What that changes:',
   '',
+  ...(selfUrl ? [
+    `0. YOUR ENDPOINT IS ${selfUrl} — use exactly this base URL in any script or curl you write, never guess one and never use openzoo.fun (that is a website, not an API). Endpoints below are relative to it: ${selfUrl}/chat/completions, ${selfUrl}/hrr/bind, ${selfUrl}/models. Do NOT insert another /v1 — this URL already ends in one.`,
+    '',
+  ] : []),
   '1. CONTEXT IS EFFECTIVELY UNBOUNDED. Any model here accepts corpora far past its own attention window: bodies over ~16KB are automatically carved and bound to a holographic (HRR) memory before the model sees them, and you then query against that. `context_length` in /v1/models is the client-usable ceiling (128M tokens), not the transformer window (that is `max_model_len`). DO NOT summarise, truncate, or "chunk to fit" a corpus to preserve context — send it whole and ask your question. One POST should stay under ~9.8M tokens (~32MiB) or the edge rejects it; for more than that, bind in several calls.',
   '',
   '2. THE BODY NEVER SHIPS TWICE. Put the corpus first, then a blank line, then your question. The corpus binds ONCE and every later question that reuses it ships only the question — near-free, and much faster. Re-pasting the same corpus each turn wastes real money.',
@@ -27,6 +38,9 @@ export const BRIEF = [
   '4. MODEL IDS ARE FORGIVING. Ask for any model id you like; unknown ids are matched to the nearest served model, and /v1/models lists what is real (each alias row carries `served_by`).',
 ].join('\n');
 
+/** Back-compat: the briefing with no endpoint line. */
+export const BRIEF = briefFor(null);
+
 /**
  * Inject the briefing as a system message. Idempotent (never doubles up if a
  * conversation already carries it), non-destructive (an existing system
@@ -34,13 +48,13 @@ export const BRIEF = [
  * harnesses often pin behaviour in the first system turn).
  * Returns null when nothing should change.
  */
-export function injectBrief(body) {
+export function injectBrief(body, selfUrl = null) {
   if (process.env.OPENZOO_NO_BRIEF === '1') return null;
   const msgs = body?.messages;
   if (!Array.isArray(msgs) || !msgs.length) return null;
   if (msgs.some((m) => typeof m?.content === 'string' && m.content.includes('connected through an openzoo proxy'))) return null;
 
-  const brief = { role: 'system', content: BRIEF };
+  const brief = { role: 'system', content: briefFor(selfUrl) };
   const lastSystem = msgs.reduce((acc, m, i) => (m?.role === 'system' ? i : acc), -1);
   const out = [...msgs];
   out.splice(lastSystem + 1, 0, brief); // after any leading system block, before the user turns

package/lib/mcp.js

@@ -50,13 +50,27 @@ export async function startMcp() {
     inputSchema: {
       prompt: z.string().describe('The question or instruction.'),
       corpus: z.string().optional().describe('Optional big context body (document dump, logs, book...). Placed before the prompt.'),
+      context_id: z.string().optional().describe('A context from zoo_bind / zoo_contexts. The question ships alone against the already-bound corpus — cheapest and fastest path. Do not also pass `corpus`.'),
       model: z.string().optional().describe(`Model id from zoo_models. Default ${DEFAULT_MODEL}.`),
       max_tokens: z.number().int().positive().optional().describe('Completion cap. Default 1024.'),
     },
-  }, async ({ prompt, corpus, model, max_tokens }) => {
+  }, async ({ prompt, corpus, context_id, model, max_tokens }) => {
     try {
       const bodyBase = { model: model || DEFAULT_MODEL, max_tokens: max_tokens || 1024 };
       let data; let receipt; let reuse = null;
+      // An explicit context is the cheap path: nothing to hash, nothing to
+      // upload, just the question plus the header.
+      if (context_id) {
+        const { response, receipt: r } = await client.fetch(`${config.apiBase}/v1/chat/completions`, {
+          method: 'POST',
+          headers: { 'content-type': 'application/json', 'x-hrr-context': context_id },
+          body: JSON.stringify({ ...bodyBase, messages: [{ role: 'user', content: prompt }] }),
+        });
+        if (!response.ok) throw new Error(`zoo returned HTTP ${response.status}: ${(await response.text()).slice(0, 300)}`);
+        data = await response.json();
+        receipt = r;
+        reuse = { contextId: context_id, corpusReused: true };
+      } else
       // "The body never ships twice": a big corpus is bound ONCE on the zoo
       // (sha256 manifest at ~/.openzoo/contexts.json) and later asks ship only
       // the prompt + an X-HRR-Context header. Small corpora ride inline.
@@ -130,6 +144,38 @@ export async function startMcp() {
     });
   });
 
+  server.registerTool('zoo_bind', {
+    description:
+      'Bind a LOCAL file or directory into the zoo\'s holographic memory, then ask questions against it with zoo_ask({context_id}). '
+      + 'USE THIS INSTEAD OF READING A LARGE FILE YOURSELF: your own file-reading tool truncates (Cursor caps every read at 100,000 characters), '
+      + 'so a big export can never reach the model through it. This server runs on the same machine as the file — it reads the whole thing, '
+      + 'splits it into transport-sized parts, and binds them into ONE context. Handles directories (text-like files only, skips node_modules/.git). '
+      + 'Free — binding costs no payment. Re-binding the same content returns the existing context without re-uploading.',
+    inputSchema: {
+      path: z.string().describe('Absolute path to a file or directory on this machine.'),
+      ext: z.string().optional().describe('Comma-separated extensions to include when path is a directory, e.g. ".py,.md". Defaults to a broad text-like set.'),
+      force: z.boolean().optional().describe('Re-bind even if this exact content is already bound.'),
+    },
+  }, async ({ path: target, ext, force }) => {
+    try {
+      const { bindPath } = await import('./bindpath.js');
+      const out = await bindPath(target, {
+        exts: ext ? ext.split(',').map((e) => (e.trim().startsWith('.') ? e.trim() : `.${e.trim()}`)) : undefined,
+        force: force === true,
+      });
+      return text({
+        context_id: out.contextId,
+        bound_bytes: out.bytes,
+        files: out.files.length,
+        parts: out.parts,
+        reused: out.reused,
+        next: `Call zoo_ask with context_id="${out.contextId}" and your question. Do NOT paste the corpus into the prompt — it is already bound.`,
+      });
+    } catch (e) {
+      return text({ error: e.message });
+    }
+  });
+
   server.registerTool('zoo_contexts', {
     description: 'List corpora already bound to the zoo\'s holographic memory (local manifest). A listed corpus is never re-uploaded — asks against it are near-free.',
     inputSchema: {},

package/lib/proxy.js

@@ -15,6 +15,11 @@ import { injectBrief } from './brief.js';
 const HOP_BY_HOP = new Set([
   'host', 'connection', 'keep-alive', 'transfer-encoding', 'upgrade',
   'proxy-authorization', 'proxy-connection', 'te', 'trailer', 'content-length',
+  // `Expect: 100-continue` is sent by curl and most HTTP libraries once a body
+  // passes ~1KB. undici REFUSES it outright ("expect header not supported"),
+  // so forwarding it made every LARGE-body request fail while small ones
+  // worked — i.e. it broke exactly the corpus calls this proxy exists for.
+  'expect',
   // The harness's api key (sk-openzoo or anything) is accepted and dropped:
   // the zoo takes payment, not keys.
   'authorization',
@@ -352,7 +357,13 @@ export async function startProxy({ silent = false, requireToken = null, sessionM
         // will read it, instead of leaving it to guess (and to chunk corpora
         // it could bind whole). See lib/brief.js.
         if ((req.url || '').includes('/chat/completions')) {
-          const briefed = injectBrief(parsed);
+          // Tell it the URL it actually reached us on — the public tunnel for
+          // a remote harness, localhost for a local one. An agent that has to
+          // guess its own endpoint guesses a website.
+          const selfUrl = viaTunnel && tunnelGate?.publicUrl
+            ? `${tunnelGate.publicUrl}/v1`
+            : `http://localhost:${config.port}/v1`;
+          const briefed = injectBrief(parsed, selfUrl);
           if (briefed) bodyBuf = Buffer.from(JSON.stringify(briefed));
         }
       } catch { /* not JSON */ }
@@ -474,7 +485,14 @@ export async function startProxy({ silent = false, requireToken = null, sessionM
         log(err.message);
         jsonErr(res, 402, err.message);
       } else {
-        jsonErr(res, 502, `openzoo proxy error: ${err.message}`);
+        // "fetch failed" alone is undiagnosable — undici hides the real
+        // network error in `cause`. Surface it (and log the stack) or every
+        // transport hiccup looks identical to a payment bug.
+        const cause = err.cause?.message || err.cause?.code || err.cause;
+        const detail = cause ? `${err.message} (${cause})` : err.message;
+        log(`proxy error: ${detail}`);
+        if (process.env.OPENZOO_DEBUG) console.error(err.stack);
+        jsonErr(res, 502, `openzoo proxy error: ${detail}`);
       }
     }
   });
@@ -574,7 +592,7 @@ export async function startProxy({ silent = false, requireToken = null, sessionM
         const cap = process.env.OPENZOO_TUNNEL_MAX_USD ? Number(process.env.OPENZOO_TUNNEL_MAX_USD) : Infinity;
         const bin = await ensureCloudflared((m) => log(m));
         const { url, proc } = await startCloudflared(bin, config.port, log);
-        tunnelGate = { token, sessionMaxUsd: cap };
+        tunnelGate = { token, sessionMaxUsd: cap, publicUrl: url };
         const bye = () => { try { proc.kill('SIGTERM'); } catch { /* already gone */ } };
         process.once('SIGINT', () => { bye(); process.exit(0); });
         process.once('SIGTERM', () => { bye(); process.exit(0); });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openzoo",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "Local x402-paying proxy + MCP server for openzoo.fun — point any OpenAI-compatible harness (Cursor, Claude Code, aider, SDKs) at localhost and it pays per call from a local burner wallet. Solana and Base rails live; Robinhood experimental.",
   "license": "MIT",
   "type": "module",