openzoo 0.11.1 → 0.11.2

package/lib/bindpath.js

@@ -38,6 +38,35 @@ const DEFAULT_EXTS = [
 
 const SKIP_DIRS = new Set(['node_modules', '.git', 'target', 'dist', 'build', '.next', '__pycache__', '.venv']);
 
+/** Refuse rather than grind. A directory of exports can be hundreds of MB;
+ *  binding that takes many minutes and mostly buys markup. Raise deliberately. */
+export const MAX_TOTAL_BYTES = Number(process.env.OPENZOO_BIND_MAX_BYTES || 32 * 1024 * 1024);
+
+/**
+ * HTML → readable text. Chat/forum exports are ~90% markup: binding the raw
+ * pages wastes the corpus on tags and makes retrieval rank div soup against
+ * the question. Deliberately crude (no parser dependency) but it keeps the
+ * text nodes in document order, which is all retrieval needs.
+ */
+export function stripHtml(html) {
+  return html
+    .replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ')
+    .replace(/<!--[\s\S]*?-->/g, ' ')
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<\/(p|div|li|tr|h[1-6])>/gi, '\n')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#39;/g, "'")
+    .replace(/[ \t]+/g, ' ')
+    .replace(/\n\s*\n\s*\n+/g, '\n\n')
+    .trim();
+}
+
+const readAsText = (file) => {
+  const raw = fs.readFileSync(file, 'utf8');
+  return /\.html?$/i.test(file) ? stripHtml(raw) : raw;
+};
+
 /** Every bindable file under `root` (or just `root` if it is a file). */
 export function collectFiles(root, { exts = DEFAULT_EXTS, maxFiles = 5000 } = {}) {
   const st = fs.statSync(root);
@@ -109,11 +138,27 @@ export async function bindPath(target, { exts, onProgress, force = false } = {})
   const files = collectFiles(resolved, exts ? { exts } : {});
   if (!files.length) throw new Error(`nothing bindable under ${resolved} (looked for text-like files)`);
 
+  // Size the job BEFORE reading anything. A 2GB export folder is 100MB of
+  // text-like files, and silently grinding through it for minutes reads as a
+  // hang — an actionable refusal is better than a spinner.
+  const onDisk = files.reduce((n, f) => n + fs.statSync(f).size, 0);
+  if (onDisk > MAX_TOTAL_BYTES) {
+    const biggest = files
+      .map((f) => ({ f, size: fs.statSync(f).size }))
+      .sort((a, b) => b.size - a.size).slice(0, 3)
+      .map(({ f, size }) => `${path.basename(f)} (${(size / 1048576).toFixed(1)}MB)`);
+    throw new Error(
+      `${(onDisk / 1048576).toFixed(0)}MB across ${files.length} files exceeds the ${(MAX_TOTAL_BYTES / 1048576).toFixed(0)}MB bind limit. `
+      + `Largest: ${biggest.join(', ')}. Point at one file, narrow with --ext/ext, `
+      + `or raise OPENZOO_BIND_MAX_BYTES if you really want all of it.`,
+    );
+  }
+
   // Each file is prefixed with its path so retrieval can cite where a passage
   // came from — a corpus of concatenated files with no provenance is much
-  // less useful to answer from.
+  // less useful to answer from. HTML is reduced to its text (see readAsText).
   const text = files
-    .map((f) => `===== ${path.relative(path.dirname(resolved), f) || path.basename(f)} =====\n${fs.readFileSync(f, 'utf8')}`)
+    .map((f) => `===== ${path.relative(path.dirname(resolved), f) || path.basename(f)} =====\n${readAsText(f)}`)
     .join('\n\n');
 
   const bytes = Buffer.byteLength(text);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openzoo",
-  "version": "0.11.1",
+  "version": "0.11.2",
   "description": "Local x402-paying proxy + MCP server for openzoo.fun — point any OpenAI-compatible harness (Cursor, Claude Code, aider, SDKs) at localhost and it pays per call from a local burner wallet. Solana and Base rails live; Robinhood experimental.",
   "license": "MIT",
   "type": "module",