@melaya/runner 1.0.52 → 1.0.54

package/dist/connection.js

@@ -14,7 +14,7 @@ import { io } from "socket.io-client";
 import chalk from "chalk";
 import ora from "ora";
 import { spawn } from "child_process";
-import { writeFileSync, mkdirSync } from "fs";
+import { writeFileSync, mkdirSync, existsSync } from "fs";
 import { dirname, join } from "path";
 import { fileURLToPath } from "url";
 import { tmpdir, homedir } from "os";
@@ -170,6 +170,66 @@ export async function connect(opts) {
             mkdirSync(runDir, { recursive: true });
             writeFileSync(join(runDir, "main.py"), payload.mainPyContent, "utf-8");
             writeFileSync(join(runDir, "config.json"), payload.configJson, "utf-8");
+            // Write rag/ static-context files alongside main.py so
+            // _load_rag_context() inside the spawned pipeline sees them.
+            // Two paths:
+            //   Cloud-upload mode: server ships payload.ragFiles (b64 bytes).
+            //   Local-folder Mode B: server sets staticContextLocalFolderPath
+            //     and omits ragFiles. We walk that folder here so the bytes
+            //     NEVER round-trip through prod.
+            const _SC_EXTS = new Set([".txt", ".md", ".csv", ".json", ".pdf", ".doc", ".docx", ".pptx", ".xlsx"]);
+            const _SC_MAX_BYTES = 4 * 1024 * 1024;
+            const ragDir = join(runDir, "rag");
+            let scFilesWritten = 0;
+            const scLocalPath = (payload.staticContextLocalFolderPath || "").trim();
+            if (scLocalPath) {
+                try {
+                    if (!existsSync(scLocalPath)) {
+                        console.log(chalk.yellow(`  ! static-context local folder not found: ${scLocalPath}`));
+                    }
+                    else {
+                        const { readdirSync, statSync, readFileSync } = await import("node:fs");
+                        const { extname, basename } = await import("node:path");
+                        mkdirSync(ragDir, { recursive: true });
+                        for (const name of readdirSync(scLocalPath)) {
+                            if (name.startsWith("."))
+                                continue;
+                            const full = join(scLocalPath, name);
+                            try {
+                                const st = statSync(full);
+                                if (!st.isFile())
+                                    continue;
+                                if (st.size > _SC_MAX_BYTES)
+                                    continue;
+                                if (!_SC_EXTS.has(extname(name).toLowerCase()))
+                                    continue;
+                                writeFileSync(join(ragDir, basename(name)), readFileSync(full));
+                                scFilesWritten++;
+                            }
+                            catch (err) {
+                                console.log(chalk.yellow(`  ! skip ${name}: ${err.message}`));
+                            }
+                        }
+                        console.log(chalk.gray(`  [rag] local-folder mode: copied ${scFilesWritten} file(s) from ${scLocalPath} to ${ragDir} (bytes never round-tripped through prod)`));
+                    }
+                }
+                catch (err) {
+                    console.log(chalk.yellow(`  ! static-context local-folder scan failed: ${err.message}`));
+                }
+            }
+            else if (payload.ragFiles && Object.keys(payload.ragFiles).length > 0) {
+                mkdirSync(ragDir, { recursive: true });
+                for (const [name, b64] of Object.entries(payload.ragFiles)) {
+                    try {
+                        writeFileSync(join(ragDir, name), Buffer.from(b64, "base64"));
+                        scFilesWritten++;
+                    }
+                    catch (err) {
+                        console.log(chalk.yellow(`  ! could not write rag/${name}: ${err.message}`));
+                    }
+                }
+                console.log(chalk.gray(`  [rag] cloud-upload mode: wrote ${scFilesWritten} static-context file(s) to ${ragDir}`));
+            }
             // Local-model preflight: ensure LM Studio / Ollama have the
             // pipeline's model loaded BEFORE we spawn the Python subprocess.
             // Without this, the subprocess hangs on its first chat completion

package/dist/pythonEnv.js

@@ -92,6 +92,19 @@ const PIP_DEPS = [
     // `scrapling install` to download Chromium — best-effort run below
     // (failure non-fatal: AsyncFetcher fast-path still works without it).
     "scrapling[fetchers]",
+    // beautifulsoup4 + lxml — HTML parser used by _search_ddg_html in
+    // shared.tools.core to scrape DDG's HTML search results page (the
+    // primary, non-rate-limited path). Without these, every web_search
+    // call raises ModuleNotFoundError, falls through to DDGS, and if
+    // DDGS is also down or rate-limited, the tool surfaces an error
+    // STRING that small local models (qwen3-vl-8b et al) read as
+    // data and hallucinate URLs around. Scrapling's [fetchers] extra
+    // does NOT pull bs4 (it brings curl_cffi+playwright), so pin
+    // explicitly. Tokano's run d38e09f747134084 hit this — Copywriter
+    // invented theinformation.com + nature.com URLs because every
+    // search returned 'Search error: ModuleNotFoundError bs4'.
+    "beautifulsoup4",
+    "lxml",
     // Telethon — Telegram MTProto user-mode client. Used by
     // shared.tools.telegram_user_tools. The bot-API path in
     // shared.tools.messaging.py uses plain HTTP and needs no extra dep.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.0.52",
+  "version": "1.0.54",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,