@melaya/runner 1.0.35 → 1.0.37

package/dist/connection.js

@@ -15,9 +15,14 @@ import chalk from "chalk";
 import ora from "ora";
 import { spawn } from "child_process";
 import { writeFileSync, mkdirSync } from "fs";
-import { join } from "path";
-import { tmpdir } from "os";
+import { dirname, join } from "path";
+import { fileURLToPath } from "url";
+import { tmpdir, homedir } from "os";
 import { startLocalRelay, setActiveProject } from "./localRelay.js";
+// ESM equivalent of __dirname — required for the rag:ingest handler to
+// locate the bundled localRagIngest.py inside the npm package.
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
 import { ensureSharedModules, getSharedDir } from "./sharedVendor.js";
 import { startLumaBrowserBridge } from "./lumaBrowserBridge.js";
 const HEARTBEAT_INTERVAL = 30_000;
@@ -374,6 +379,144 @@ export async function connect(opts) {
             socket.emit("runner:runComplete", { runId: payload.runId, status: "failed" });
         }
     });
+    // ── RAG Mode B: local-folder ingest (privacy-preserving) ─────────────
+    // The user picks "Local folder" in the Agent Builder DocsTab and types
+    // a path on their machine (e.g. C:\Users\me\Documents\refs). The server
+    // routes the ingest to this handler. The runner reads the files locally,
+    // embeds via local Ollama, and persists the Qdrant store at
+    // ~/.melaya-runner/rag/<pipelineName>/store/. The text NEVER leaves the
+    // user's box — that's the entire point of Mode B.
+    //
+    // Hard refusal: if embedderProvider is "openai", we refuse with a clear
+    // error message. Cloud embedders would defeat the data-residency
+    // guarantee. Mode B = local-only embedder, period.
+    socket.on("rag:ingest", async (payload) => {
+        const sid = payload?.sessionId;
+        if (!sid)
+            return;
+        const emitProgress = (kind, fields = {}) => {
+            socket.emit("rag:ingest-progress", { session_id: sid, kind, ...fields });
+        };
+        try {
+            const prov = (payload.embedderProvider || "").toLowerCase();
+            if (prov === "openai") {
+                const reason = ("Mode B (local folder) refuses cloud embedders. " +
+                    "Pick ollama or lmstudio to keep your documents private.");
+                console.log(chalk.red(`  ✗ rag:ingest refused: ${reason}`));
+                socket.emit("rag:ingest-result", {
+                    session_id: sid, ok: false, error: reason,
+                });
+                return;
+            }
+            console.log(chalk.hex("#10B981")(`\n  🔒 RAG local-folder ingest: ${payload.pipelineName} ` +
+                `← ${payload.localFolderPath} via ${prov}:${payload.embedderModel}\n`));
+            emitProgress("started");
+            // Reuse the same venv as runner:run — agentscope.rag + ollama +
+            // qdrant-client land here.
+            const { ensurePythonEnv } = await import("./pythonEnv.js");
+            const env = await ensurePythonEnv(opts.pythonPath, payload.sharedVersion, (m) => emitProgress("venv", { message: m }));
+            if (!env.ok) {
+                emitProgress("error", { error: `venv bootstrap failed: ${env.reason}` });
+                socket.emit("rag:ingest-result", {
+                    session_id: sid, ok: false, error: `venv bootstrap failed: ${env.reason}`,
+                });
+                return;
+            }
+            // Stage the ingest script into the venv work dir so the spawned
+            // python sees the shared.runtime + the agentscope vendored tree.
+            await ensureSharedModules(opts.serverUrl, payload.sharedVersion);
+            const sharedDir = getSharedDir();
+            const ingestScript = join(sharedDir, "..", "localRagIngest.py");
+            // The script is published alongside the runner's shared bundle in
+            // the npm package itself — we copy it to a tmp dir per-invocation
+            // so re-runs are clean.
+            const workDir = join(tmpdir(), `melaya-rag-ingest-${payload.pipelineName}-${Date.now()}`);
+            mkdirSync(workDir, { recursive: true });
+            const { copyFileSync, existsSync } = await import("fs");
+            // Fallback path: when running from npm package layout
+            const candidates = [
+                ingestScript,
+                join(__dirname, "localRagIngest.py"),
+                join(__dirname, "..", "localRagIngest.py"),
+            ];
+            let foundIngest = "";
+            for (const c of candidates) {
+                if (existsSync(c)) {
+                    foundIngest = c;
+                    break;
+                }
+            }
+            if (!foundIngest) {
+                const msg = "localRagIngest.py not found in runner package — reinstall @melaya/runner";
+                emitProgress("error", { error: msg });
+                socket.emit("rag:ingest-result", {
+                    session_id: sid, ok: false, error: msg,
+                });
+                return;
+            }
+            const stagedScript = join(workDir, "localRagIngest.py");
+            copyFileSync(foundIngest, stagedScript);
+            // Resolve the on-disk path where the store will live. The Python
+            // script can compute this itself from pipelineName, but we surface
+            // it here for the FE event so the user can see where their data
+            // lives.
+            const storePath = join(homedir(), ".melaya-runner", "rag", payload.pipelineName, "store");
+            console.log(chalk.gray(`  [rag] store → ${storePath}`));
+            const proc = spawn(env.pythonPath, [
+                "-u", stagedScript,
+                "--pipeline-name", payload.pipelineName,
+                "--folder", payload.localFolderPath,
+                "--embedder-provider", prov,
+                "--embedder-model", payload.embedderModel,
+            ], {
+                env: {
+                    ...process.env,
+                    PYTHONPATH: sharedDir, // so `from agentscope.rag import …` resolves
+                },
+                cwd: workDir,
+            });
+            let stdoutBuf = "";
+            let stderrBuf = "";
+            proc.stdout.on("data", (data) => {
+                stdoutBuf += data.toString("utf-8");
+                // Stream each "progress: ..." JSON line up to the FE
+                const lines = stdoutBuf.split("\n");
+                stdoutBuf = lines.pop() || "";
+                for (const line of lines) {
+                    if (line.startsWith("progress: ")) {
+                        try {
+                            const ev = JSON.parse(line.slice("progress: ".length));
+                            emitProgress("step", ev);
+                        }
+                        catch { /* ignore malformed */ }
+                    }
+                    else if (opts.verbose && line.trim()) {
+                        console.log(chalk.gray(`  [ingest] ${line}`));
+                    }
+                }
+            });
+            proc.stderr.on("data", (data) => {
+                stderrBuf += data.toString("utf-8");
+            });
+            proc.on("close", (code) => {
+                const ok = code === 0;
+                console.log(ok
+                    ? chalk.green(`  ✓ RAG ingest complete (exit ${code})`)
+                    : chalk.red(`  ✗ RAG ingest failed (exit ${code})\n${stderrBuf.slice(-1000)}`));
+                socket.emit("rag:ingest-result", {
+                    session_id: sid, ok, exit: code,
+                    store_path: storePath,
+                    stderr_tail: ok ? "" : stderrBuf.slice(-2000),
+                });
+            });
+        }
+        catch (e) {
+            console.log(chalk.red(`  ✗ rag:ingest crashed: ${e?.message || e}`));
+            socket.emit("rag:ingest-result", {
+                session_id: sid, ok: false, error: e?.message || String(e),
+            });
+        }
+    });
     // ── LinkedIn cookie capture (one-time) ────────────────────────────
     // When the user clicks "Connect LinkedIn" in the platform UI, the
     // server emits this event. We launch a controlled Chromium window on

package/dist/modelLoader.js

@@ -382,42 +382,38 @@ export async function preflightModel(configJson, onProgress = (m) => console.log
     catch {
         return { ok: true }; /* let main.py error naturally */
     }
-    // Pipeline-level + per-step / per-agent providers — match the same
-    // walk as builder_server.py:2716-2725 so we honour the agent that
-    // overrides the top-level pick.
-    const providers = new Set();
-    const models = new Set();
-    const top = String(cfg.model_provider ?? cfg.model?.provider ?? "").toLowerCase();
-    if (top)
-        providers.add(top);
-    const topModel = String(cfg.model_name ?? cfg.model?.name ?? "");
-    if (topModel)
-        models.add(topModel);
+    // Pipeline-level + per-step / per-agent (provider, model) PAIRS — must
+    // stay paired so a melaya_ai / openai / anthropic agent isn't checked
+    // against LM Studio just because a different agent in the same pipeline
+    // happens to use LM Studio. Mixed-provider pipelines were preflighting
+    // every cloud model against every local runner → spurious 'not found
+    // in LM Studio' failures.
+    const pairs = new Set(); // serialized "provider|model" for dedup
+    const addPair = (p, m) => {
+        if (p && m)
+            pairs.add(`${p.toLowerCase()}|${m}`);
+    };
+    addPair(String(cfg.model_provider ?? cfg.model?.provider ?? ""), String(cfg.model_name ?? cfg.model?.name ?? ""));
     for (const step of (cfg.steps ?? [])) {
         const a = step?.agent ?? {};
-        const p = String(a?.model?.provider ?? "").toLowerCase();
-        const n = String(a?.model?.name ?? "");
-        if (p)
-            providers.add(p);
-        if (n)
-            models.add(n);
+        addPair(String(a?.model?.provider ?? a?.model_provider ?? ""), String(a?.model?.name ?? a?.model_name ?? ""));
     }
     for (const a of (cfg.agents ?? [])) {
-        const p = String(a?.model_provider ?? a?.model?.provider ?? "").toLowerCase();
-        const n = String(a?.model_name ?? a?.model?.name ?? "");
-        if (p)
-            providers.add(p);
-        if (n)
-            models.add(n);
+        // Nested wins over flat — the ModelPicker writes `agent.model`
+        // but doesn't always purge the legacy `agent.model_provider` /
+        // `agent.model_name` fields, so they drift stale across edits.
+        addPair(String(a?.model?.provider ?? a?.model_provider ?? ""), String(a?.model?.name ?? a?.model_name ?? ""));
     }
-    // Run preflight for every distinct (provider, model) combo this
-    // pipeline will exercise. One bad agent = whole run fails fast.
+    // Run preflight only for LOCAL providers — cloud (openai / anthropic /
+    // claude_code / codex / melaya_ai / zhipu) doesn't need the runner.
     const tasks = [];
-    for (const m of models) {
-        if (providers.has("lmstudio"))
-            tasks.push(preflightLmStudio(m, onProgress));
-        if (providers.has("ollama"))
-            tasks.push(preflightOllama(m, onProgress));
+    for (const pair of pairs) {
+        const [provider, model] = pair.split("|");
+        if (provider === "lmstudio")
+            tasks.push(preflightLmStudio(model, onProgress));
+        else if (provider === "ollama")
+            tasks.push(preflightOllama(model, onProgress));
+        // every other provider is cloud-only — no preflight needed.
     }
     if (tasks.length === 0)
         return { ok: true };

package/dist/pythonEnv.js

@@ -30,6 +30,17 @@ const AGENTSCOPE = join(CACHE_DIR, "agentscope");
 // PLUS the runner-tool extras (aiohttp, requests, python-dotenv).
 // If you bump the agentscope pyproject deps, bump this list too —
 // or extract from pyproject at server-side and ship via the bundle.
+// ── TODO(arch): move this list into the server's shared-bundle response ──
+// Right now any new Python dep required by a shared tool means:
+//   1. add the dep to this hardcoded list
+//   2. bump the @melaya/runner npm version
+//   3. publish + each user reinstalls
+// The cleaner pattern is to ship a `requirements.txt` (or a JSON list) IN
+// the shared-bundle endpoint, alongside the .py files. The runner then
+// reads it at venv-bootstrap time. Result: adding a tool's Python dep
+// only needs a server deploy — runners pick it up next reconnect, same
+// way they pick up shared/*.py changes today.
+// Until that lands, every change to this list = a runner npm version bump.
 const PIP_DEPS = [
     // ── agentscope core (mirrors pyproject.toml `dependencies`) ──
     "aioitertools",
@@ -81,6 +92,22 @@ const PIP_DEPS = [
     // `scrapling install` to download Chromium — best-effort run below
     // (failure non-fatal: AsyncFetcher fast-path still works without it).
     "scrapling[fetchers]",
+    // Telethon — Telegram MTProto user-mode client. Used by
+    // shared.tools.telegram_user_tools. The bot-API path in
+    // shared.tools.messaging.py uses plain HTTP and needs no extra dep.
+    "telethon",
+    // ── Vector RAG (retrieval mode) ────────────────────────────────────────
+    // ollama — Python client used by agentscope.embedding.OllamaTextEmbedding.
+    //   Talks to localhost:11434 (the user's local Ollama daemon) to embed
+    //   chunks without exfiltrating the underlying text to a cloud API. This
+    //   is the only embedder allowed in privacy-preserving Mode B.
+    //
+    // (qdrant-client is ALREADY in this list above — it provides the
+    // embedded Qdrant store via agentscope.rag._store.QdrantStore at the
+    // ~/.melaya-runner/rag/<pipeline_name>/store/ path. Path-based Qdrant
+    // ships Windows wheels, MilvusLite does not — using Qdrant for OS
+    // parity across runners.)
+    "ollama",
 ];
 export function venvPython() {
     return platform() === "win32"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.0.35",
+  "version": "1.0.37",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,