@melaya/runner 1.0.36 → 1.0.38

package/dist/connection.js

@@ -15,9 +15,14 @@ import chalk from "chalk";
 import ora from "ora";
 import { spawn } from "child_process";
 import { writeFileSync, mkdirSync } from "fs";
-import { join } from "path";
-import { tmpdir } from "os";
+import { dirname, join } from "path";
+import { fileURLToPath } from "url";
+import { tmpdir, homedir } from "os";
 import { startLocalRelay, setActiveProject } from "./localRelay.js";
+// ESM equivalent of __dirname — required for the rag:ingest handler to
+// locate the bundled localRagIngest.py inside the npm package.
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
 import { ensureSharedModules, getSharedDir } from "./sharedVendor.js";
 import { startLumaBrowserBridge } from "./lumaBrowserBridge.js";
 const HEARTBEAT_INTERVAL = 30_000;
@@ -374,6 +379,211 @@ export async function connect(opts) {
             socket.emit("runner:runComplete", { runId: payload.runId, status: "failed" });
         }
     });
+    // ── RAG Mode B: local-folder ingest (privacy-preserving) ─────────────
+    // The user picks "Local folder" in the Agent Builder DocsTab and types
+    // a path on their machine (e.g. C:\Users\me\Documents\refs). The server
+    // routes the ingest to this handler. The runner reads the files locally,
+    // embeds via local Ollama, and persists the Qdrant store at
+    // ~/.melaya-runner/rag/<pipelineName>/store/. The text NEVER leaves the
+    // user's box — that's the entire point of Mode B.
+    //
+    // Hard refusal: if embedderProvider is "openai", we refuse with a clear
+    // error message. Cloud embedders would defeat the data-residency
+    // guarantee. Mode B = local-only embedder, period.
+    // ── Native folder picker ──────────────────────────────────────────────
+    // FE asks the runner to pop the OS-native folder dialog so the user can
+    // pick a path WITHOUT having to type it. Browsers can't reveal absolute
+    // paths for security, so the runner — which lives on the user's box —
+    // does it. Mac: osascript. Windows: PowerShell WinForms. Linux: zenity.
+    socket.on("runner:pick-folder", async (payload) => {
+        const sid = payload?.sessionId;
+        const reply = (ok, path, error) => {
+            socket.emit("runner:pick-folder-result", { session_id: sid, ok, path, error });
+        };
+        try {
+            const { execFile } = await import("child_process");
+            const platform = process.platform;
+            const run = (cmd, args) => new Promise((resolve, reject) => {
+                execFile(cmd, args, { maxBuffer: 1024 * 64 }, (err, stdout, stderr) => {
+                    if (err) {
+                        const out = (stdout || "").trim();
+                        const errMsg = (stderr || err.message || "").trim();
+                        // osascript returns code 1 when user cancels — empty stdout
+                        if (!out && /User canceled|cancelled/i.test(errMsg))
+                            return resolve("");
+                        return reject(new Error(errMsg || "pick failed"));
+                    }
+                    resolve((stdout || "").trim());
+                });
+            });
+            let picked = "";
+            if (platform === "darwin") {
+                // osascript returns "Macintosh HD:Users:me:Documents" — convert to POSIX
+                const raw = await run("osascript", [
+                    "-e", 'tell application "System Events"',
+                    "-e", "activate",
+                    "-e", "set p to choose folder with prompt \"Select your reference docs folder\"",
+                    "-e", "end tell",
+                    "-e", "return POSIX path of p",
+                ]);
+                picked = raw.replace(/\/$/, "");
+            }
+            else if (platform === "win32") {
+                const ps = [
+                    "Add-Type -AssemblyName System.Windows.Forms;",
+                    "$f = New-Object System.Windows.Forms.FolderBrowserDialog;",
+                    "$f.Description = 'Select your reference docs folder';",
+                    "$f.ShowNewFolderButton = $true;",
+                    "if ($f.ShowDialog() -eq 'OK') { [Console]::Out.Write($f.SelectedPath) }",
+                ].join(" ");
+                picked = await run("powershell", ["-NoProfile", "-STA", "-Command", ps]);
+            }
+            else {
+                // Linux — try zenity, fall back to error if missing
+                picked = await run("zenity", [
+                    "--file-selection", "--directory",
+                    "--title=Select your reference docs folder",
+                ]);
+            }
+            if (!picked)
+                return reply(false, null, "cancelled");
+            reply(true, picked);
+        }
+        catch (e) {
+            const msg = String(e?.message || "pick failed");
+            const hint = /ENOENT|not found|spawn/i.test(msg) && process.platform === "linux"
+                ? "Install 'zenity' (apt install zenity) or type the path manually."
+                : "";
+            reply(false, null, hint ? `${msg} — ${hint}` : msg);
+        }
+    });
+    socket.on("rag:ingest", async (payload) => {
+        const sid = payload?.sessionId;
+        if (!sid)
+            return;
+        const emitProgress = (kind, fields = {}) => {
+            socket.emit("rag:ingest-progress", { session_id: sid, kind, ...fields });
+        };
+        try {
+            const prov = (payload.embedderProvider || "").toLowerCase();
+            if (prov === "openai") {
+                const reason = ("Mode B (local folder) refuses cloud embedders. " +
+                    "Pick ollama or lmstudio to keep your documents private.");
+                console.log(chalk.red(`  ✗ rag:ingest refused: ${reason}`));
+                socket.emit("rag:ingest-result", {
+                    session_id: sid, ok: false, error: reason,
+                });
+                return;
+            }
+            console.log(chalk.hex("#10B981")(`\n  🔒 RAG local-folder ingest: ${payload.pipelineName} ` +
+                `← ${payload.localFolderPath} via ${prov}:${payload.embedderModel}\n`));
+            emitProgress("started");
+            // Reuse the same venv as runner:run — agentscope.rag + ollama +
+            // qdrant-client land here.
+            const { ensurePythonEnv } = await import("./pythonEnv.js");
+            const env = await ensurePythonEnv(opts.pythonPath, payload.sharedVersion, (m) => emitProgress("venv", { message: m }));
+            if (!env.ok) {
+                emitProgress("error", { error: `venv bootstrap failed: ${env.reason}` });
+                socket.emit("rag:ingest-result", {
+                    session_id: sid, ok: false, error: `venv bootstrap failed: ${env.reason}`,
+                });
+                return;
+            }
+            // Stage the ingest script into the venv work dir so the spawned
+            // python sees the shared.runtime + the agentscope vendored tree.
+            await ensureSharedModules(opts.serverUrl, payload.sharedVersion);
+            const sharedDir = getSharedDir();
+            const ingestScript = join(sharedDir, "..", "localRagIngest.py");
+            // The script is published alongside the runner's shared bundle in
+            // the npm package itself — we copy it to a tmp dir per-invocation
+            // so re-runs are clean.
+            const workDir = join(tmpdir(), `melaya-rag-ingest-${payload.pipelineName}-${Date.now()}`);
+            mkdirSync(workDir, { recursive: true });
+            const { copyFileSync, existsSync } = await import("fs");
+            // Fallback path: when running from npm package layout
+            const candidates = [
+                ingestScript,
+                join(__dirname, "localRagIngest.py"),
+                join(__dirname, "..", "localRagIngest.py"),
+            ];
+            let foundIngest = "";
+            for (const c of candidates) {
+                if (existsSync(c)) {
+                    foundIngest = c;
+                    break;
+                }
+            }
+            if (!foundIngest) {
+                const msg = "localRagIngest.py not found in runner package — reinstall @melaya/runner";
+                emitProgress("error", { error: msg });
+                socket.emit("rag:ingest-result", {
+                    session_id: sid, ok: false, error: msg,
+                });
+                return;
+            }
+            const stagedScript = join(workDir, "localRagIngest.py");
+            copyFileSync(foundIngest, stagedScript);
+            // Resolve the on-disk path where the store will live. The Python
+            // script can compute this itself from pipelineName, but we surface
+            // it here for the FE event so the user can see where their data
+            // lives.
+            const storePath = join(homedir(), ".melaya-runner", "rag", payload.pipelineName, "store");
+            console.log(chalk.gray(`  [rag] store → ${storePath}`));
+            const proc = spawn(env.pythonPath, [
+                "-u", stagedScript,
+                "--pipeline-name", payload.pipelineName,
+                "--folder", payload.localFolderPath,
+                "--embedder-provider", prov,
+                "--embedder-model", payload.embedderModel,
+            ], {
+                env: {
+                    ...process.env,
+                    PYTHONPATH: sharedDir, // so `from agentscope.rag import …` resolves
+                },
+                cwd: workDir,
+            });
+            let stdoutBuf = "";
+            let stderrBuf = "";
+            proc.stdout.on("data", (data) => {
+                stdoutBuf += data.toString("utf-8");
+                // Stream each "progress: ..." JSON line up to the FE
+                const lines = stdoutBuf.split("\n");
+                stdoutBuf = lines.pop() || "";
+                for (const line of lines) {
+                    if (line.startsWith("progress: ")) {
+                        try {
+                            const ev = JSON.parse(line.slice("progress: ".length));
+                            emitProgress("step", ev);
+                        }
+                        catch { /* ignore malformed */ }
+                    }
+                    else if (opts.verbose && line.trim()) {
+                        console.log(chalk.gray(`  [ingest] ${line}`));
+                    }
+                }
+            });
+            proc.stderr.on("data", (data) => {
+                stderrBuf += data.toString("utf-8");
+            });
+            proc.on("close", (code) => {
+                const ok = code === 0;
+                console.log(ok
+                    ? chalk.green(`  ✓ RAG ingest complete (exit ${code})`)
+                    : chalk.red(`  ✗ RAG ingest failed (exit ${code})\n${stderrBuf.slice(-1000)}`));
+                socket.emit("rag:ingest-result", {
+                    session_id: sid, ok, exit: code,
+                    store_path: storePath,
+                    stderr_tail: ok ? "" : stderrBuf.slice(-2000),
+                });
+            });
+        }
+        catch (e) {
+            console.log(chalk.red(`  ✗ rag:ingest crashed: ${e?.message || e}`));
+            socket.emit("rag:ingest-result", {
+                session_id: sid, ok: false, error: e?.message || String(e),
+            });
+        }
+    });
     // ── LinkedIn cookie capture (one-time) ────────────────────────────
     // When the user clicks "Connect LinkedIn" in the platform UI, the
     // server emits this event. We launch a controlled Chromium window on

package/dist/localRagIngest.py

@@ -0,0 +1,239 @@
+"""Local-folder RAG ingest (Mode B) — runs on the USER'S machine via the
+@melaya/runner subprocess. Reads docs from a local folder, embeds via the
+user's local Ollama (or LM Studio), persists the Qdrant vector store at
+`~/.melaya-runner/rag/<pipeline_name>/store/`.
+
+Data residency guarantee:
+  * Document text is NEVER uploaded to the prod box
+  * Embedding API calls go to localhost (user's Ollama at :11434)
+  * Vector store sits at ~/.melaya-runner/... on the user's filesystem
+  * The only network egress is the Socket.IO progress events
+    (file names + chunk counts only, NO chunk text)
+
+Invoked by packages/runner/src/connection.ts on `rag:ingest` Socket.IO event.
+Emits `progress: <json>` lines to stdout that the parent process forwards to
+the FE so the user sees per-file ingest progress in real time.
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import datetime as _dt
+import hashlib
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+# Path hack — the runner's spawned venv has agentscope vendored at the
+# PYTHONPATH it sets (the parent connection.ts sets env.PYTHONPATH to the
+# shared bundle dir). Confirm imports work, fail loud otherwise.
+try:
+    from agentscope.rag import (  # noqa: F401
+        TextReader, PDFReader, WordReader, PowerPointReader, ExcelReader,
+    )
+except ImportError as exc:
+    print(f"ImportError: agentscope.rag not reachable on PYTHONPATH={os.environ.get('PYTHONPATH')!r}: {exc}", file=sys.stderr)
+    sys.exit(2)
+
+
+def _emit(kind: str, **fields):
+    """Stream a progress event to the runner subprocess parent which
+    forwards it to the FE over Socket.IO."""
+    payload = {"kind": kind, **fields, "ts": _dt.datetime.utcnow().isoformat() + "Z"}
+    print("progress: " + json.dumps(payload), flush=True)
+
+
+SUPPORTED_EXTS = {".txt", ".md", ".csv", ".json", ".pdf",
+                  ".docx", ".pptx", ".xlsx"}
+
+
+def _reader_for(ext: str):
+    """Map extension to agentscope reader. Returns None when no extractor
+    exists (legacy .doc, image-only PDFs that need OCR, etc.)."""
+    return {
+        ".txt":  TextReader, ".md":   TextReader,
+        ".csv":  TextReader, ".json": TextReader,
+        ".pdf":  PDFReader,
+        ".docx": WordReader,
+        ".pptx": PowerPointReader,
+        ".xlsx": ExcelReader,
+    }.get(ext)
+
+
+def _resolve_store_path(pipeline_name: str) -> Path:
+    """Mode B store path — under ~/.melaya-runner/rag/<pipeline>/store/.
+    Matches the server-side facade's resolve_store_path(mode="local_folder")
+    so cross-runtime debugging is consistent."""
+    return Path.home() / ".melaya-runner" / "rag" / pipeline_name / "store"
+
+
+def _resolve_manifest_path(pipeline_name: str) -> Path:
+    return _resolve_store_path(pipeline_name).parent / ".ingest.json"
+
+
+async def _ingest(args) -> int:
+    folder = Path(args.folder).expanduser()
+    if not folder.exists() or not folder.is_dir():
+        _emit("error", reason=f"folder not found or not a directory: {folder}")
+        return 3
+
+    # Build the facade-equivalent: pick the embedder, instantiate Qdrant.
+    # We import lazily so a missing dep produces a clear error message.
+    try:
+        if args.embedder_provider == "ollama":
+            from agentscope.embedding import OllamaTextEmbedding
+            # Centralized dimension lookup — same as the server-side facade
+            DIMS = {
+                "nomic-embed-text": 768, "nomic-embed-text:v1.5": 768,
+                "mxbai-embed-large": 1024, "all-minilm": 384,
+                "snowflake-arctic-embed": 1024,
+            }
+            dim = DIMS.get(args.embedder_model.lower())
+            if dim is None:
+                _emit("error", reason=(
+                    f"unknown ollama embedding model '{args.embedder_model}' — "
+                    f"supported: {sorted(DIMS)}"
+                ))
+                return 4
+            embedder = OllamaTextEmbedding(
+                model_name=args.embedder_model,
+                dimensions=dim,
+                host=None,  # → uses default localhost:11434
+            )
+        elif args.embedder_provider == "lmstudio":
+            from agentscope.embedding import OpenAITextEmbedding
+            # LM Studio exposes an OpenAI-compatible /v1; reuse the OpenAI
+            # client pointed at its local endpoint. Dim default = 768 (the
+            # FE picker should report the real dim per loaded model).
+            embedder = OpenAITextEmbedding(
+                api_key="lmstudio",
+                model_name=args.embedder_model,
+                dimensions=768,
+            )
+            embedder.client.base_url = "http://localhost:1234/v1"
+        else:
+            _emit("error", reason=(
+                f"refused embedder provider '{args.embedder_provider}' for Mode B "
+                "— only ollama and lmstudio keep documents private"
+            ))
+            return 5
+
+        from agentscope.rag import QdrantStore, SimpleKnowledge
+        store_path = _resolve_store_path(args.pipeline_name)
+        store_path.mkdir(parents=True, exist_ok=True)
+        store = QdrantStore(
+            location=str(store_path),
+            collection_name=args.pipeline_name,
+            dimensions=embedder.dimensions,
+            distance="Cosine",
+        )
+        kb = SimpleKnowledge(embedding_store=store, embedding_model=embedder)
+    except Exception as exc:
+        _emit("error", reason=f"failed to build KB: {type(exc).__name__}: {exc}")
+        return 6
+
+    # Read the manifest to skip unchanged files
+    manifest_path = _resolve_manifest_path(args.pipeline_name)
+    if manifest_path.exists():
+        try:
+            manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+        except Exception:
+            manifest = {"files": {}, "embedder": None}
+    else:
+        manifest = {"files": {}, "embedder": None}
+    stored = manifest.get("embedder") or {}
+    embedder_changed = (
+        stored.get("provider") != args.embedder_provider
+        or stored.get("model") != args.embedder_model
+    )
+    if embedder_changed:
+        _emit("embedder_changed", was=stored, now={
+            "provider": args.embedder_provider, "model": args.embedder_model,
+        })
+
+    started = time.time()
+    chunks_added = 0
+    files_processed = 0
+    skipped = 0
+    files_failed = []
+
+    # Walk the folder — only top-level + one level deep. Going arbitrarily
+    # deep would silently include things like venv/.git which would be
+    # surprising. If someone needs recursive, they can manually flatten.
+    candidates = [p for p in sorted(folder.iterdir()) if p.is_file()]
+    candidates += [p for sub in folder.iterdir() if sub.is_dir() and not sub.name.startswith(".")
+                   for p in sorted(sub.iterdir()) if p.is_file()]
+
+    total = sum(1 for p in candidates if p.suffix.lower() in SUPPORTED_EXTS)
+    _emit("scanned", folder=str(folder), total_supported=total)
+
+    for f in candidates:
+        ext = f.suffix.lower()
+        if ext not in SUPPORTED_EXTS:
+            continue
+        try:
+            raw = f.read_bytes()
+        except Exception as exc:
+            files_failed.append({"file": str(f), "error": f"read: {type(exc).__name__}: {exc}"})
+            continue
+        sha = hashlib.sha256(raw).hexdigest()
+        prev = manifest["files"].get(f.name, {})
+        if (not embedder_changed) and prev.get("sha256") == sha:
+            skipped += 1
+            _emit("skipped_sha_clean", file=f.name)
+            continue
+        reader_cls = _reader_for(ext)
+        if reader_cls is None:
+            files_failed.append({"file": str(f), "error": "no extractor"})
+            continue
+        try:
+            reader = reader_cls()
+            docs = await reader(str(f))
+            for doc in docs:
+                if hasattr(doc.metadata, "doc_id"):
+                    doc.metadata.doc_id = f.name
+            await kb.add_documents(docs)
+            chunks_added += len(docs)
+            manifest["files"][f.name] = {
+                "sha256": sha, "chunks": len(docs),
+                "ingested_at": _dt.datetime.utcnow().isoformat() + "Z",
+            }
+            files_processed += 1
+            _emit("file_done", file=f.name, chunks=len(docs))
+        except Exception as exc:
+            files_failed.append({
+                "file": str(f), "error": f"{type(exc).__name__}: {exc}",
+            })
+            _emit("file_failed", file=f.name, error=str(exc))
+
+    manifest["embedder"] = {
+        "provider": args.embedder_provider, "model": args.embedder_model,
+    }
+    manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
+
+    elapsed_ms = int((time.time() - started) * 1000)
+    _emit("done",
+          files_processed=files_processed,
+          chunks_added=chunks_added,
+          skipped=skipped,
+          embedder_changed=embedder_changed,
+          elapsed_ms=elapsed_ms,
+          errors=files_failed)
+    return 0
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--pipeline-name", required=True)
+    ap.add_argument("--folder", required=True)
+    ap.add_argument("--embedder-provider", required=True, choices=["ollama", "lmstudio"])
+    ap.add_argument("--embedder-model", required=True)
+    args = ap.parse_args()
+    sys.exit(asyncio.run(_ingest(args)))
+
+
+if __name__ == "__main__":
+    main()

package/dist/modelLoader.js

@@ -382,42 +382,38 @@ export async function preflightModel(configJson, onProgress = (m) => console.log
     catch {
         return { ok: true }; /* let main.py error naturally */
     }
-    // Pipeline-level + per-step / per-agent providers — match the same
-    // walk as builder_server.py:2716-2725 so we honour the agent that
-    // overrides the top-level pick.
-    const providers = new Set();
-    const models = new Set();
-    const top = String(cfg.model_provider ?? cfg.model?.provider ?? "").toLowerCase();
-    if (top)
-        providers.add(top);
-    const topModel = String(cfg.model_name ?? cfg.model?.name ?? "");
-    if (topModel)
-        models.add(topModel);
+    // Pipeline-level + per-step / per-agent (provider, model) PAIRS — must
+    // stay paired so a melaya_ai / openai / anthropic agent isn't checked
+    // against LM Studio just because a different agent in the same pipeline
+    // happens to use LM Studio. Mixed-provider pipelines were preflighting
+    // every cloud model against every local runner → spurious 'not found
+    // in LM Studio' failures.
+    const pairs = new Set(); // serialized "provider|model" for dedup
+    const addPair = (p, m) => {
+        if (p && m)
+            pairs.add(`${p.toLowerCase()}|${m}`);
+    };
+    addPair(String(cfg.model_provider ?? cfg.model?.provider ?? ""), String(cfg.model_name ?? cfg.model?.name ?? ""));
     for (const step of (cfg.steps ?? [])) {
         const a = step?.agent ?? {};
-        const p = String(a?.model?.provider ?? "").toLowerCase();
-        const n = String(a?.model?.name ?? "");
-        if (p)
-            providers.add(p);
-        if (n)
-            models.add(n);
+        addPair(String(a?.model?.provider ?? a?.model_provider ?? ""), String(a?.model?.name ?? a?.model_name ?? ""));
     }
     for (const a of (cfg.agents ?? [])) {
-        const p = String(a?.model_provider ?? a?.model?.provider ?? "").toLowerCase();
-        const n = String(a?.model_name ?? a?.model?.name ?? "");
-        if (p)
-            providers.add(p);
-        if (n)
-            models.add(n);
+        // Nested wins over flat — the ModelPicker writes `agent.model`
+        // but doesn't always purge the legacy `agent.model_provider` /
+        // `agent.model_name` fields, so they drift stale across edits.
+        addPair(String(a?.model?.provider ?? a?.model_provider ?? ""), String(a?.model?.name ?? a?.model_name ?? ""));
     }
-    // Run preflight for every distinct (provider, model) combo this
-    // pipeline will exercise. One bad agent = whole run fails fast.
+    // Run preflight only for LOCAL providers — cloud (openai / anthropic /
+    // claude_code / codex / melaya_ai / zhipu) doesn't need the runner.
     const tasks = [];
-    for (const m of models) {
-        if (providers.has("lmstudio"))
-            tasks.push(preflightLmStudio(m, onProgress));
-        if (providers.has("ollama"))
-            tasks.push(preflightOllama(m, onProgress));
+    for (const pair of pairs) {
+        const [provider, model] = pair.split("|");
+        if (provider === "lmstudio")
+            tasks.push(preflightLmStudio(model, onProgress));
+        else if (provider === "ollama")
+            tasks.push(preflightOllama(model, onProgress));
+        // every other provider is cloud-only — no preflight needed.
     }
     if (tasks.length === 0)
         return { ok: true };

package/dist/pythonEnv.js

@@ -96,6 +96,18 @@ const PIP_DEPS = [
     // shared.tools.telegram_user_tools. The bot-API path in
     // shared.tools.messaging.py uses plain HTTP and needs no extra dep.
     "telethon",
+    // ── Vector RAG (retrieval mode) ────────────────────────────────────────
+    // ollama — Python client used by agentscope.embedding.OllamaTextEmbedding.
+    //   Talks to localhost:11434 (the user's local Ollama daemon) to embed
+    //   chunks without exfiltrating the underlying text to a cloud API. This
+    //   is the only embedder allowed in privacy-preserving Mode B.
+    //
+    // (qdrant-client is ALREADY in this list above — it provides the
+    // embedded Qdrant store via agentscope.rag._store.QdrantStore at the
+    // ~/.melaya-runner/rag/<pipeline_name>/store/ path. Path-based Qdrant
+    // ships Windows wheels, MilvusLite does not — using Qdrant for OS
+    // parity across runners.)
+    "ollama",
 ];
 export function venvPython() {
     return platform() === "win32"

package/localRagIngest.py

@@ -0,0 +1,239 @@
+"""Local-folder RAG ingest (Mode B) — runs on the USER'S machine via the
+@melaya/runner subprocess. Reads docs from a local folder, embeds via the
+user's local Ollama (or LM Studio), persists the Qdrant vector store at
+`~/.melaya-runner/rag/<pipeline_name>/store/`.
+
+Data residency guarantee:
+  * Document text is NEVER uploaded to the prod box
+  * Embedding API calls go to localhost (user's Ollama at :11434)
+  * Vector store sits at ~/.melaya-runner/... on the user's filesystem
+  * The only network egress is the Socket.IO progress events
+    (file names + chunk counts only, NO chunk text)
+
+Invoked by packages/runner/src/connection.ts on `rag:ingest` Socket.IO event.
+Emits `progress: <json>` lines to stdout that the parent process forwards to
+the FE so the user sees per-file ingest progress in real time.
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import datetime as _dt
+import hashlib
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+# Path hack — the runner's spawned venv has agentscope vendored at the
+# PYTHONPATH it sets (the parent connection.ts sets env.PYTHONPATH to the
+# shared bundle dir). Confirm imports work, fail loud otherwise.
+try:
+    from agentscope.rag import (  # noqa: F401
+        TextReader, PDFReader, WordReader, PowerPointReader, ExcelReader,
+    )
+except ImportError as exc:
+    print(f"ImportError: agentscope.rag not reachable on PYTHONPATH={os.environ.get('PYTHONPATH')!r}: {exc}", file=sys.stderr)
+    sys.exit(2)
+
+
+def _emit(kind: str, **fields):
+    """Stream a progress event to the runner subprocess parent which
+    forwards it to the FE over Socket.IO."""
+    payload = {"kind": kind, **fields, "ts": _dt.datetime.utcnow().isoformat() + "Z"}
+    print("progress: " + json.dumps(payload), flush=True)
+
+
+SUPPORTED_EXTS = {".txt", ".md", ".csv", ".json", ".pdf",
+                  ".docx", ".pptx", ".xlsx"}
+
+
+def _reader_for(ext: str):
+    """Map extension to agentscope reader. Returns None when no extractor
+    exists (legacy .doc, image-only PDFs that need OCR, etc.)."""
+    return {
+        ".txt":  TextReader, ".md":   TextReader,
+        ".csv":  TextReader, ".json": TextReader,
+        ".pdf":  PDFReader,
+        ".docx": WordReader,
+        ".pptx": PowerPointReader,
+        ".xlsx": ExcelReader,
+    }.get(ext)
+
+
+def _resolve_store_path(pipeline_name: str) -> Path:
+    """Mode B store path — under ~/.melaya-runner/rag/<pipeline>/store/.
+    Matches the server-side facade's resolve_store_path(mode="local_folder")
+    so cross-runtime debugging is consistent."""
+    return Path.home() / ".melaya-runner" / "rag" / pipeline_name / "store"
+
+
+def _resolve_manifest_path(pipeline_name: str) -> Path:
+    return _resolve_store_path(pipeline_name).parent / ".ingest.json"
+
+
+async def _ingest(args) -> int:
+    folder = Path(args.folder).expanduser()
+    if not folder.exists() or not folder.is_dir():
+        _emit("error", reason=f"folder not found or not a directory: {folder}")
+        return 3
+
+    # Build the facade-equivalent: pick the embedder, instantiate Qdrant.
+    # We import lazily so a missing dep produces a clear error message.
+    try:
+        if args.embedder_provider == "ollama":
+            from agentscope.embedding import OllamaTextEmbedding
+            # Centralized dimension lookup — same as the server-side facade
+            DIMS = {
+                "nomic-embed-text": 768, "nomic-embed-text:v1.5": 768,
+                "mxbai-embed-large": 1024, "all-minilm": 384,
+                "snowflake-arctic-embed": 1024,
+            }
+            dim = DIMS.get(args.embedder_model.lower())
+            if dim is None:
+                _emit("error", reason=(
+                    f"unknown ollama embedding model '{args.embedder_model}' — "
+                    f"supported: {sorted(DIMS)}"
+                ))
+                return 4
+            embedder = OllamaTextEmbedding(
+                model_name=args.embedder_model,
+                dimensions=dim,
+                host=None,  # → uses default localhost:11434
+            )
+        elif args.embedder_provider == "lmstudio":
+            from agentscope.embedding import OpenAITextEmbedding
+            # LM Studio exposes an OpenAI-compatible /v1; reuse the OpenAI
+            # client pointed at its local endpoint. Dim default = 768 (the
+            # FE picker should report the real dim per loaded model).
+            embedder = OpenAITextEmbedding(
+                api_key="lmstudio",
+                model_name=args.embedder_model,
+                dimensions=768,
+            )
+            embedder.client.base_url = "http://localhost:1234/v1"
+        else:
+            _emit("error", reason=(
+                f"refused embedder provider '{args.embedder_provider}' for Mode B "
+                "— only ollama and lmstudio keep documents private"
+            ))
+            return 5
+
+        from agentscope.rag import QdrantStore, SimpleKnowledge
+        store_path = _resolve_store_path(args.pipeline_name)
+        store_path.mkdir(parents=True, exist_ok=True)
+        store = QdrantStore(
+            location=str(store_path),
+            collection_name=args.pipeline_name,
+            dimensions=embedder.dimensions,
+            distance="Cosine",
+        )
+        kb = SimpleKnowledge(embedding_store=store, embedding_model=embedder)
+    except Exception as exc:
+        _emit("error", reason=f"failed to build KB: {type(exc).__name__}: {exc}")
+        return 6
+
+    # Read the manifest to skip unchanged files
+    manifest_path = _resolve_manifest_path(args.pipeline_name)
+    if manifest_path.exists():
+        try:
+            manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+        except Exception:
+            manifest = {"files": {}, "embedder": None}
+    else:
+        manifest = {"files": {}, "embedder": None}
+    stored = manifest.get("embedder") or {}
+    embedder_changed = (
+        stored.get("provider") != args.embedder_provider
+        or stored.get("model") != args.embedder_model
+    )
+    if embedder_changed:
+        _emit("embedder_changed", was=stored, now={
+            "provider": args.embedder_provider, "model": args.embedder_model,
+        })
+
+    started = time.time()
+    chunks_added = 0
+    files_processed = 0
+    skipped = 0
+    files_failed = []
+
+    # Walk the folder — only top-level + one level deep. Going arbitrarily
+    # deep would silently include things like venv/.git which would be
+    # surprising. If someone needs recursive, they can manually flatten.
+    candidates = [p for p in sorted(folder.iterdir()) if p.is_file()]
+    candidates += [p for sub in folder.iterdir() if sub.is_dir() and not sub.name.startswith(".")
+                   for p in sorted(sub.iterdir()) if p.is_file()]
+
+    total = sum(1 for p in candidates if p.suffix.lower() in SUPPORTED_EXTS)
+    _emit("scanned", folder=str(folder), total_supported=total)
+
+    for f in candidates:
+        ext = f.suffix.lower()
+        if ext not in SUPPORTED_EXTS:
+            continue
+        try:
+            raw = f.read_bytes()
+        except Exception as exc:
+            files_failed.append({"file": str(f), "error": f"read: {type(exc).__name__}: {exc}"})
+            continue
+        sha = hashlib.sha256(raw).hexdigest()
+        prev = manifest["files"].get(f.name, {})
+        if (not embedder_changed) and prev.get("sha256") == sha:
+            skipped += 1
+            _emit("skipped_sha_clean", file=f.name)
+            continue
+        reader_cls = _reader_for(ext)
+        if reader_cls is None:
+            files_failed.append({"file": str(f), "error": "no extractor"})
+            continue
+        try:
+            reader = reader_cls()
+            docs = await reader(str(f))
+            for doc in docs:
+                if hasattr(doc.metadata, "doc_id"):
+                    doc.metadata.doc_id = f.name
+            await kb.add_documents(docs)
+            chunks_added += len(docs)
+            manifest["files"][f.name] = {
+                "sha256": sha, "chunks": len(docs),
+                "ingested_at": _dt.datetime.utcnow().isoformat() + "Z",
+            }
+            files_processed += 1
+            _emit("file_done", file=f.name, chunks=len(docs))
+        except Exception as exc:
+            files_failed.append({
+                "file": str(f), "error": f"{type(exc).__name__}: {exc}",
+            })
+            _emit("file_failed", file=f.name, error=str(exc))
+
+    manifest["embedder"] = {
+        "provider": args.embedder_provider, "model": args.embedder_model,
+    }
+    manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
+
+    elapsed_ms = int((time.time() - started) * 1000)
+    _emit("done",
+          files_processed=files_processed,
+          chunks_added=chunks_added,
+          skipped=skipped,
+          embedder_changed=embedder_changed,
+          elapsed_ms=elapsed_ms,
+          errors=files_failed)
+    return 0
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--pipeline-name", required=True)
+    ap.add_argument("--folder", required=True)
+    ap.add_argument("--embedder-provider", required=True, choices=["ollama", "lmstudio"])
+    ap.add_argument("--embedder-model", required=True)
+    args = ap.parse_args()
+    sys.exit(asyncio.run(_ingest(args)))
+
+
+if __name__ == "__main__":
+    main()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.0.36",
+  "version": "1.0.38",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,
@@ -12,10 +12,12 @@
   "files": [
     "dist/**/*.js",
     "dist/**/*.d.ts",
+    "dist/**/*.py",
+    "localRagIngest.py",
     "README.md"
   ],
   "scripts": {
-    "build": "tsc",
+    "build": "tsc && node -e \"require('fs').copyFileSync('localRagIngest.py','dist/localRagIngest.py')\"",
     "prepublishOnly": "npm run build"
   },
   "dependencies": {