@melaya/runner 1.0.37 → 1.0.39

package/dist/connection.js

@@ -390,6 +390,73 @@ export async function connect(opts) {
     // Hard refusal: if embedderProvider is "openai", we refuse with a clear
     // error message. Cloud embedders would defeat the data-residency
     // guarantee. Mode B = local-only embedder, period.
+    // ── Native folder picker ──────────────────────────────────────────────
+    // FE asks the runner to pop the OS-native folder dialog so the user can
+    // pick a path WITHOUT having to type it. Browsers can't reveal absolute
+    // paths for security, so the runner — which lives on the user's box —
+    // does it. Mac: osascript. Windows: PowerShell WinForms. Linux: zenity.
+    socket.on("runner:pick-folder", async (payload) => {
+        const sid = payload?.sessionId;
+        const reply = (ok, path, error) => {
+            socket.emit("runner:pick-folder-result", { session_id: sid, ok, path, error });
+        };
+        try {
+            const { execFile } = await import("child_process");
+            const platform = process.platform;
+            const run = (cmd, args) => new Promise((resolve, reject) => {
+                execFile(cmd, args, { maxBuffer: 1024 * 64 }, (err, stdout, stderr) => {
+                    if (err) {
+                        const out = (stdout || "").trim();
+                        const errMsg = (stderr || err.message || "").trim();
+                        // osascript returns code 1 when user cancels — empty stdout
+                        if (!out && /User canceled|cancelled/i.test(errMsg))
+                            return resolve("");
+                        return reject(new Error(errMsg || "pick failed"));
+                    }
+                    resolve((stdout || "").trim());
+                });
+            });
+            let picked = "";
+            if (platform === "darwin") {
+                // osascript returns "Macintosh HD:Users:me:Documents" — convert to POSIX
+                const raw = await run("osascript", [
+                    "-e", 'tell application "System Events"',
+                    "-e", "activate",
+                    "-e", "set p to choose folder with prompt \"Select your reference docs folder\"",
+                    "-e", "end tell",
+                    "-e", "return POSIX path of p",
+                ]);
+                picked = raw.replace(/\/$/, "");
+            }
+            else if (platform === "win32") {
+                const ps = [
+                    "Add-Type -AssemblyName System.Windows.Forms;",
+                    "$f = New-Object System.Windows.Forms.FolderBrowserDialog;",
+                    "$f.Description = 'Select your reference docs folder';",
+                    "$f.ShowNewFolderButton = $true;",
+                    "if ($f.ShowDialog() -eq 'OK') { [Console]::Out.Write($f.SelectedPath) }",
+                ].join(" ");
+                picked = await run("powershell", ["-NoProfile", "-STA", "-Command", ps]);
+            }
+            else {
+                // Linux — try zenity, fall back to error if missing
+                picked = await run("zenity", [
+                    "--file-selection", "--directory",
+                    "--title=Select your reference docs folder",
+                ]);
+            }
+            if (!picked)
+                return reply(false, null, "cancelled");
+            reply(true, picked);
+        }
+        catch (e) {
+            const msg = String(e?.message || "pick failed");
+            const hint = /ENOENT|not found|spawn/i.test(msg) && process.platform === "linux"
+                ? "Install 'zenity' (apt install zenity) or type the path manually."
+                : "";
+            reply(false, null, hint ? `${msg} — ${hint}` : msg);
+        }
+    });
     socket.on("rag:ingest", async (payload) => {
         const sid = payload?.sessionId;
         if (!sid)

package/dist/localRagIngest.py

@@ -0,0 +1,252 @@
+"""Local-folder RAG ingest (Mode B) — runs on the USER'S machine via the
+@melaya/runner subprocess. Reads docs from a local folder, embeds via the
+user's local Ollama (or LM Studio), persists the Qdrant vector store at
+`~/.melaya-runner/rag/<pipeline_name>/store/`.
+
+Data residency guarantee:
+  * Document text is NEVER uploaded to the prod box
+  * Embedding API calls go to localhost (user's Ollama at :11434)
+  * Vector store sits at ~/.melaya-runner/... on the user's filesystem
+  * The only network egress is the Socket.IO progress events
+    (file names + chunk counts only, NO chunk text)
+
+Invoked by packages/runner/src/connection.ts on `rag:ingest` Socket.IO event.
+Emits `progress: <json>` lines to stdout that the parent process forwards to
+the FE so the user sees per-file ingest progress in real time.
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import datetime as _dt
+import hashlib
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+# Path hack — the runner's spawned venv has agentscope vendored at the
+# PYTHONPATH it sets (the parent connection.ts sets env.PYTHONPATH to the
+# shared bundle dir). Confirm imports work, fail loud otherwise.
+try:
+    from agentscope.rag import (  # noqa: F401
+        TextReader, PDFReader, WordReader, PowerPointReader, ExcelReader,
+    )
+except ImportError as exc:
+    print(f"ImportError: agentscope.rag not reachable on PYTHONPATH={os.environ.get('PYTHONPATH')!r}: {exc}", file=sys.stderr)
+    sys.exit(2)
+
+
+def _emit(kind: str, **fields):
+    """Stream a progress event to the runner subprocess parent which
+    forwards it to the FE over Socket.IO."""
+    payload = {"kind": kind, **fields, "ts": _dt.datetime.utcnow().isoformat() + "Z"}
+    print("progress: " + json.dumps(payload), flush=True)
+
+
+SUPPORTED_EXTS = {".txt", ".md", ".csv", ".json", ".pdf",
+                  ".docx", ".pptx", ".xlsx"}
+
+
+def _reader_for(ext: str):
+    """Map extension to agentscope reader. Returns None when no extractor
+    exists (legacy .doc, image-only PDFs that need OCR, etc.)."""
+    return {
+        ".txt":  TextReader, ".md":   TextReader,
+        ".csv":  TextReader, ".json": TextReader,
+        ".pdf":  PDFReader,
+        ".docx": WordReader,
+        ".pptx": PowerPointReader,
+        ".xlsx": ExcelReader,
+    }.get(ext)
+
+
+def _resolve_store_path(pipeline_name: str) -> Path:
+    """Mode B store path — under ~/.melaya-runner/rag/<pipeline>/store/.
+    Matches the server-side facade's resolve_store_path(mode="local_folder")
+    so cross-runtime debugging is consistent."""
+    return Path.home() / ".melaya-runner" / "rag" / pipeline_name / "store"
+
+
+def _resolve_manifest_path(pipeline_name: str) -> Path:
+    return _resolve_store_path(pipeline_name).parent / ".ingest.json"
+
+
+async def _ingest(args) -> int:
+    folder = Path(args.folder).expanduser()
+    if not folder.exists() or not folder.is_dir():
+        _emit("error", reason=f"folder not found or not a directory: {folder}")
+        return 3
+
+    # Build the facade-equivalent: pick the embedder, instantiate Qdrant.
+    # We import lazily so a missing dep produces a clear error message.
+    try:
+        if args.embedder_provider == "ollama":
+            from agentscope.embedding import OllamaTextEmbedding
+            # Centralized dimension lookup — same as the server-side facade
+            DIMS = {
+                "nomic-embed-text": 768, "nomic-embed-text:v1.5": 768,
+                "mxbai-embed-large": 1024, "all-minilm": 384,
+                "snowflake-arctic-embed": 1024,
+            }
+            dim = DIMS.get(args.embedder_model.lower())
+            if dim is None:
+                _emit("error", reason=(
+                    f"unknown ollama embedding model '{args.embedder_model}' — "
+                    f"supported: {sorted(DIMS)}"
+                ))
+                return 4
+            embedder = OllamaTextEmbedding(
+                model_name=args.embedder_model,
+                dimensions=dim,
+                host=None,  # → uses default localhost:11434
+            )
+        elif args.embedder_provider == "lmstudio":
+            from agentscope.embedding import OpenAITextEmbedding
+            # LM Studio exposes an OpenAI-compatible /v1; reuse the OpenAI
+            # client pointed at its local endpoint. Dim default = 768 (the
+            # FE picker should report the real dim per loaded model).
+            embedder = OpenAITextEmbedding(
+                api_key="lmstudio",
+                model_name=args.embedder_model,
+                dimensions=768,
+            )
+            embedder.client.base_url = "http://localhost:1234/v1"
+            # LM Studio's /v1/embeddings rejects the OpenAI-specific
+            # `dimensions` kwarg (only OpenAI text-embedding-3 supports
+            # output-dim reduction). Strip it before each call so the
+            # request is shaped exactly as LMS expects.
+            _orig_create = embedder.client.embeddings.create
+            async def _lms_create(**kw):
+                kw.pop("dimensions", None)
+                return await _orig_create(**kw)
+            embedder.client.embeddings.create = _lms_create  # type: ignore[assignment]
+        else:
+            _emit("error", reason=(
+                f"refused embedder provider '{args.embedder_provider}' for Mode B "
+                "— only ollama and lmstudio keep documents private"
+            ))
+            return 5
+
+        from agentscope.rag import QdrantStore, SimpleKnowledge
+        store_path = _resolve_store_path(args.pipeline_name)
+        store_path.mkdir(parents=True, exist_ok=True)
+        store = QdrantStore(
+            location=str(store_path),
+            collection_name=args.pipeline_name,
+            dimensions=embedder.dimensions,
+            distance="Cosine",
+        )
+        kb = SimpleKnowledge(embedding_store=store, embedding_model=embedder)
+    except Exception as exc:
+        _emit("error", reason=f"failed to build KB: {type(exc).__name__}: {exc}")
+        return 6
+
+    # Read the manifest to skip unchanged files
+    manifest_path = _resolve_manifest_path(args.pipeline_name)
+    if manifest_path.exists():
+        try:
+            manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+        except Exception:
+            manifest = {"files": {}, "embedder": None}
+    else:
+        manifest = {"files": {}, "embedder": None}
+    stored = manifest.get("embedder") or {}
+    embedder_changed = (
+        stored.get("provider") != args.embedder_provider
+        or stored.get("model") != args.embedder_model
+    )
+    if embedder_changed:
+        _emit("embedder_changed", was=stored, now={
+            "provider": args.embedder_provider, "model": args.embedder_model,
+        })
+
+    started = time.time()
+    chunks_added = 0
+    files_processed = 0
+    skipped = 0
+    files_failed = []
+
+    # Walk the folder — only top-level + one level deep. Going arbitrarily
+    # deep would silently include things like venv/.git which would be
+    # surprising. If someone needs recursive, they can manually flatten.
+    candidates = [p for p in sorted(folder.iterdir()) if p.is_file()]
+    candidates += [p for sub in folder.iterdir() if sub.is_dir() and not sub.name.startswith(".")
+                   for p in sorted(sub.iterdir()) if p.is_file()]
+
+    total = sum(1 for p in candidates if p.suffix.lower() in SUPPORTED_EXTS)
+    _emit("scanned", folder=str(folder), total_supported=total)
+
+    for f in candidates:
+        ext = f.suffix.lower()
+        if ext not in SUPPORTED_EXTS:
+            continue
+        # macOS AppleDouble resource forks (._foo.docx, ._.DS_Store, …) —
+        # they're not real documents and every extractor will choke on them.
+        if f.name.startswith("._") or f.name == ".DS_Store":
+            continue
+        try:
+            raw = f.read_bytes()
+        except Exception as exc:
+            files_failed.append({"file": str(f), "error": f"read: {type(exc).__name__}: {exc}"})
+            continue
+        sha = hashlib.sha256(raw).hexdigest()
+        prev = manifest["files"].get(f.name, {})
+        if (not embedder_changed) and prev.get("sha256") == sha:
+            skipped += 1
+            _emit("skipped_sha_clean", file=f.name)
+            continue
+        reader_cls = _reader_for(ext)
+        if reader_cls is None:
+            files_failed.append({"file": str(f), "error": "no extractor"})
+            continue
+        try:
+            reader = reader_cls()
+            docs = await reader(str(f))
+            for doc in docs:
+                if hasattr(doc.metadata, "doc_id"):
+                    doc.metadata.doc_id = f.name
+            await kb.add_documents(docs)
+            chunks_added += len(docs)
+            manifest["files"][f.name] = {
+                "sha256": sha, "chunks": len(docs),
+                "ingested_at": _dt.datetime.utcnow().isoformat() + "Z",
+            }
+            files_processed += 1
+            _emit("file_done", file=f.name, chunks=len(docs))
+        except Exception as exc:
+            files_failed.append({
+                "file": str(f), "error": f"{type(exc).__name__}: {exc}",
+            })
+            _emit("file_failed", file=f.name, error=str(exc))
+
+    manifest["embedder"] = {
+        "provider": args.embedder_provider, "model": args.embedder_model,
+    }
+    manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
+
+    elapsed_ms = int((time.time() - started) * 1000)
+    _emit("done",
+          files_processed=files_processed,
+          chunks_added=chunks_added,
+          skipped=skipped,
+          embedder_changed=embedder_changed,
+          elapsed_ms=elapsed_ms,
+          errors=files_failed)
+    return 0
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--pipeline-name", required=True)
+    ap.add_argument("--folder", required=True)
+    ap.add_argument("--embedder-provider", required=True, choices=["ollama", "lmstudio"])
+    ap.add_argument("--embedder-model", required=True)
+    args = ap.parse_args()
+    sys.exit(asyncio.run(_ingest(args)))
+
+
+if __name__ == "__main__":
+    main()

package/localRagIngest.py

@@ -0,0 +1,252 @@
+"""Local-folder RAG ingest (Mode B) — runs on the USER'S machine via the
+@melaya/runner subprocess. Reads docs from a local folder, embeds via the
+user's local Ollama (or LM Studio), persists the Qdrant vector store at
+`~/.melaya-runner/rag/<pipeline_name>/store/`.
+
+Data residency guarantee:
+  * Document text is NEVER uploaded to the prod box
+  * Embedding API calls go to localhost (user's Ollama at :11434)
+  * Vector store sits at ~/.melaya-runner/... on the user's filesystem
+  * The only network egress is the Socket.IO progress events
+    (file names + chunk counts only, NO chunk text)
+
+Invoked by packages/runner/src/connection.ts on `rag:ingest` Socket.IO event.
+Emits `progress: <json>` lines to stdout that the parent process forwards to
+the FE so the user sees per-file ingest progress in real time.
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import datetime as _dt
+import hashlib
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+# Path hack — the runner's spawned venv has agentscope vendored at the
+# PYTHONPATH it sets (the parent connection.ts sets env.PYTHONPATH to the
+# shared bundle dir). Confirm imports work, fail loud otherwise.
+try:
+    from agentscope.rag import (  # noqa: F401
+        TextReader, PDFReader, WordReader, PowerPointReader, ExcelReader,
+    )
+except ImportError as exc:
+    print(f"ImportError: agentscope.rag not reachable on PYTHONPATH={os.environ.get('PYTHONPATH')!r}: {exc}", file=sys.stderr)
+    sys.exit(2)
+
+
+def _emit(kind: str, **fields):
+    """Stream a progress event to the runner subprocess parent which
+    forwards it to the FE over Socket.IO."""
+    payload = {"kind": kind, **fields, "ts": _dt.datetime.utcnow().isoformat() + "Z"}
+    print("progress: " + json.dumps(payload), flush=True)
+
+
+SUPPORTED_EXTS = {".txt", ".md", ".csv", ".json", ".pdf",
+                  ".docx", ".pptx", ".xlsx"}
+
+
+def _reader_for(ext: str):
+    """Map extension to agentscope reader. Returns None when no extractor
+    exists (legacy .doc, image-only PDFs that need OCR, etc.)."""
+    return {
+        ".txt":  TextReader, ".md":   TextReader,
+        ".csv":  TextReader, ".json": TextReader,
+        ".pdf":  PDFReader,
+        ".docx": WordReader,
+        ".pptx": PowerPointReader,
+        ".xlsx": ExcelReader,
+    }.get(ext)
+
+
+def _resolve_store_path(pipeline_name: str) -> Path:
+    """Mode B store path — under ~/.melaya-runner/rag/<pipeline>/store/.
+    Matches the server-side facade's resolve_store_path(mode="local_folder")
+    so cross-runtime debugging is consistent."""
+    return Path.home() / ".melaya-runner" / "rag" / pipeline_name / "store"
+
+
+def _resolve_manifest_path(pipeline_name: str) -> Path:
+    return _resolve_store_path(pipeline_name).parent / ".ingest.json"
+
+
+async def _ingest(args) -> int:
+    folder = Path(args.folder).expanduser()
+    if not folder.exists() or not folder.is_dir():
+        _emit("error", reason=f"folder not found or not a directory: {folder}")
+        return 3
+
+    # Build the facade-equivalent: pick the embedder, instantiate Qdrant.
+    # We import lazily so a missing dep produces a clear error message.
+    try:
+        if args.embedder_provider == "ollama":
+            from agentscope.embedding import OllamaTextEmbedding
+            # Centralized dimension lookup — same as the server-side facade
+            DIMS = {
+                "nomic-embed-text": 768, "nomic-embed-text:v1.5": 768,
+                "mxbai-embed-large": 1024, "all-minilm": 384,
+                "snowflake-arctic-embed": 1024,
+            }
+            dim = DIMS.get(args.embedder_model.lower())
+            if dim is None:
+                _emit("error", reason=(
+                    f"unknown ollama embedding model '{args.embedder_model}' — "
+                    f"supported: {sorted(DIMS)}"
+                ))
+                return 4
+            embedder = OllamaTextEmbedding(
+                model_name=args.embedder_model,
+                dimensions=dim,
+                host=None,  # → uses default localhost:11434
+            )
+        elif args.embedder_provider == "lmstudio":
+            from agentscope.embedding import OpenAITextEmbedding
+            # LM Studio exposes an OpenAI-compatible /v1; reuse the OpenAI
+            # client pointed at its local endpoint. Dim default = 768 (the
+            # FE picker should report the real dim per loaded model).
+            embedder = OpenAITextEmbedding(
+                api_key="lmstudio",
+                model_name=args.embedder_model,
+                dimensions=768,
+            )
+            embedder.client.base_url = "http://localhost:1234/v1"
+            # LM Studio's /v1/embeddings rejects the OpenAI-specific
+            # `dimensions` kwarg (only OpenAI text-embedding-3 supports
+            # output-dim reduction). Strip it before each call so the
+            # request is shaped exactly as LMS expects.
+            _orig_create = embedder.client.embeddings.create
+            async def _lms_create(**kw):
+                kw.pop("dimensions", None)
+                return await _orig_create(**kw)
+            embedder.client.embeddings.create = _lms_create  # type: ignore[assignment]
+        else:
+            _emit("error", reason=(
+                f"refused embedder provider '{args.embedder_provider}' for Mode B "
+                "— only ollama and lmstudio keep documents private"
+            ))
+            return 5
+
+        from agentscope.rag import QdrantStore, SimpleKnowledge
+        store_path = _resolve_store_path(args.pipeline_name)
+        store_path.mkdir(parents=True, exist_ok=True)
+        store = QdrantStore(
+            location=str(store_path),
+            collection_name=args.pipeline_name,
+            dimensions=embedder.dimensions,
+            distance="Cosine",
+        )
+        kb = SimpleKnowledge(embedding_store=store, embedding_model=embedder)
+    except Exception as exc:
+        _emit("error", reason=f"failed to build KB: {type(exc).__name__}: {exc}")
+        return 6
+
+    # Read the manifest to skip unchanged files
+    manifest_path = _resolve_manifest_path(args.pipeline_name)
+    if manifest_path.exists():
+        try:
+            manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+        except Exception:
+            manifest = {"files": {}, "embedder": None}
+    else:
+        manifest = {"files": {}, "embedder": None}
+    stored = manifest.get("embedder") or {}
+    embedder_changed = (
+        stored.get("provider") != args.embedder_provider
+        or stored.get("model") != args.embedder_model
+    )
+    if embedder_changed:
+        _emit("embedder_changed", was=stored, now={
+            "provider": args.embedder_provider, "model": args.embedder_model,
+        })
+
+    started = time.time()
+    chunks_added = 0
+    files_processed = 0
+    skipped = 0
+    files_failed = []
+
+    # Walk the folder — only top-level + one level deep. Going arbitrarily
+    # deep would silently include things like venv/.git which would be
+    # surprising. If someone needs recursive, they can manually flatten.
+    candidates = [p for p in sorted(folder.iterdir()) if p.is_file()]
+    candidates += [p for sub in folder.iterdir() if sub.is_dir() and not sub.name.startswith(".")
+                   for p in sorted(sub.iterdir()) if p.is_file()]
+
+    total = sum(1 for p in candidates if p.suffix.lower() in SUPPORTED_EXTS)
+    _emit("scanned", folder=str(folder), total_supported=total)
+
+    for f in candidates:
+        ext = f.suffix.lower()
+        if ext not in SUPPORTED_EXTS:
+            continue
+        # macOS AppleDouble resource forks (._foo.docx, ._.DS_Store, …) —
+        # they're not real documents and every extractor will choke on them.
+        if f.name.startswith("._") or f.name == ".DS_Store":
+            continue
+        try:
+            raw = f.read_bytes()
+        except Exception as exc:
+            files_failed.append({"file": str(f), "error": f"read: {type(exc).__name__}: {exc}"})
+            continue
+        sha = hashlib.sha256(raw).hexdigest()
+        prev = manifest["files"].get(f.name, {})
+        if (not embedder_changed) and prev.get("sha256") == sha:
+            skipped += 1
+            _emit("skipped_sha_clean", file=f.name)
+            continue
+        reader_cls = _reader_for(ext)
+        if reader_cls is None:
+            files_failed.append({"file": str(f), "error": "no extractor"})
+            continue
+        try:
+            reader = reader_cls()
+            docs = await reader(str(f))
+            for doc in docs:
+                if hasattr(doc.metadata, "doc_id"):
+                    doc.metadata.doc_id = f.name
+            await kb.add_documents(docs)
+            chunks_added += len(docs)
+            manifest["files"][f.name] = {
+                "sha256": sha, "chunks": len(docs),
+                "ingested_at": _dt.datetime.utcnow().isoformat() + "Z",
+            }
+            files_processed += 1
+            _emit("file_done", file=f.name, chunks=len(docs))
+        except Exception as exc:
+            files_failed.append({
+                "file": str(f), "error": f"{type(exc).__name__}: {exc}",
+            })
+            _emit("file_failed", file=f.name, error=str(exc))
+
+    manifest["embedder"] = {
+        "provider": args.embedder_provider, "model": args.embedder_model,
+    }
+    manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
+
+    elapsed_ms = int((time.time() - started) * 1000)
+    _emit("done",
+          files_processed=files_processed,
+          chunks_added=chunks_added,
+          skipped=skipped,
+          embedder_changed=embedder_changed,
+          elapsed_ms=elapsed_ms,
+          errors=files_failed)
+    return 0
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--pipeline-name", required=True)
+    ap.add_argument("--folder", required=True)
+    ap.add_argument("--embedder-provider", required=True, choices=["ollama", "lmstudio"])
+    ap.add_argument("--embedder-model", required=True)
+    args = ap.parse_args()
+    sys.exit(asyncio.run(_ingest(args)))
+
+
+if __name__ == "__main__":
+    main()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.0.37",
+  "version": "1.0.39",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,
@@ -12,10 +12,12 @@
   "files": [
     "dist/**/*.js",
     "dist/**/*.d.ts",
+    "dist/**/*.py",
+    "localRagIngest.py",
     "README.md"
   ],
   "scripts": {
-    "build": "tsc",
+    "build": "tsc && node -e \"require('fs').copyFileSync('localRagIngest.py','dist/localRagIngest.py')\"",
     "prepublishOnly": "npm run build"
   },
   "dependencies": {