@melaya/runner 1.0.49 → 1.0.50

package/dist/connection.js

@@ -480,16 +480,25 @@ export async function connect(opts) {
         try {
             const prov = (payload.embedderProvider || "").toLowerCase();
             if (prov === "openai") {
-                const reason = ("Mode B (local folder) refuses cloud embedders. " +
-                    "Pick ollama or lmstudio to keep your documents private.");
+                const reason = ("Runner-side ingest refuses cloud embedders. " +
+                    "Pick ollama or lmstudio (the whole point of routing through " +
+                    "the runner is to use a local model).");
                 console.log(chalk.red(`  ✗ rag:ingest refused: ${reason}`));
                 socket.emit("rag:ingest-result", {
                     session_id: sid, ok: false, error: reason,
                 });
                 return;
             }
-            console.log(chalk.hex("#10B981")(`\n  🔒 RAG local-folder ingest: ${payload.pipelineName} ` +
-                `← ${payload.localFolderPath} via ${prov}:${payload.embedderModel}\n`));
+            const _isModeC = payload.mode === "cloud_files";
+            if (_isModeC) {
+                console.log(chalk.hex("#7C6FF0")(`\n  ☁️→💻 RAG cloud-files ingest (Mode C): ${payload.pipelineName} ` +
+                    `via ${prov}:${payload.embedderModel}\n` +
+                    `       (files fetched from ${opts.serverUrl}, embedded locally, vectors pushed back)\n`));
+            }
+            else {
+                console.log(chalk.hex("#10B981")(`\n  🔒 RAG local-folder ingest (Mode B): ${payload.pipelineName} ` +
+                    `← ${payload.localFolderPath} via ${prov}:${payload.embedderModel}\n`));
+            }
             emitProgress("started");
             // Reuse the same venv as runner:run — agentscope.rag + ollama +
             // qdrant-client land here.
@@ -562,13 +571,44 @@ export async function connect(opts) {
             const ragSslEnv = ragCertBundle
                 ? { SSL_CERT_FILE: ragCertBundle, REQUESTS_CA_BUNDLE: ragCertBundle }
                 : {};
-            const proc = spawn(env.pythonPath, [
+            const ingestMode = payload.mode === "cloud_files" ? "cloud_files" : "local_folder";
+            const ingestArgs = [
                 "-u", stagedScript,
+                "--mode", ingestMode,
                 "--pipeline-name", payload.pipelineName,
-                "--folder", payload.localFolderPath,
                 "--embedder-provider", prov,
                 "--embedder-model", payload.embedderModel,
-            ], {
+            ];
+            if (ingestMode === "cloud_files") {
+                // Server bundled file bytes into the event. Unpack them to a
+                // temp dir + a small manifest JSON so the Python script can
+                // walk them with the same code path as Mode B reads from disk.
+                const filesDir = join(workDir, "cloud-files");
+                mkdirSync(filesDir, { recursive: true });
+                const { writeFileSync } = await import("fs");
+                const filesArr = payload.cloudFiles || [];
+                for (const f of filesArr) {
+                    try {
+                        writeFileSync(join(filesDir, f.name), Buffer.from(f.contentB64, "base64"));
+                    }
+                    catch (e) {
+                        console.log(chalk.red(`  [rag] failed to unpack ${f.name}: ${e?.message || e}`));
+                    }
+                }
+                const manifestPath = join(workDir, "cloud-manifest.json");
+                writeFileSync(manifestPath, JSON.stringify({
+                    files: filesArr.map(f => ({ name: f.name, sha256: f.sha256, ext: f.ext })),
+                    stored_embedder: payload.cloudManifest?.stored_embedder || {},
+                    previously_ingested: payload.cloudManifest?.previously_ingested || {},
+                }, null, 2));
+                ingestArgs.push("--files-dir", filesDir);
+                ingestArgs.push("--manifest-json", manifestPath);
+                console.log(chalk.gray(`  [rag] unpacked ${filesArr.length} cloud file(s) → ${filesDir}`));
+            }
+            else {
+                ingestArgs.push("--folder", payload.localFolderPath || "");
+            }
+            const proc = spawn(env.pythonPath, ingestArgs, {
                 env: {
                     ...process.env,
                     PYTHONPATH: sharedDir, // so `from agentscope.rag import …` resolves

package/dist/localRagIngest.py

@@ -331,14 +331,245 @@ async def _ingest(args) -> int:
     return 0
 
 
+def _build_embedder(provider: str, model: str):
+    """Embedder factory shared between Mode B (local folder) and Mode C
+    (cloud files via HTTP). Returns the agentscope embedder instance +
+    a probe error message string when something is misconfigured (empty
+    string means OK)."""
+    if provider == "ollama":
+        from agentscope.embedding import OllamaTextEmbedding
+        DIMS = {
+            "nomic-embed-text": 768, "nomic-embed-text:v1.5": 768,
+            "mxbai-embed-large": 1024, "all-minilm": 384,
+            "snowflake-arctic-embed": 1024,
+        }
+        dim = DIMS.get(model.lower())
+        if dim is None:
+            return None, (
+                f"unknown ollama embedding model '{model}' — supported: {sorted(DIMS)}"
+            )
+        return OllamaTextEmbedding(
+            model_name=model, dimensions=dim, host=None,
+        ), ""
+    if provider == "lmstudio":
+        from agentscope.embedding import OpenAITextEmbedding
+        embedder = OpenAITextEmbedding(
+            api_key="lmstudio", model_name=model, dimensions=768,
+            base_url="http://localhost:1234/v1",
+        )
+        # Strip OpenAI-specific `dimensions` kwarg that LMS rejects
+        _orig_create = embedder.client.embeddings.create
+        async def _lms_create(**kw):
+            kw.pop("dimensions", None)
+            return await _orig_create(**kw)
+        embedder.client.embeddings.create = _lms_create  # type: ignore[assignment]
+        return embedder, ""
+    return None, f"unsupported embedder provider '{provider}'"
+
+
+async def _ingest_cloud_files(args) -> int:
+    """Mode C ingest.
+
+    Files originate on prod but the runner does the embedding so the
+    operator can use a free local model (ollama / lmstudio) without
+    needing an OpenAI key. Both the file CONTENTS and the resulting
+    (chunks, vectors) tuples flow over Socket.IO end-to-end:
+
+      • The server (tRPC bridge in runnerNamespace.ts) reads the files
+        from prod's <pipeline_dir>/rag/retrieval/, base64-encodes them,
+        and bakes them into the `rag:ingest` event payload. The runner
+        connection.ts unpacks them into ``<tmp>/files-dir/`` and
+        passes ``--files-dir <tmp> --manifest-json <tmp>/manifest.json``
+        here.
+      • For each file we parse + chunk + embed locally, then emit a
+        ``progress: {"kind":"upsert_chunks", ...}`` line to stdout. The
+        runner forwards that line as a ``rag:ingest-progress`` Socket.IO
+        event; the server's progress handler recognises the
+        ``upsert_chunks`` kind and POSTs the payload to
+        ``http://127.0.0.1:8766/pipelines/{name}/docs/retrieval/upsert``
+        — the loopback-only builder API that owns the QdrantStore.
+      • Nothing on the wire ever needs to leave the user's machine
+        via HTTP. The whole transport is the already-authenticated
+        Socket.IO channel, which means we don't have to expose any
+        builder endpoints publicly.
+    """
+    files_dir = Path(args.files_dir)
+    if not files_dir.exists() or not files_dir.is_dir():
+        _emit("error", reason=f"files-dir not found: {files_dir}")
+        return 3
+
+    embedder, err = _build_embedder(args.embedder_provider, args.embedder_model)
+    if not embedder:
+        _emit("error", reason=err)
+        return 4
+
+    manifest: dict = {}
+    if args.manifest_json:
+        try:
+            manifest = json.loads(Path(args.manifest_json).read_text(encoding="utf-8"))
+        except Exception as exc:
+            _emit("warning", reason=f"manifest read failed: {exc}")
+    files: list[dict] = manifest.get("files") or []
+    stored_embedder = manifest.get("stored_embedder") or {}
+    previously_ingested = manifest.get("previously_ingested") or {}
+    embedder_changed = (
+        stored_embedder.get("provider") != args.embedder_provider
+        or stored_embedder.get("model") != args.embedder_model
+    )
+    _emit("started", mode="cloud_files", pipeline=args.pipeline_name,
+          embedder_changed=embedder_changed, total_supported=len(files))
+
+    started = time.time()
+    chunks_added_total = 0
+    files_processed = 0
+    skipped = 0
+    files_failed: list[dict] = []
+
+    embedder_payload = {
+        "provider": args.embedder_provider,
+        "model": args.embedder_model,
+        "dimensions": embedder.dimensions,
+    }
+
+    first_emit = True
+    for entry in files:
+        name = entry["name"]
+        sha = entry["sha256"]
+        ext = (entry.get("ext") or "").lower()
+        if ext not in SUPPORTED_EXTS:
+            files_failed.append({"file": name, "error": f"unsupported ext '{ext}'"})
+            _emit("file_failed", file=name, error=f"unsupported ext '{ext}'")
+            continue
+        if not embedder_changed and previously_ingested.get(name, {}).get("sha256") == sha:
+            skipped += 1
+            _emit("skipped_sha_clean", file=name)
+            continue
+        tmp_path = files_dir / name
+        if not tmp_path.is_file():
+            files_failed.append({"file": name, "error": "file missing from cloud-files dir"})
+            _emit("file_failed", file=name, error="missing locally")
+            continue
+        reader_cls = _reader_for(ext)
+        if reader_cls is None:
+            files_failed.append({"file": name, "error": "no extractor"})
+            _emit("file_failed", file=name, error="no extractor")
+            continue
+        try:
+            reader = reader_cls()
+            docs = await reader(str(tmp_path))
+            for i, doc in enumerate(docs):
+                if hasattr(doc.metadata, "doc_id"):
+                    doc.metadata.doc_id = name
+                if hasattr(doc.metadata, "chunk_id"):
+                    doc.metadata.chunk_id = i
+            texts: list[str] = []
+            for doc in docs:
+                content = getattr(doc.metadata, "content", None)
+                text = (
+                    (content or {}).get("text", "")
+                    if isinstance(content, dict)
+                    else getattr(content, "text", "")
+                )
+                if text:
+                    texts.append(text)
+            if not texts:
+                files_processed += 1
+                _emit("file_done", file=name, chunks=0)
+                _emit("upsert_chunks",
+                      embedder=embedder_payload,
+                      chunks=[],
+                      files_done=[{"name": name, "sha256": sha, "chunks": 0}],
+                      embedder_changed=embedder_changed and first_emit,
+                      finalize=False)
+                first_emit = False
+                continue
+            vectors_resp = await embedder(text=texts)
+            vectors = [eb.embedding for eb in vectors_resp.embeddings]
+            if len(vectors) != len(texts):
+                raise RuntimeError(
+                    f"embedder returned {len(vectors)} vectors for {len(texts)} chunks"
+                )
+            upsert_chunks = [
+                {
+                    "text": text,
+                    "vector": [float(v) for v in vec],
+                    "metadata": {
+                        "doc_id": name,
+                        "chunk_id": i,
+                        "total_chunks": len(texts),
+                    },
+                }
+                for i, (text, vec) in enumerate(zip(texts, vectors))
+            ]
+            # Emit one upsert event per file. Server forwards to builder
+            # /upsert. For small per-file payloads this is well below the
+            # Socket.IO default 1 MB/event limit (60-doc corpora ≈ 6 MB
+            # spread across ~60 events).
+            _emit("upsert_chunks",
+                  embedder=embedder_payload,
+                  chunks=upsert_chunks,
+                  files_done=[{"name": name, "sha256": sha, "chunks": len(texts)}],
+                  embedder_changed=embedder_changed and first_emit,
+                  finalize=False)
+            first_emit = False
+            chunks_added_total += len(texts)
+            files_processed += 1
+            _emit("file_done", file=name, chunks=len(texts))
+        except Exception as exc:
+            files_failed.append({
+                "file": name, "error": f"{type(exc).__name__}: {exc}",
+            })
+            _emit("file_failed", file=name, error=str(exc))
+
+    # Finalize: tells the server to stamp the embedder identity into the
+    # manifest. Sent as one final, empty-chunk upsert event.
+    _emit("upsert_chunks",
+          embedder=embedder_payload,
+          chunks=[],
+          files_done=[],
+          embedder_changed=False,
+          finalize=True)
+
+    elapsed_ms = int((time.time() - started) * 1000)
+    _emit("done",
+          mode="cloud_files",
+          files_processed=files_processed,
+          chunks_added=chunks_added_total,
+          skipped=skipped,
+          embedder_changed=embedder_changed,
+          elapsed_ms=elapsed_ms,
+          errors=files_failed)
+    return 0
+
+
 def main() -> None:
     ap = argparse.ArgumentParser()
+    ap.add_argument("--mode", choices=["local_folder", "cloud_files"],
+                    default="local_folder")
     ap.add_argument("--pipeline-name", required=True)
-    ap.add_argument("--folder", required=True)
+    ap.add_argument("--folder", default="")
     ap.add_argument("--embedder-provider", required=True, choices=["ollama", "lmstudio"])
     ap.add_argument("--embedder-model", required=True)
+    # Mode C only (cloud_files): files were base64-encoded by the
+    # server into the rag:ingest payload and the runner's connection.ts
+    # unpacked them to disk at --files-dir. --manifest-json points at
+    # a small JSON manifest (file list + sha + previously-ingested map
+    # so we can skip unchanged files).
+    ap.add_argument("--files-dir", default="",
+                    help="Mode C only — directory containing the unpacked file payload.")
+    ap.add_argument("--manifest-json", default="",
+                    help="Mode C only — path to manifest JSON (file list, prior ingest, embedder).")
     args = ap.parse_args()
-    sys.exit(asyncio.run(_ingest(args)))
+    if args.mode == "cloud_files":
+        if not args.files_dir or not args.manifest_json:
+            _emit("error", reason="cloud_files mode requires --files-dir + --manifest-json")
+            sys.exit(2)
+        sys.exit(asyncio.run(_ingest_cloud_files(args)))
+    else:
+        if not args.folder:
+            _emit("error", reason="local_folder mode requires --folder")
+            sys.exit(2)
+        sys.exit(asyncio.run(_ingest(args)))
 
 
 if __name__ == "__main__":

package/localRagIngest.py

@@ -331,14 +331,245 @@ async def _ingest(args) -> int:
     return 0
 
 
+def _build_embedder(provider: str, model: str):
+    """Embedder factory shared between Mode B (local folder) and Mode C
+    (cloud files via HTTP). Returns the agentscope embedder instance +
+    a probe error message string when something is misconfigured (empty
+    string means OK)."""
+    if provider == "ollama":
+        from agentscope.embedding import OllamaTextEmbedding
+        DIMS = {
+            "nomic-embed-text": 768, "nomic-embed-text:v1.5": 768,
+            "mxbai-embed-large": 1024, "all-minilm": 384,
+            "snowflake-arctic-embed": 1024,
+        }
+        dim = DIMS.get(model.lower())
+        if dim is None:
+            return None, (
+                f"unknown ollama embedding model '{model}' — supported: {sorted(DIMS)}"
+            )
+        return OllamaTextEmbedding(
+            model_name=model, dimensions=dim, host=None,
+        ), ""
+    if provider == "lmstudio":
+        from agentscope.embedding import OpenAITextEmbedding
+        embedder = OpenAITextEmbedding(
+            api_key="lmstudio", model_name=model, dimensions=768,
+            base_url="http://localhost:1234/v1",
+        )
+        # Strip OpenAI-specific `dimensions` kwarg that LMS rejects
+        _orig_create = embedder.client.embeddings.create
+        async def _lms_create(**kw):
+            kw.pop("dimensions", None)
+            return await _orig_create(**kw)
+        embedder.client.embeddings.create = _lms_create  # type: ignore[assignment]
+        return embedder, ""
+    return None, f"unsupported embedder provider '{provider}'"
+
+
+async def _ingest_cloud_files(args) -> int:
+    """Mode C ingest.
+
+    Files originate on prod but the runner does the embedding so the
+    operator can use a free local model (ollama / lmstudio) without
+    needing an OpenAI key. Both the file CONTENTS and the resulting
+    (chunks, vectors) tuples flow over Socket.IO end-to-end:
+
+      • The server (tRPC bridge in runnerNamespace.ts) reads the files
+        from prod's <pipeline_dir>/rag/retrieval/, base64-encodes them,
+        and bakes them into the `rag:ingest` event payload. The runner
+        connection.ts unpacks them into ``<tmp>/files-dir/`` and
+        passes ``--files-dir <tmp> --manifest-json <tmp>/manifest.json``
+        here.
+      • For each file we parse + chunk + embed locally, then emit a
+        ``progress: {"kind":"upsert_chunks", ...}`` line to stdout. The
+        runner forwards that line as a ``rag:ingest-progress`` Socket.IO
+        event; the server's progress handler recognises the
+        ``upsert_chunks`` kind and POSTs the payload to
+        ``http://127.0.0.1:8766/pipelines/{name}/docs/retrieval/upsert``
+        — the loopback-only builder API that owns the QdrantStore.
+      • Nothing on the wire ever needs to leave the user's machine
+        via HTTP. The whole transport is the already-authenticated
+        Socket.IO channel, which means we don't have to expose any
+        builder endpoints publicly.
+    """
+    files_dir = Path(args.files_dir)
+    if not files_dir.exists() or not files_dir.is_dir():
+        _emit("error", reason=f"files-dir not found: {files_dir}")
+        return 3
+
+    embedder, err = _build_embedder(args.embedder_provider, args.embedder_model)
+    if not embedder:
+        _emit("error", reason=err)
+        return 4
+
+    manifest: dict = {}
+    if args.manifest_json:
+        try:
+            manifest = json.loads(Path(args.manifest_json).read_text(encoding="utf-8"))
+        except Exception as exc:
+            _emit("warning", reason=f"manifest read failed: {exc}")
+    files: list[dict] = manifest.get("files") or []
+    stored_embedder = manifest.get("stored_embedder") or {}
+    previously_ingested = manifest.get("previously_ingested") or {}
+    embedder_changed = (
+        stored_embedder.get("provider") != args.embedder_provider
+        or stored_embedder.get("model") != args.embedder_model
+    )
+    _emit("started", mode="cloud_files", pipeline=args.pipeline_name,
+          embedder_changed=embedder_changed, total_supported=len(files))
+
+    started = time.time()
+    chunks_added_total = 0
+    files_processed = 0
+    skipped = 0
+    files_failed: list[dict] = []
+
+    embedder_payload = {
+        "provider": args.embedder_provider,
+        "model": args.embedder_model,
+        "dimensions": embedder.dimensions,
+    }
+
+    first_emit = True
+    for entry in files:
+        name = entry["name"]
+        sha = entry["sha256"]
+        ext = (entry.get("ext") or "").lower()
+        if ext not in SUPPORTED_EXTS:
+            files_failed.append({"file": name, "error": f"unsupported ext '{ext}'"})
+            _emit("file_failed", file=name, error=f"unsupported ext '{ext}'")
+            continue
+        if not embedder_changed and previously_ingested.get(name, {}).get("sha256") == sha:
+            skipped += 1
+            _emit("skipped_sha_clean", file=name)
+            continue
+        tmp_path = files_dir / name
+        if not tmp_path.is_file():
+            files_failed.append({"file": name, "error": "file missing from cloud-files dir"})
+            _emit("file_failed", file=name, error="missing locally")
+            continue
+        reader_cls = _reader_for(ext)
+        if reader_cls is None:
+            files_failed.append({"file": name, "error": "no extractor"})
+            _emit("file_failed", file=name, error="no extractor")
+            continue
+        try:
+            reader = reader_cls()
+            docs = await reader(str(tmp_path))
+            for i, doc in enumerate(docs):
+                if hasattr(doc.metadata, "doc_id"):
+                    doc.metadata.doc_id = name
+                if hasattr(doc.metadata, "chunk_id"):
+                    doc.metadata.chunk_id = i
+            texts: list[str] = []
+            for doc in docs:
+                content = getattr(doc.metadata, "content", None)
+                text = (
+                    (content or {}).get("text", "")
+                    if isinstance(content, dict)
+                    else getattr(content, "text", "")
+                )
+                if text:
+                    texts.append(text)
+            if not texts:
+                files_processed += 1
+                _emit("file_done", file=name, chunks=0)
+                _emit("upsert_chunks",
+                      embedder=embedder_payload,
+                      chunks=[],
+                      files_done=[{"name": name, "sha256": sha, "chunks": 0}],
+                      embedder_changed=embedder_changed and first_emit,
+                      finalize=False)
+                first_emit = False
+                continue
+            vectors_resp = await embedder(text=texts)
+            vectors = [eb.embedding for eb in vectors_resp.embeddings]
+            if len(vectors) != len(texts):
+                raise RuntimeError(
+                    f"embedder returned {len(vectors)} vectors for {len(texts)} chunks"
+                )
+            upsert_chunks = [
+                {
+                    "text": text,
+                    "vector": [float(v) for v in vec],
+                    "metadata": {
+                        "doc_id": name,
+                        "chunk_id": i,
+                        "total_chunks": len(texts),
+                    },
+                }
+                for i, (text, vec) in enumerate(zip(texts, vectors))
+            ]
+            # Emit one upsert event per file. Server forwards to builder
+            # /upsert. For small per-file payloads this is well below the
+            # Socket.IO default 1 MB/event limit (60-doc corpora ≈ 6 MB
+            # spread across ~60 events).
+            _emit("upsert_chunks",
+                  embedder=embedder_payload,
+                  chunks=upsert_chunks,
+                  files_done=[{"name": name, "sha256": sha, "chunks": len(texts)}],
+                  embedder_changed=embedder_changed and first_emit,
+                  finalize=False)
+            first_emit = False
+            chunks_added_total += len(texts)
+            files_processed += 1
+            _emit("file_done", file=name, chunks=len(texts))
+        except Exception as exc:
+            files_failed.append({
+                "file": name, "error": f"{type(exc).__name__}: {exc}",
+            })
+            _emit("file_failed", file=name, error=str(exc))
+
+    # Finalize: tells the server to stamp the embedder identity into the
+    # manifest. Sent as one final, empty-chunk upsert event.
+    _emit("upsert_chunks",
+          embedder=embedder_payload,
+          chunks=[],
+          files_done=[],
+          embedder_changed=False,
+          finalize=True)
+
+    elapsed_ms = int((time.time() - started) * 1000)
+    _emit("done",
+          mode="cloud_files",
+          files_processed=files_processed,
+          chunks_added=chunks_added_total,
+          skipped=skipped,
+          embedder_changed=embedder_changed,
+          elapsed_ms=elapsed_ms,
+          errors=files_failed)
+    return 0
+
+
 def main() -> None:
     ap = argparse.ArgumentParser()
+    ap.add_argument("--mode", choices=["local_folder", "cloud_files"],
+                    default="local_folder")
     ap.add_argument("--pipeline-name", required=True)
-    ap.add_argument("--folder", required=True)
+    ap.add_argument("--folder", default="")
     ap.add_argument("--embedder-provider", required=True, choices=["ollama", "lmstudio"])
     ap.add_argument("--embedder-model", required=True)
+    # Mode C only (cloud_files): files were base64-encoded by the
+    # server into the rag:ingest payload and the runner's connection.ts
+    # unpacked them to disk at --files-dir. --manifest-json points at
+    # a small JSON manifest (file list + sha + previously-ingested map
+    # so we can skip unchanged files).
+    ap.add_argument("--files-dir", default="",
+                    help="Mode C only — directory containing the unpacked file payload.")
+    ap.add_argument("--manifest-json", default="",
+                    help="Mode C only — path to manifest JSON (file list, prior ingest, embedder).")
     args = ap.parse_args()
-    sys.exit(asyncio.run(_ingest(args)))
+    if args.mode == "cloud_files":
+        if not args.files_dir or not args.manifest_json:
+            _emit("error", reason="cloud_files mode requires --files-dir + --manifest-json")
+            sys.exit(2)
+        sys.exit(asyncio.run(_ingest_cloud_files(args)))
+    else:
+        if not args.folder:
+            _emit("error", reason="local_folder mode requires --folder")
+            sys.exit(2)
+        sys.exit(asyncio.run(_ingest(args)))
 
 
 if __name__ == "__main__":

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.0.49",
+  "version": "1.0.50",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,