@melaya/runner 1.0.48 → 1.0.50

package/dist/connection.js

@@ -229,11 +229,24 @@ export async function connect(opts) {
                 if (k.startsWith("MEL_MODEL_"))
                     delete inherited[k];
             }
+            // certifi CA-bundle path resolved at venv setup. Set as both
+            // SSL_CERT_FILE (urllib / openssl) and REQUESTS_CA_BUNDLE (requests
+            // library) so EVERY HTTPS-using tool — gmail_send, slack_post_text,
+            // anthropic SDK, openai SDK, web_search, etc. — verifies against
+            // certifi's bundle rather than the platform default. Without this,
+            // macOS Python crashes with `[SSL: CERTIFICATE_VERIFY_FAILED]` on
+            // any cloud call (run eb1b8f67df56491b hit exactly this for
+            // gmail_send → api.googleapis.com).
+            const certBundle = (await import("./pythonEnv.js")).getCertBundlePath();
+            const sslEnv = certBundle
+                ? { SSL_CERT_FILE: certBundle, REQUESTS_CA_BUNDLE: certBundle }
+                : {};
             const env = {
                 ...inherited,
                 PYTHONPATH: sharedDir,
                 PYTHONIOENCODING: "utf-8",
                 PYTHONUTF8: "1",
+                ...sslEnv,
                 MEL_RUN_ID: payload.runId,
                 // Point agentscope hooks at the REAL server (not the browser's Vite URL).
                 // Convert wss://api.melaya.org → https://api.melaya.org so pushMessage,
@@ -467,16 +480,25 @@ export async function connect(opts) {
         try {
             const prov = (payload.embedderProvider || "").toLowerCase();
             if (prov === "openai") {
-                const reason = ("Mode B (local folder) refuses cloud embedders. " +
-                    "Pick ollama or lmstudio to keep your documents private.");
+                const reason = ("Runner-side ingest refuses cloud embedders. " +
+                    "Pick ollama or lmstudio (the whole point of routing through " +
+                    "the runner is to use a local model).");
                 console.log(chalk.red(`  ✗ rag:ingest refused: ${reason}`));
                 socket.emit("rag:ingest-result", {
                     session_id: sid, ok: false, error: reason,
                 });
                 return;
             }
-            console.log(chalk.hex("#10B981")(`\n  🔒 RAG local-folder ingest: ${payload.pipelineName} ` +
-                `← ${payload.localFolderPath} via ${prov}:${payload.embedderModel}\n`));
+            const _isModeC = payload.mode === "cloud_files";
+            if (_isModeC) {
+                console.log(chalk.hex("#7C6FF0")(`\n  ☁️→💻 RAG cloud-files ingest (Mode C): ${payload.pipelineName} ` +
+                    `via ${prov}:${payload.embedderModel}\n` +
+                    `       (files fetched from ${opts.serverUrl}, embedded locally, vectors pushed back)\n`));
+            }
+            else {
+                console.log(chalk.hex("#10B981")(`\n  🔒 RAG local-folder ingest (Mode B): ${payload.pipelineName} ` +
+                    `← ${payload.localFolderPath} via ${prov}:${payload.embedderModel}\n`));
+            }
             emitProgress("started");
             // Reuse the same venv as runner:run — agentscope.rag + ollama +
             // qdrant-client land here.
@@ -542,16 +564,55 @@ export async function connect(opts) {
             ];
             const { existsSync: _exists2 } = await import("fs");
             const bundledNltkData = candidateNltkPaths.find((p) => _exists2(p)) || "";
-            const proc = spawn(env.pythonPath, [
+            // Resolve certifi bundle for HTTPS verification (same rationale as
+            // the runner:run spawn — embedder providers like OpenAI hit
+            // api.openai.com over TLS and crash on macOS without certifi).
+            const ragCertBundle = (await import("./pythonEnv.js")).getCertBundlePath();
+            const ragSslEnv = ragCertBundle
+                ? { SSL_CERT_FILE: ragCertBundle, REQUESTS_CA_BUNDLE: ragCertBundle }
+                : {};
+            const ingestMode = payload.mode === "cloud_files" ? "cloud_files" : "local_folder";
+            const ingestArgs = [
                 "-u", stagedScript,
+                "--mode", ingestMode,
                 "--pipeline-name", payload.pipelineName,
-                "--folder", payload.localFolderPath,
                 "--embedder-provider", prov,
                 "--embedder-model", payload.embedderModel,
-            ], {
+            ];
+            if (ingestMode === "cloud_files") {
+                // Server bundled file bytes into the event. Unpack them to a
+                // temp dir + a small manifest JSON so the Python script can
+                // walk them with the same code path as Mode B reads from disk.
+                const filesDir = join(workDir, "cloud-files");
+                mkdirSync(filesDir, { recursive: true });
+                const { writeFileSync } = await import("fs");
+                const filesArr = payload.cloudFiles || [];
+                for (const f of filesArr) {
+                    try {
+                        writeFileSync(join(filesDir, f.name), Buffer.from(f.contentB64, "base64"));
+                    }
+                    catch (e) {
+                        console.log(chalk.red(`  [rag] failed to unpack ${f.name}: ${e?.message || e}`));
+                    }
+                }
+                const manifestPath = join(workDir, "cloud-manifest.json");
+                writeFileSync(manifestPath, JSON.stringify({
+                    files: filesArr.map(f => ({ name: f.name, sha256: f.sha256, ext: f.ext })),
+                    stored_embedder: payload.cloudManifest?.stored_embedder || {},
+                    previously_ingested: payload.cloudManifest?.previously_ingested || {},
+                }, null, 2));
+                ingestArgs.push("--files-dir", filesDir);
+                ingestArgs.push("--manifest-json", manifestPath);
+                console.log(chalk.gray(`  [rag] unpacked ${filesArr.length} cloud file(s) → ${filesDir}`));
+            }
+            else {
+                ingestArgs.push("--folder", payload.localFolderPath || "");
+            }
+            const proc = spawn(env.pythonPath, ingestArgs, {
                 env: {
                     ...process.env,
                     PYTHONPATH: sharedDir, // so `from agentscope.rag import …` resolves
+                    ...ragSslEnv,
                     ...(bundledNltkData ? { NLTK_DATA: bundledNltkData } : {}),
                 },
                 cwd: workDir,

package/dist/localRagIngest.py

@@ -331,14 +331,245 @@ async def _ingest(args) -> int:
     return 0
 
 
+def _build_embedder(provider: str, model: str):
+    """Embedder factory shared between Mode B (local folder) and Mode C
+    (cloud files via HTTP). Returns the agentscope embedder instance +
+    a probe error message string when something is misconfigured (empty
+    string means OK)."""
+    if provider == "ollama":
+        from agentscope.embedding import OllamaTextEmbedding
+        DIMS = {
+            "nomic-embed-text": 768, "nomic-embed-text:v1.5": 768,
+            "mxbai-embed-large": 1024, "all-minilm": 384,
+            "snowflake-arctic-embed": 1024,
+        }
+        dim = DIMS.get(model.lower())
+        if dim is None:
+            return None, (
+                f"unknown ollama embedding model '{model}' — supported: {sorted(DIMS)}"
+            )
+        return OllamaTextEmbedding(
+            model_name=model, dimensions=dim, host=None,
+        ), ""
+    if provider == "lmstudio":
+        from agentscope.embedding import OpenAITextEmbedding
+        embedder = OpenAITextEmbedding(
+            api_key="lmstudio", model_name=model, dimensions=768,
+            base_url="http://localhost:1234/v1",
+        )
+        # Strip OpenAI-specific `dimensions` kwarg that LMS rejects
+        _orig_create = embedder.client.embeddings.create
+        async def _lms_create(**kw):
+            kw.pop("dimensions", None)
+            return await _orig_create(**kw)
+        embedder.client.embeddings.create = _lms_create  # type: ignore[assignment]
+        return embedder, ""
+    return None, f"unsupported embedder provider '{provider}'"
+
+
+async def _ingest_cloud_files(args) -> int:
+    """Mode C ingest.
+
+    Files originate on prod but the runner does the embedding so the
+    operator can use a free local model (ollama / lmstudio) without
+    needing an OpenAI key. Both the file CONTENTS and the resulting
+    (chunks, vectors) tuples flow over Socket.IO end-to-end:
+
+      • The server (tRPC bridge in runnerNamespace.ts) reads the files
+        from prod's <pipeline_dir>/rag/retrieval/, base64-encodes them,
+        and bakes them into the `rag:ingest` event payload. The runner
+        connection.ts unpacks them into ``<tmp>/files-dir/`` and
+        passes ``--files-dir <tmp> --manifest-json <tmp>/manifest.json``
+        here.
+      • For each file we parse + chunk + embed locally, then emit a
+        ``progress: {"kind":"upsert_chunks", ...}`` line to stdout. The
+        runner forwards that line as a ``rag:ingest-progress`` Socket.IO
+        event; the server's progress handler recognises the
+        ``upsert_chunks`` kind and POSTs the payload to
+        ``http://127.0.0.1:8766/pipelines/{name}/docs/retrieval/upsert``
+        — the loopback-only builder API that owns the QdrantStore.
+      • Nothing on the wire ever needs to leave the user's machine
+        via HTTP. The whole transport is the already-authenticated
+        Socket.IO channel, which means we don't have to expose any
+        builder endpoints publicly.
+    """
+    files_dir = Path(args.files_dir)
+    if not files_dir.exists() or not files_dir.is_dir():
+        _emit("error", reason=f"files-dir not found: {files_dir}")
+        return 3
+
+    embedder, err = _build_embedder(args.embedder_provider, args.embedder_model)
+    if not embedder:
+        _emit("error", reason=err)
+        return 4
+
+    manifest: dict = {}
+    if args.manifest_json:
+        try:
+            manifest = json.loads(Path(args.manifest_json).read_text(encoding="utf-8"))
+        except Exception as exc:
+            _emit("warning", reason=f"manifest read failed: {exc}")
+    files: list[dict] = manifest.get("files") or []
+    stored_embedder = manifest.get("stored_embedder") or {}
+    previously_ingested = manifest.get("previously_ingested") or {}
+    embedder_changed = (
+        stored_embedder.get("provider") != args.embedder_provider
+        or stored_embedder.get("model") != args.embedder_model
+    )
+    _emit("started", mode="cloud_files", pipeline=args.pipeline_name,
+          embedder_changed=embedder_changed, total_supported=len(files))
+
+    started = time.time()
+    chunks_added_total = 0
+    files_processed = 0
+    skipped = 0
+    files_failed: list[dict] = []
+
+    embedder_payload = {
+        "provider": args.embedder_provider,
+        "model": args.embedder_model,
+        "dimensions": embedder.dimensions,
+    }
+
+    first_emit = True
+    for entry in files:
+        name = entry["name"]
+        sha = entry["sha256"]
+        ext = (entry.get("ext") or "").lower()
+        if ext not in SUPPORTED_EXTS:
+            files_failed.append({"file": name, "error": f"unsupported ext '{ext}'"})
+            _emit("file_failed", file=name, error=f"unsupported ext '{ext}'")
+            continue
+        if not embedder_changed and previously_ingested.get(name, {}).get("sha256") == sha:
+            skipped += 1
+            _emit("skipped_sha_clean", file=name)
+            continue
+        tmp_path = files_dir / name
+        if not tmp_path.is_file():
+            files_failed.append({"file": name, "error": "file missing from cloud-files dir"})
+            _emit("file_failed", file=name, error="missing locally")
+            continue
+        reader_cls = _reader_for(ext)
+        if reader_cls is None:
+            files_failed.append({"file": name, "error": "no extractor"})
+            _emit("file_failed", file=name, error="no extractor")
+            continue
+        try:
+            reader = reader_cls()
+            docs = await reader(str(tmp_path))
+            for i, doc in enumerate(docs):
+                if hasattr(doc.metadata, "doc_id"):
+                    doc.metadata.doc_id = name
+                if hasattr(doc.metadata, "chunk_id"):
+                    doc.metadata.chunk_id = i
+            texts: list[str] = []
+            for doc in docs:
+                content = getattr(doc.metadata, "content", None)
+                text = (
+                    (content or {}).get("text", "")
+                    if isinstance(content, dict)
+                    else getattr(content, "text", "")
+                )
+                if text:
+                    texts.append(text)
+            if not texts:
+                files_processed += 1
+                _emit("file_done", file=name, chunks=0)
+                _emit("upsert_chunks",
+                      embedder=embedder_payload,
+                      chunks=[],
+                      files_done=[{"name": name, "sha256": sha, "chunks": 0}],
+                      embedder_changed=embedder_changed and first_emit,
+                      finalize=False)
+                first_emit = False
+                continue
+            vectors_resp = await embedder(text=texts)
+            vectors = [eb.embedding for eb in vectors_resp.embeddings]
+            if len(vectors) != len(texts):
+                raise RuntimeError(
+                    f"embedder returned {len(vectors)} vectors for {len(texts)} chunks"
+                )
+            upsert_chunks = [
+                {
+                    "text": text,
+                    "vector": [float(v) for v in vec],
+                    "metadata": {
+                        "doc_id": name,
+                        "chunk_id": i,
+                        "total_chunks": len(texts),
+                    },
+                }
+                for i, (text, vec) in enumerate(zip(texts, vectors))
+            ]
+            # Emit one upsert event per file. Server forwards to builder
+            # /upsert. For small per-file payloads this is well below the
+            # Socket.IO default 1 MB/event limit (60-doc corpora ≈ 6 MB
+            # spread across ~60 events).
+            _emit("upsert_chunks",
+                  embedder=embedder_payload,
+                  chunks=upsert_chunks,
+                  files_done=[{"name": name, "sha256": sha, "chunks": len(texts)}],
+                  embedder_changed=embedder_changed and first_emit,
+                  finalize=False)
+            first_emit = False
+            chunks_added_total += len(texts)
+            files_processed += 1
+            _emit("file_done", file=name, chunks=len(texts))
+        except Exception as exc:
+            files_failed.append({
+                "file": name, "error": f"{type(exc).__name__}: {exc}",
+            })
+            _emit("file_failed", file=name, error=str(exc))
+
+    # Finalize: tells the server to stamp the embedder identity into the
+    # manifest. Sent as one final, empty-chunk upsert event.
+    _emit("upsert_chunks",
+          embedder=embedder_payload,
+          chunks=[],
+          files_done=[],
+          embedder_changed=False,
+          finalize=True)
+
+    elapsed_ms = int((time.time() - started) * 1000)
+    _emit("done",
+          mode="cloud_files",
+          files_processed=files_processed,
+          chunks_added=chunks_added_total,
+          skipped=skipped,
+          embedder_changed=embedder_changed,
+          elapsed_ms=elapsed_ms,
+          errors=files_failed)
+    return 0
+
+
 def main() -> None:
     ap = argparse.ArgumentParser()
+    ap.add_argument("--mode", choices=["local_folder", "cloud_files"],
+                    default="local_folder")
     ap.add_argument("--pipeline-name", required=True)
-    ap.add_argument("--folder", required=True)
+    ap.add_argument("--folder", default="")
     ap.add_argument("--embedder-provider", required=True, choices=["ollama", "lmstudio"])
     ap.add_argument("--embedder-model", required=True)
+    # Mode C only (cloud_files): files were base64-encoded by the
+    # server into the rag:ingest payload and the runner's connection.ts
+    # unpacked them to disk at --files-dir. --manifest-json points at
+    # a small JSON manifest (file list + sha + previously-ingested map
+    # so we can skip unchanged files).
+    ap.add_argument("--files-dir", default="",
+                    help="Mode C only — directory containing the unpacked file payload.")
+    ap.add_argument("--manifest-json", default="",
+                    help="Mode C only — path to manifest JSON (file list, prior ingest, embedder).")
     args = ap.parse_args()
-    sys.exit(asyncio.run(_ingest(args)))
+    if args.mode == "cloud_files":
+        if not args.files_dir or not args.manifest_json:
+            _emit("error", reason="cloud_files mode requires --files-dir + --manifest-json")
+            sys.exit(2)
+        sys.exit(asyncio.run(_ingest_cloud_files(args)))
+    else:
+        if not args.folder:
+            _emit("error", reason="local_folder mode requires --folder")
+            sys.exit(2)
+        sys.exit(asyncio.run(_ingest(args)))
 
 
 if __name__ == "__main__":

package/dist/pythonEnv.d.ts

@@ -10,6 +10,7 @@
  * shared-bundle version. Re-bootstraps when the bundle version changes.
  */
 export declare function venvPython(): string;
+export declare function getCertBundlePath(): string;
 export declare function ensurePythonEnv(systemPython: string, expectedVersion: string, onProgress?: (msg: string) => void): Promise<{
     ok: boolean;
     pythonPath: string;

package/dist/pythonEnv.js

@@ -119,12 +119,69 @@ const PIP_DEPS = [
     // "pandas is not installed". Numpy is already in PIP_DEPS (line 57) so
     // the wheel install is fast.
     "pandas",
+    // certifi — provides the CA root bundle that tools like gmail_send /
+    // any other HTTPS API rely on for cert verification. On macOS Python
+    // 3.x doesn't read the system keychain by default; without certifi +
+    // SSL_CERT_FILE pointed at its bundle, every Gmail / Slack / Anthropic
+    // / etc. API call raises SSL: CERTIFICATE_VERIFY_FAILED (run
+    // eb1b8f67df56491b hit exactly this — gmail_send dispatched but
+    // urllib couldn't verify api.googleapis.com). The runner exports
+    // SSL_CERT_FILE + REQUESTS_CA_BUNDLE pointing here when spawning
+    // pipeline Python — see getCertBundlePath() + spawn env merging in
+    // connection.ts.
+    "certifi",
 ];
 export function venvPython() {
     return platform() === "win32"
         ? join(VENV_DIR, "Scripts", "python.exe")
         : join(VENV_DIR, "bin", "python");
 }
+// ── certifi CA-bundle path resolution + cache ────────────────────────────
+// Set as SSL_CERT_FILE + REQUESTS_CA_BUNDLE when spawning pipeline Python
+// so that ANY HTTPS-using tool (gmail_send, slack_post_text, anthropic /
+// openai SDKs, requests, urllib, etc.) verifies certs against certifi's
+// bundle rather than the platform default. On macOS specifically, the
+// system Python doesn't read the keychain — without this, every API call
+// crashes with `[SSL: CERTIFICATE_VERIFY_FAILED]`.
+//
+// We cache the path in ~/.melaya-runner/cert-bundle-path so we don't pay
+// a Python startup roundtrip on every spawn. The cache is invalidated
+// when the venv is rebuilt (the file lives outside VENV_DIR but is
+// re-written every ensurePythonEnv).
+const CERT_PATH_CACHE = join(CACHE_DIR, "cert-bundle-path");
+export function getCertBundlePath() {
+    try {
+        if (existsSync(CERT_PATH_CACHE)) {
+            const cached = readFileSync(CERT_PATH_CACHE, "utf-8").trim();
+            if (cached && existsSync(cached))
+                return cached;
+        }
+    }
+    catch { /* fall through to resolution */ }
+    return "";
+}
+async function _resolveAndCacheCertBundle(onProgress) {
+    if (!existsSync(venvPython()))
+        return;
+    // Resolve certifi.where() — fast subprocess (<300 ms typical).
+    let resolved = "";
+    await runProc(venvPython(), ["-c", "import certifi, sys; sys.stdout.write(certifi.where())"], (line) => {
+        if (line && existsSync(line.trim()))
+            resolved = line.trim();
+    });
+    if (resolved) {
+        try {
+            writeFileSync(CERT_PATH_CACHE, resolved + "\n", "utf-8");
+            onProgress(`cert bundle: ${resolved}`);
+        }
+        catch (e) {
+            onProgress(`(warn: failed to cache cert bundle path: ${e?.message})`);
+        }
+    }
+    else {
+        onProgress("(warn: certifi.where() did not resolve a valid path — HTTPS calls may fail with cert verify errors)");
+    }
+}
 // The marker is keyed on sharedVersion + a hash of PIP_DEPS so changes
 // to either invalidate the venv. Without the PIP_DEPS half, a runner
 // update that adds a dep (e.g. scrapling[fetchers] in 1.0.32) would
@@ -226,6 +283,7 @@ export async function ensurePythonEnv(systemPython, expectedVersion, onProgress
         // says we're up to date. This is what previously broke for boxes
         // that upgraded the runner without invalidating the marker.
         await ensureNltkData(onProgress);
+        await _resolveAndCacheCertBundle(onProgress);
         return { ok: true, pythonPath: venvPython() };
     }
     if (!existsSync(AGENTSCOPE)) {
@@ -298,6 +356,7 @@ export async function ensurePythonEnv(systemPython, expectedVersion, onProgress
     // Centralised in `ensureNltkData()` so the success/failure surface is
     // identical whether the venv is fresh or already valid.
     await ensureNltkData(onProgress);
+    await _resolveAndCacheCertBundle(onProgress);
     writeFileSync(VENV_MARK, venvMarkerValue(expectedVersion), "utf-8");
     // sanity: confirm shortuuid + agentscope (via PYTHONPATH) resolve now.
     // Probe must mirror the spawn env so PYTHONPATH=CACHE_DIR points at

package/localRagIngest.py

@@ -331,14 +331,245 @@ async def _ingest(args) -> int:
     return 0
 
 
+def _build_embedder(provider: str, model: str):
+    """Embedder factory shared between Mode B (local folder) and Mode C
+    (cloud files via HTTP). Returns the agentscope embedder instance +
+    a probe error message string when something is misconfigured (empty
+    string means OK)."""
+    if provider == "ollama":
+        from agentscope.embedding import OllamaTextEmbedding
+        DIMS = {
+            "nomic-embed-text": 768, "nomic-embed-text:v1.5": 768,
+            "mxbai-embed-large": 1024, "all-minilm": 384,
+            "snowflake-arctic-embed": 1024,
+        }
+        dim = DIMS.get(model.lower())
+        if dim is None:
+            return None, (
+                f"unknown ollama embedding model '{model}' — supported: {sorted(DIMS)}"
+            )
+        return OllamaTextEmbedding(
+            model_name=model, dimensions=dim, host=None,
+        ), ""
+    if provider == "lmstudio":
+        from agentscope.embedding import OpenAITextEmbedding
+        embedder = OpenAITextEmbedding(
+            api_key="lmstudio", model_name=model, dimensions=768,
+            base_url="http://localhost:1234/v1",
+        )
+        # Strip OpenAI-specific `dimensions` kwarg that LMS rejects
+        _orig_create = embedder.client.embeddings.create
+        async def _lms_create(**kw):
+            kw.pop("dimensions", None)
+            return await _orig_create(**kw)
+        embedder.client.embeddings.create = _lms_create  # type: ignore[assignment]
+        return embedder, ""
+    return None, f"unsupported embedder provider '{provider}'"
+
+
+async def _ingest_cloud_files(args) -> int:
+    """Mode C ingest.
+
+    Files originate on prod but the runner does the embedding so the
+    operator can use a free local model (ollama / lmstudio) without
+    needing an OpenAI key. Both the file CONTENTS and the resulting
+    (chunks, vectors) tuples flow over Socket.IO end-to-end:
+
+      • The server (tRPC bridge in runnerNamespace.ts) reads the files
+        from prod's <pipeline_dir>/rag/retrieval/, base64-encodes them,
+        and bakes them into the `rag:ingest` event payload. The runner
+        connection.ts unpacks them into ``<tmp>/files-dir/`` and
+        passes ``--files-dir <tmp> --manifest-json <tmp>/manifest.json``
+        here.
+      • For each file we parse + chunk + embed locally, then emit a
+        ``progress: {"kind":"upsert_chunks", ...}`` line to stdout. The
+        runner forwards that line as a ``rag:ingest-progress`` Socket.IO
+        event; the server's progress handler recognises the
+        ``upsert_chunks`` kind and POSTs the payload to
+        ``http://127.0.0.1:8766/pipelines/{name}/docs/retrieval/upsert``
+        — the loopback-only builder API that owns the QdrantStore.
+      • Nothing on the wire ever needs to leave the user's machine
+        via HTTP. The whole transport is the already-authenticated
+        Socket.IO channel, which means we don't have to expose any
+        builder endpoints publicly.
+    """
+    files_dir = Path(args.files_dir)
+    if not files_dir.exists() or not files_dir.is_dir():
+        _emit("error", reason=f"files-dir not found: {files_dir}")
+        return 3
+
+    embedder, err = _build_embedder(args.embedder_provider, args.embedder_model)
+    if not embedder:
+        _emit("error", reason=err)
+        return 4
+
+    manifest: dict = {}
+    if args.manifest_json:
+        try:
+            manifest = json.loads(Path(args.manifest_json).read_text(encoding="utf-8"))
+        except Exception as exc:
+            _emit("warning", reason=f"manifest read failed: {exc}")
+    files: list[dict] = manifest.get("files") or []
+    stored_embedder = manifest.get("stored_embedder") or {}
+    previously_ingested = manifest.get("previously_ingested") or {}
+    embedder_changed = (
+        stored_embedder.get("provider") != args.embedder_provider
+        or stored_embedder.get("model") != args.embedder_model
+    )
+    _emit("started", mode="cloud_files", pipeline=args.pipeline_name,
+          embedder_changed=embedder_changed, total_supported=len(files))
+
+    started = time.time()
+    chunks_added_total = 0
+    files_processed = 0
+    skipped = 0
+    files_failed: list[dict] = []
+
+    embedder_payload = {
+        "provider": args.embedder_provider,
+        "model": args.embedder_model,
+        "dimensions": embedder.dimensions,
+    }
+
+    first_emit = True
+    for entry in files:
+        name = entry["name"]
+        sha = entry["sha256"]
+        ext = (entry.get("ext") or "").lower()
+        if ext not in SUPPORTED_EXTS:
+            files_failed.append({"file": name, "error": f"unsupported ext '{ext}'"})
+            _emit("file_failed", file=name, error=f"unsupported ext '{ext}'")
+            continue
+        if not embedder_changed and previously_ingested.get(name, {}).get("sha256") == sha:
+            skipped += 1
+            _emit("skipped_sha_clean", file=name)
+            continue
+        tmp_path = files_dir / name
+        if not tmp_path.is_file():
+            files_failed.append({"file": name, "error": "file missing from cloud-files dir"})
+            _emit("file_failed", file=name, error="missing locally")
+            continue
+        reader_cls = _reader_for(ext)
+        if reader_cls is None:
+            files_failed.append({"file": name, "error": "no extractor"})
+            _emit("file_failed", file=name, error="no extractor")
+            continue
+        try:
+            reader = reader_cls()
+            docs = await reader(str(tmp_path))
+            for i, doc in enumerate(docs):
+                if hasattr(doc.metadata, "doc_id"):
+                    doc.metadata.doc_id = name
+                if hasattr(doc.metadata, "chunk_id"):
+                    doc.metadata.chunk_id = i
+            texts: list[str] = []
+            for doc in docs:
+                content = getattr(doc.metadata, "content", None)
+                text = (
+                    (content or {}).get("text", "")
+                    if isinstance(content, dict)
+                    else getattr(content, "text", "")
+                )
+                if text:
+                    texts.append(text)
+            if not texts:
+                files_processed += 1
+                _emit("file_done", file=name, chunks=0)
+                _emit("upsert_chunks",
+                      embedder=embedder_payload,
+                      chunks=[],
+                      files_done=[{"name": name, "sha256": sha, "chunks": 0}],
+                      embedder_changed=embedder_changed and first_emit,
+                      finalize=False)
+                first_emit = False
+                continue
+            vectors_resp = await embedder(text=texts)
+            vectors = [eb.embedding for eb in vectors_resp.embeddings]
+            if len(vectors) != len(texts):
+                raise RuntimeError(
+                    f"embedder returned {len(vectors)} vectors for {len(texts)} chunks"
+                )
+            upsert_chunks = [
+                {
+                    "text": text,
+                    "vector": [float(v) for v in vec],
+                    "metadata": {
+                        "doc_id": name,
+                        "chunk_id": i,
+                        "total_chunks": len(texts),
+                    },
+                }
+                for i, (text, vec) in enumerate(zip(texts, vectors))
+            ]
+            # Emit one upsert event per file. Server forwards to builder
+            # /upsert. For small per-file payloads this is well below the
+            # Socket.IO default 1 MB/event limit (60-doc corpora ≈ 6 MB
+            # spread across ~60 events).
+            _emit("upsert_chunks",
+                  embedder=embedder_payload,
+                  chunks=upsert_chunks,
+                  files_done=[{"name": name, "sha256": sha, "chunks": len(texts)}],
+                  embedder_changed=embedder_changed and first_emit,
+                  finalize=False)
+            first_emit = False
+            chunks_added_total += len(texts)
+            files_processed += 1
+            _emit("file_done", file=name, chunks=len(texts))
+        except Exception as exc:
+            files_failed.append({
+                "file": name, "error": f"{type(exc).__name__}: {exc}",
+            })
+            _emit("file_failed", file=name, error=str(exc))
+
+    # Finalize: tells the server to stamp the embedder identity into the
+    # manifest. Sent as one final, empty-chunk upsert event.
+    _emit("upsert_chunks",
+          embedder=embedder_payload,
+          chunks=[],
+          files_done=[],
+          embedder_changed=False,
+          finalize=True)
+
+    elapsed_ms = int((time.time() - started) * 1000)
+    _emit("done",
+          mode="cloud_files",
+          files_processed=files_processed,
+          chunks_added=chunks_added_total,
+          skipped=skipped,
+          embedder_changed=embedder_changed,
+          elapsed_ms=elapsed_ms,
+          errors=files_failed)
+    return 0
+
+
 def main() -> None:
     ap = argparse.ArgumentParser()
+    ap.add_argument("--mode", choices=["local_folder", "cloud_files"],
+                    default="local_folder")
     ap.add_argument("--pipeline-name", required=True)
-    ap.add_argument("--folder", required=True)
+    ap.add_argument("--folder", default="")
     ap.add_argument("--embedder-provider", required=True, choices=["ollama", "lmstudio"])
     ap.add_argument("--embedder-model", required=True)
+    # Mode C only (cloud_files): files were base64-encoded by the
+    # server into the rag:ingest payload and the runner's connection.ts
+    # unpacked them to disk at --files-dir. --manifest-json points at
+    # a small JSON manifest (file list + sha + previously-ingested map
+    # so we can skip unchanged files).
+    ap.add_argument("--files-dir", default="",
+                    help="Mode C only — directory containing the unpacked file payload.")
+    ap.add_argument("--manifest-json", default="",
+                    help="Mode C only — path to manifest JSON (file list, prior ingest, embedder).")
     args = ap.parse_args()
-    sys.exit(asyncio.run(_ingest(args)))
+    if args.mode == "cloud_files":
+        if not args.files_dir or not args.manifest_json:
+            _emit("error", reason="cloud_files mode requires --files-dir + --manifest-json")
+            sys.exit(2)
+        sys.exit(asyncio.run(_ingest_cloud_files(args)))
+    else:
+        if not args.folder:
+            _emit("error", reason="local_folder mode requires --folder")
+            sys.exit(2)
+        sys.exit(asyncio.run(_ingest(args)))
 
 
 if __name__ == "__main__":

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.0.48",
+  "version": "1.0.50",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,