@melaya/runner 1.0.25 → 1.0.28

package/dist/connection.js

@@ -104,6 +104,58 @@ export async function connect(opts) {
             }
             // Ensure shared modules are cached
             await ensureSharedModules(opts.serverUrl, payload.sharedVersion);
+            // Ensure the dedicated venv at ~/.melaya-runner/venv/ has
+            // agentscope + its transitive deps installed. Without this, the
+            // operator's system python3 typically doesn't have shortuuid /
+            // anthropic / openai / opentelemetry / etc. pre-installed and
+            // every pipeline run crashes during `import agentscope` with a
+            // ModuleNotFoundError. The venv is created lazily on first run
+            // and reused until sharedVersion changes.
+            const { ensurePythonEnv } = await import("./pythonEnv.js");
+            const envResult = await ensurePythonEnv(opts.pythonPath, payload.sharedVersion, (msg) => {
+                console.log(chalk.gray(`  [venv] ${msg}`));
+                // Surface bootstrap progress to the FE so the launching
+                // animation reflects the real state instead of staying on a
+                // generic spinner during the 1-2 min first-time install.
+                socket.emit("runner:event", {
+                    runId: payload.runId,
+                    payload: {
+                        event_type: "pipeline_phase",
+                        run_id: payload.runId,
+                        project: payload.project,
+                        step: 1,
+                        total: 5,
+                        label: `venv: ${msg}`,
+                        status: "started",
+                    },
+                });
+            });
+            if (!envResult.ok) {
+                console.log(chalk.red(`  ✗ python venv bootstrap failed: ${envResult.reason}`));
+                socket.emit("runner:event", {
+                    runId: payload.runId,
+                    payload: {
+                        event_type: "agent_message",
+                        run_id: payload.runId,
+                        project: payload.project,
+                        replyId: `venv-${payload.runId}`,
+                        replyName: "Runner",
+                        replyRole: "system",
+                        msg: {
+                            id: `venv-${payload.runId}`,
+                            name: "Runner",
+                            role: "system",
+                            content: [{ type: "text", text: `Python env bootstrap failed: ${envResult.reason}` }],
+                            metadata: {},
+                            timestamp: new Date().toISOString(),
+                        },
+                    },
+                });
+                socket.emit("runner:runComplete", { runId: payload.runId, status: "failed" });
+                return;
+            }
+            // Replace the system python with the venv python for THIS spawn.
+            const pipelinePython = envResult.pythonPath;
             // Register run in DB + set active project for event relay
             setActiveProject(payload.project);
             socket.emit("runner:registerRun", {
@@ -199,8 +251,8 @@ export async function connect(opts) {
                     }
                     : {}),
             };
-            // Spawn Python subprocess
-            const proc = spawn(opts.pythonPath, ["-u", join(runDir, "main.py")], {
+            // Spawn Python subprocess (uses the bootstrapped venv python).
+            const proc = spawn(pipelinePython, ["-u", join(runDir, "main.py")], {
                 cwd: runDir,
                 env,
                 stdio: ["ignore", "pipe", "pipe"],

package/dist/pythonEnv.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Manage a dedicated Python venv at ~/.melaya-runner/venv/ so pipelines
+ * don't depend on whatever's globally installed in the operator's
+ * `python3`. Without this, every fresh runner install hits import errors
+ * (`ModuleNotFoundError: No module named 'shortuuid'`, etc.) on the
+ * first pipeline run because agentscope and its 23 transitive deps are
+ * not present in the operator's system Python.
+ *
+ * Idempotent: re-uses the venv if a marker file matches the current
+ * shared-bundle version. Re-bootstraps when the bundle version changes.
+ */
+export declare function venvPython(): string;
+export declare function ensurePythonEnv(systemPython: string, expectedVersion: string, onProgress?: (msg: string) => void): Promise<{
+    ok: boolean;
+    pythonPath: string;
+    reason?: string;
+}>;

package/dist/pythonEnv.js

@@ -0,0 +1,153 @@
+/**
+ * Manage a dedicated Python venv at ~/.melaya-runner/venv/ so pipelines
+ * don't depend on whatever's globally installed in the operator's
+ * `python3`. Without this, every fresh runner install hits import errors
+ * (`ModuleNotFoundError: No module named 'shortuuid'`, etc.) on the
+ * first pipeline run because agentscope and its 23 transitive deps are
+ * not present in the operator's system Python.
+ *
+ * Idempotent: re-uses the venv if a marker file matches the current
+ * shared-bundle version. Re-bootstraps when the bundle version changes.
+ */
+import { spawn } from "child_process";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { join } from "path";
+import { homedir, platform } from "os";
+import chalk from "chalk";
+const CACHE_DIR = join(homedir(), ".melaya-runner");
+const VENV_DIR = join(CACHE_DIR, "venv");
+const VENV_MARK = join(CACHE_DIR, "venv-version.txt");
+const AGENTSCOPE = join(CACHE_DIR, "agentscope");
+// Explicit dependency list. The shared-bundle endpoint ships only
+// `*.py` files — no pyproject.toml — so `pip install -e
+// ~/.melaya-runner/agentscope` does NOT work (pip can't find a project
+// definition). PYTHONPATH=~/.melaya-runner/ already makes `import
+// agentscope` resolve to the cached source; we only need pip to bring
+// in agentscope's transitive deps + the tool-runtime extras.
+//
+// Mirrors melayaAgents/agentscope/pyproject.toml `dependencies = [...]`
+// PLUS the runner-tool extras (aiohttp, requests, python-dotenv).
+// If you bump the agentscope pyproject deps, bump this list too —
+// or extract from pyproject at server-side and ship via the bundle.
+const PIP_DEPS = [
+    // ── agentscope core (mirrors pyproject.toml `dependencies`) ──
+    "aioitertools",
+    "anthropic",
+    "dashscope",
+    "docstring_parser",
+    "filetype",
+    "json5",
+    "json_repair",
+    "mcp>=1.13",
+    "numpy",
+    "openai",
+    "python-datauri",
+    "opentelemetry-api>=1.39.0",
+    "opentelemetry-sdk>=1.39.0",
+    "opentelemetry-exporter-otlp>=1.39.0",
+    "opentelemetry-semantic-conventions>=0.60b0",
+    "python-socketio",
+    "shortuuid",
+    "tiktoken",
+    "sqlalchemy",
+    "python-frontmatter",
+    "aiofiles",
+    // ── runner tools (shared/tools/*.py) ──
+    "aiohttp",
+    "requests",
+    "python-dotenv",
+];
+export function venvPython() {
+    return platform() === "win32"
+        ? join(VENV_DIR, "Scripts", "python.exe")
+        : join(VENV_DIR, "bin", "python");
+}
+function venvIsValid(expectedVersion) {
+    if (!existsSync(venvPython()))
+        return false;
+    if (!existsSync(VENV_MARK))
+        return false;
+    try {
+        const cached = readFileSync(VENV_MARK, "utf-8").trim();
+        return cached === expectedVersion;
+    }
+    catch {
+        return false;
+    }
+}
+function runProc(cmd, args, onLine, envExtra = {}) {
+    return new Promise((resolve) => {
+        const child = spawn(cmd, args, {
+            stdio: ["ignore", "pipe", "pipe"],
+            env: { ...process.env, ...envExtra },
+        });
+        const chew = (b) => {
+            for (const ln of b.toString().split("\n")) {
+                const t = ln.trimEnd();
+                if (t)
+                    onLine(t);
+            }
+        };
+        child.stdout?.on("data", chew);
+        child.stderr?.on("data", chew);
+        child.on("exit", (code) => resolve(code ?? 1));
+        child.on("error", () => resolve(1));
+    });
+}
+export async function ensurePythonEnv(systemPython, expectedVersion, onProgress = (m) => console.log(chalk.gray(`  [venv] ${m}`))) {
+    if (venvIsValid(expectedVersion)) {
+        return { ok: true, pythonPath: venvPython() };
+    }
+    if (!existsSync(AGENTSCOPE)) {
+        return {
+            ok: false,
+            pythonPath: systemPython,
+            reason: `agentscope cache missing at ${AGENTSCOPE} — restart the runner so the shared bundle downloads first.`,
+        };
+    }
+    mkdirSync(CACHE_DIR, { recursive: true });
+    // Step 1: create the venv if missing.
+    if (!existsSync(venvPython())) {
+        onProgress(`creating venv at ${VENV_DIR} (one-time, ~5s)`);
+        const code = await runProc(systemPython, ["-m", "venv", VENV_DIR], onProgress);
+        if (code !== 0 || !existsSync(venvPython())) {
+            return {
+                ok: false,
+                pythonPath: systemPython,
+                reason: `python -m venv failed (exit ${code}). Confirm the venv module is available: ${systemPython} -m venv --help`,
+            };
+        }
+    }
+    // Step 2: upgrade pip + wheel so wheel-based deps install cleanly.
+    onProgress("upgrading pip / wheel inside the venv");
+    await runProc(venvPython(), ["-m", "pip", "install", "--quiet", "--upgrade", "pip", "wheel"], onProgress);
+    // Step 3: install agentscope's transitive deps + tool extras directly.
+    // We do NOT pip-install agentscope itself — the shared bundle ships
+    // only *.py files (no pyproject.toml), so PYTHONPATH-based import is
+    // the only path. The cached agentscope source under
+    // ~/.melaya-runner/agentscope/ resolves via PYTHONPATH at spawn time.
+    onProgress(`installing ${PIP_DEPS.length} python deps (anthropic, openai, opentelemetry, …) — first run can take 1-2 min`);
+    const acode = await runProc(venvPython(), ["-m", "pip", "install", "--quiet", "--disable-pip-version-check", ...PIP_DEPS], onProgress);
+    if (acode !== 0) {
+        return {
+            ok: false,
+            pythonPath: systemPython,
+            reason: `pip install failed (exit ${acode}). Check the lines above for the actual pip error.`,
+        };
+    }
+    writeFileSync(VENV_MARK, expectedVersion, "utf-8");
+    // sanity: confirm shortuuid + agentscope (via PYTHONPATH) resolve now.
+    // Probe must mirror the spawn env so PYTHONPATH=CACHE_DIR points at
+    // the cached agentscope source — without this the probe ImportError's
+    // even though the real spawn would work.
+    const probe = await runProc(venvPython(), ["-c", "import shortuuid, agentscope, anthropic, openai"], onProgress, { PYTHONPATH: CACHE_DIR });
+    if (probe !== 0) {
+        return {
+            ok: false,
+            pythonPath: systemPython,
+            reason: "venv created but agentscope import probe failed. Delete ~/.melaya-runner/venv and restart the runner.",
+        };
+    }
+    onProgress(`✓ venv ready (python=${venvPython()})`);
+    return { ok: true, pythonPath: venvPython() };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.0.25",
+  "version": "1.0.28",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,