@melaya/runner 1.0.25 → 1.0.27

package/dist/connection.js

@@ -104,6 +104,58 @@ export async function connect(opts) {
             }
             // Ensure shared modules are cached
             await ensureSharedModules(opts.serverUrl, payload.sharedVersion);
+            // Ensure the dedicated venv at ~/.melaya-runner/venv/ has
+            // agentscope + its transitive deps installed. Without this, the
+            // operator's system python3 typically doesn't have shortuuid /
+            // anthropic / openai / opentelemetry / etc. pre-installed and
+            // every pipeline run crashes during `import agentscope` with a
+            // ModuleNotFoundError. The venv is created lazily on first run
+            // and reused until sharedVersion changes.
+            const { ensurePythonEnv } = await import("./pythonEnv.js");
+            const envResult = await ensurePythonEnv(opts.pythonPath, payload.sharedVersion, (msg) => {
+                console.log(chalk.gray(`  [venv] ${msg}`));
+                // Surface bootstrap progress to the FE so the launching
+                // animation reflects the real state instead of staying on a
+                // generic spinner during the 1-2 min first-time install.
+                socket.emit("runner:event", {
+                    runId: payload.runId,
+                    payload: {
+                        event_type: "pipeline_phase",
+                        run_id: payload.runId,
+                        project: payload.project,
+                        step: 1,
+                        total: 5,
+                        label: `venv: ${msg}`,
+                        status: "started",
+                    },
+                });
+            });
+            if (!envResult.ok) {
+                console.log(chalk.red(`  ✗ python venv bootstrap failed: ${envResult.reason}`));
+                socket.emit("runner:event", {
+                    runId: payload.runId,
+                    payload: {
+                        event_type: "agent_message",
+                        run_id: payload.runId,
+                        project: payload.project,
+                        replyId: `venv-${payload.runId}`,
+                        replyName: "Runner",
+                        replyRole: "system",
+                        msg: {
+                            id: `venv-${payload.runId}`,
+                            name: "Runner",
+                            role: "system",
+                            content: [{ type: "text", text: `Python env bootstrap failed: ${envResult.reason}` }],
+                            metadata: {},
+                            timestamp: new Date().toISOString(),
+                        },
+                    },
+                });
+                socket.emit("runner:runComplete", { runId: payload.runId, status: "failed" });
+                return;
+            }
+            // Replace the system python with the venv python for THIS spawn.
+            const pipelinePython = envResult.pythonPath;
             // Register run in DB + set active project for event relay
             setActiveProject(payload.project);
             socket.emit("runner:registerRun", {
@@ -199,8 +251,8 @@ export async function connect(opts) {
                     }
                     : {}),
             };
-            // Spawn Python subprocess
-            const proc = spawn(opts.pythonPath, ["-u", join(runDir, "main.py")], {
+            // Spawn Python subprocess (uses the bootstrapped venv python).
+            const proc = spawn(pipelinePython, ["-u", join(runDir, "main.py")], {
                 cwd: runDir,
                 env,
                 stdio: ["ignore", "pipe", "pipe"],

package/dist/pythonEnv.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Manage a dedicated Python venv at ~/.melaya-runner/venv/ so pipelines
+ * don't depend on whatever's globally installed in the operator's
+ * `python3`. Without this, every fresh runner install hits import errors
+ * (`ModuleNotFoundError: No module named 'shortuuid'`, etc.) on the
+ * first pipeline run because agentscope and its 23 transitive deps are
+ * not present in the operator's system Python.
+ *
+ * Idempotent: re-uses the venv if a marker file matches the current
+ * shared-bundle version. Re-bootstraps when the bundle version changes.
+ */
+export declare function venvPython(): string;
+export declare function ensurePythonEnv(systemPython: string, expectedVersion: string, onProgress?: (msg: string) => void): Promise<{
+    ok: boolean;
+    pythonPath: string;
+    reason?: string;
+}>;

package/dist/pythonEnv.js

@@ -0,0 +1,112 @@
+/**
+ * Manage a dedicated Python venv at ~/.melaya-runner/venv/ so pipelines
+ * don't depend on whatever's globally installed in the operator's
+ * `python3`. Without this, every fresh runner install hits import errors
+ * (`ModuleNotFoundError: No module named 'shortuuid'`, etc.) on the
+ * first pipeline run because agentscope and its 23 transitive deps are
+ * not present in the operator's system Python.
+ *
+ * Idempotent: re-uses the venv if a marker file matches the current
+ * shared-bundle version. Re-bootstraps when the bundle version changes.
+ */
+import { spawn } from "child_process";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { join } from "path";
+import { homedir, platform } from "os";
+import chalk from "chalk";
+const CACHE_DIR = join(homedir(), ".melaya-runner");
+const VENV_DIR = join(CACHE_DIR, "venv");
+const VENV_MARK = join(CACHE_DIR, "venv-version.txt");
+const AGENTSCOPE = join(CACHE_DIR, "agentscope");
+// Extra deps the cached `shared/tools/*.py` import that aren't in
+// agentscope's pyproject. Keep this list narrow — full tool-specific
+// SDKs (linkedin_api, twilio, stripe, etc.) install lazily on first
+// use via tool-specific scripts. The minimum to GET A PIPELINE RUNNING
+// is agentscope + aiohttp + requests + redis (used by some tools).
+const EXTRA_DEPS = ["aiohttp", "requests", "python-dotenv"];
+export function venvPython() {
+    return platform() === "win32"
+        ? join(VENV_DIR, "Scripts", "python.exe")
+        : join(VENV_DIR, "bin", "python");
+}
+function venvIsValid(expectedVersion) {
+    if (!existsSync(venvPython()))
+        return false;
+    if (!existsSync(VENV_MARK))
+        return false;
+    try {
+        const cached = readFileSync(VENV_MARK, "utf-8").trim();
+        return cached === expectedVersion;
+    }
+    catch {
+        return false;
+    }
+}
+function runProc(cmd, args, onLine) {
+    return new Promise((resolve) => {
+        const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });
+        const chew = (b) => {
+            for (const ln of b.toString().split("\n")) {
+                const t = ln.trimEnd();
+                if (t)
+                    onLine(t);
+            }
+        };
+        child.stdout?.on("data", chew);
+        child.stderr?.on("data", chew);
+        child.on("exit", (code) => resolve(code ?? 1));
+        child.on("error", () => resolve(1));
+    });
+}
+export async function ensurePythonEnv(systemPython, expectedVersion, onProgress = (m) => console.log(chalk.gray(`  [venv] ${m}`))) {
+    if (venvIsValid(expectedVersion)) {
+        return { ok: true, pythonPath: venvPython() };
+    }
+    if (!existsSync(AGENTSCOPE)) {
+        return {
+            ok: false,
+            pythonPath: systemPython,
+            reason: `agentscope cache missing at ${AGENTSCOPE} — restart the runner so the shared bundle downloads first.`,
+        };
+    }
+    mkdirSync(CACHE_DIR, { recursive: true });
+    // Step 1: create the venv if missing.
+    if (!existsSync(venvPython())) {
+        onProgress(`creating venv at ${VENV_DIR} (one-time, ~5s)`);
+        const code = await runProc(systemPython, ["-m", "venv", VENV_DIR], onProgress);
+        if (code !== 0 || !existsSync(venvPython())) {
+            return {
+                ok: false,
+                pythonPath: systemPython,
+                reason: `python -m venv failed (exit ${code}). Confirm the venv module is available: ${systemPython} -m venv --help`,
+            };
+        }
+    }
+    // Step 2: upgrade pip + wheel so wheel-based deps install cleanly.
+    onProgress("upgrading pip / wheel inside the venv");
+    await runProc(venvPython(), ["-m", "pip", "install", "--quiet", "--upgrade", "pip", "wheel"], onProgress);
+    // Step 3: install agentscope from the local cache (its pyproject pulls
+    // anthropic, openai, shortuuid, opentelemetry-*, etc.).
+    onProgress("installing agentscope + 23 transitive deps (anthropic, openai, opentelemetry, …) — first run can take 1-2 min");
+    const acode = await runProc(venvPython(), ["-m", "pip", "install", "--quiet", "-e", AGENTSCOPE, ...EXTRA_DEPS], onProgress);
+    if (acode !== 0) {
+        return {
+            ok: false,
+            pythonPath: systemPython,
+            reason: `pip install agentscope failed (exit ${acode}). Check the lines above for the actual pip error.`,
+        };
+    }
+    writeFileSync(VENV_MARK, expectedVersion, "utf-8");
+    // sanity: confirm shortuuid resolves now (it's the historic canary
+    // import that fails on a stale env).
+    const probe = await runProc(venvPython(), ["-c", "import shortuuid, agentscope, anthropic, openai"], onProgress);
+    if (probe !== 0) {
+        return {
+            ok: false,
+            pythonPath: systemPython,
+            reason: "venv created but agentscope import probe failed. Delete ~/.melaya-runner/venv and restart the runner.",
+        };
+    }
+    onProgress(`✓ venv ready (python=${venvPython()})`);
+    return { ok: true, pythonPath: venvPython() };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.0.25",
+  "version": "1.0.27",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,