@melaya/runner 1.1.48 → 1.1.50

package/dist/assistantHost.py

@@ -170,6 +170,8 @@ def _config_hash() -> str:
     pinned_tools = ",".join(sorted(t.lower() for t in raw_pinned.split(",") if t.strip()))
     if pinned_tools:
         fields.append(pinned_tools)
+    if os.environ.get("MEL_ASSISTANT_TELEMETRY_SURFACE") == "marketing":
+        fields.append("marketing-scoped-tools-v1")
     canon = "|".join(fields)
     return hashlib.sha256(canon.encode("utf-8")).hexdigest()[:32]
 
@@ -607,6 +609,7 @@ def _build_agent():
     # names are case-sensitive Python identifiers (unlike connector_services,
     # which are lowercased service ids). Registered ACTIVE at boot below so the
     # model calls them directly instead of search_tools/activate_tool.
+    marketing_tool_scope = os.environ.get("MEL_ASSISTANT_TELEMETRY_SURFACE") == "marketing"
     pinned_tool_names = [t.strip() for t in os.environ.get("MEL_ASSISTANT_PINNED_TOOLS", "").split(",") if t.strip()]
     # Self-arm the write-approval gate whenever connectors are active, so it can't
     # be silently OFF on an older runner build that didn't set this env. Writes
@@ -659,11 +662,12 @@ def _build_agent():
             include_categories=categories + core_categories + connector_services,
             budget=_budget,
             pinned_names=pinned_tool_names or None,
+            allowed_names=pinned_tool_names if marketing_tool_scope else None,
         )
     except Exception as exc:
         _log(f"toolkit build failed (connectors={connector_services}, core): {exc}; retrying melaya_agent only")
         try:
-            toolkit = build_toolkit(categories=["melaya_agent"])
+            toolkit = build_toolkit(names=[]) if marketing_tool_scope else build_toolkit(categories=["melaya_agent"])
         except Exception:
             toolkit = build_toolkit(names=[])
 
@@ -1302,6 +1306,7 @@ def main() -> int:
             except Exception:
                 _log("restore failed:\n" + traceback.format_exc())
             continue
+        os.environ["MEL_APPROVAL_CONVERSATION_ID"] = str(req.get("conversation_id") or "")
         turn_id = str(req.get("turnId") or "")
         message = str(req.get("message") or "")
         # Vision: base64 image blocks for THIS turn (validated server-side). Used

package/dist/connection.js

@@ -77,6 +77,11 @@ let warmSpareSeed = null;
 // SYNCHRONOUSLY here closes that window.
 const bootingAssistants = new Set();
 const ASSISTANT_IDLE_MS = 15 * 60 * 1000;
+// OOM backstop for the assistant stdout line-framer: if a single unterminated
+// line (no newline) grows past this, flush + reset rather than let `+=` build an
+// unbounded string. Generous — legitimate frames (tool results, transcripts) are
+// far smaller; a line this large would exceed the socket buffer anyway.
+const ASSISTANT_STDOUT_MAX_CARRY = 64 * 1024 * 1024;
 export async function connect(opts) {
     const spinner = ora("Connecting to Melaya...").start();
     // Connect to the /api/v1/runner NAMESPACE on the existing Socket.IO server.
@@ -1324,11 +1329,25 @@ export async function connect(opts) {
         console.log(chalk.hex("#7c6ff0")(`  ◆ Assistant ${a.forward ? "session" : "spare"} ${a.sid.slice(0, 12)}… (${a.providerLabel})`));
         proc.stdout?.on("data", (data) => {
             session.lastActivity = Date.now();
+            // O(n) line framing: accumulate, split ONCE, carry the trailing partial.
+            // The old `buf += …; while(indexOf/slice)` loop re-sliced the WHOLE buffer
+            // per line — O(n²) on a big burst (a large tool result / browser tree in
+            // the transcript). That blocked the Node event loop long enough that
+            // Socket.IO couldn't service the connection (send the engine.io pong), so
+            // the server's pingTimeout dropped the socket MID-TURN — the "heartbeat
+            // stops while working" failure. Keeping this O(n) means the heartbeat and
+            // pong keep flowing no matter how much the host streams.
             session.stdoutBuf += data.toString();
-            let idx;
-            while ((idx = session.stdoutBuf.indexOf("\n")) >= 0) {
-                const line = session.stdoutBuf.slice(0, idx);
-                session.stdoutBuf = session.stdoutBuf.slice(idx + 1);
+            const parts = session.stdoutBuf.split("\n");
+            session.stdoutBuf = parts.pop() ?? "";
+            // OOM backstop: an unterminated line that keeps growing across chunks would
+            // make the `+=` above quadratic again. Flush it as its own line and reset.
+            if (session.stdoutBuf.length > ASSISTANT_STDOUT_MAX_CARRY) {
+                parts.push(session.stdoutBuf);
+                session.stdoutBuf = "";
+                console.log(chalk.yellow(`  [assistant-stdout] flushed oversized line (${(ASSISTANT_STDOUT_MAX_CARRY / 1048576) | 0}MB+) to protect the event loop`));
+            }
+            for (const line of parts) {
                 const t = line.trim();
                 // A protocol frame can arrive GLUED to a preceding raw print that had no
                 // trailing newline (agentscope prints the agent transcript to the same
@@ -1865,6 +1884,7 @@ export async function connect(opts) {
                 message: payload.message,
                 hitl_mode: hitlMode,
                 static_context: typeof payload.staticContext === "string" ? payload.staticContext : "",
+                conversation_id: String(payload.sessionId || "").split(":").slice(1).join(":"),
                 telemetry_surface: typeof payload.telemetrySurface === "string" ? payload.telemetrySurface : "assistant",
                 // Per-turn browser grant: forwarded to the host as the raw compact
                 // JWS string so the Python browser toolkit can verify it again
@@ -2454,6 +2474,124 @@ export async function connect(opts) {
             reply(false, undefined, e?.message || String(e));
         }
     });
+    // ── Residential-IP page fetch (Melaya backlink presence check) ──────────────
+    // The Melaya cloud runs from a datacenter IP that Cloudflare "managed" Turnstile
+    // forces into an interactive challenge no headless engine clears. THIS machine
+    // is on a residential IP, where the same real-browser fetch passes in seconds.
+    // The server hands us ONE public https url; we fetch it with the local Scrapling
+    // ladder (localBacklinkFetch.py) and return the HTML for the server's presence
+    // scan. We receive only the url — no secrets, no internal hosts, no catalog data.
+    socket.on("runner:backlink-fetch", async (payload, ack) => {
+        const rid = payload?.requestId;
+        if (!rid)
+            return;
+        ack?.({ ok: true });
+        const reply = (r) => socket.emit("runner:backlink-fetch-result", {
+            requestId: rid,
+            ok: !r.error && !!r.html && !r.challenged,
+            status: r.status ?? 0,
+            engine: r.engine ?? null,
+            challenged: !!r.challenged,
+            html: r.html ?? "",
+            finalUrl: r.finalUrl ?? payload.url,
+            error: r.error,
+        });
+        // HTTPS-only, public url — defence in depth (the server already validated it).
+        let u = null;
+        try {
+            u = new URL(payload.url);
+        }
+        catch {
+            u = null;
+        }
+        if (!u || u.protocol !== "https:") {
+            reply({ error: "invalid_url" });
+            return;
+        }
+        try {
+            const { ensurePythonEnv } = await import("./pythonEnv.js");
+            const { getLocalSharedVersion } = await import("./sharedVendor.js");
+            const localVersion = getLocalSharedVersion();
+            if (!localVersion) {
+                reply({ error: "runner python runtime still setting up" });
+                return;
+            }
+            const env = await ensurePythonEnv(opts.pythonPath, localVersion, (m) => { if (opts.verbose)
+                console.log(chalk.gray(`  [backlink-fetch venv] ${m}`)); });
+            if (!env.ok) {
+                reply({ error: `venv bootstrap failed: ${env.reason}` });
+                return;
+            }
+            const { existsSync, copyFileSync, mkdirSync } = await import("fs");
+            const workDir = join(tmpdir(), `melaya-blf-${Date.now()}`);
+            mkdirSync(workDir, { recursive: true });
+            const candidates = [
+                join(__dirname, "localBacklinkFetch.py"),
+                join(__dirname, "..", "localBacklinkFetch.py"),
+            ];
+            let found = "";
+            for (const c of candidates) {
+                if (existsSync(c)) {
+                    found = c;
+                    break;
+                }
+            }
+            if (!found) {
+                reply({ error: "localBacklinkFetch.py not found — update @melaya/runner" });
+                return;
+            }
+            const staged = join(workDir, "localBacklinkFetch.py");
+            copyFileSync(found, staged);
+            const certBundle = (await import("./pythonEnv.js")).getCertBundlePath();
+            const sslEnv = certBundle
+                ? { SSL_CERT_FILE: certBundle, REQUESTS_CA_BUNDLE: certBundle } : {};
+            const budget = Math.max(6, Math.min(75, Math.floor(payload.budgetSeconds || 40)));
+            const proc = spawn(env.pythonPath, ["-u", staged], {
+                env: { ...process.env, ...sslEnv },
+                cwd: workDir,
+            });
+            let stdout = "";
+            let stderr = "";
+            let done = false;
+            const finish = (r) => { if (done)
+                return; done = true; reply(r); };
+            // Hard wall-clock cap slightly above the fetch budget so a wedged browser
+            // can't hold the session; the server has its own (budget+8s) timeout too.
+            const killTimer = setTimeout(() => {
+                try {
+                    proc.kill();
+                }
+                catch { /* already gone */ }
+                finish({ error: "fetch timed out" });
+            }, (budget + 10) * 1000);
+            proc.stdout.on("data", (d) => { stdout += d.toString("utf-8"); });
+            proc.stderr.on("data", (d) => { stderr += d.toString("utf-8"); });
+            proc.on("error", (err) => { clearTimeout(killTimer); finish({ error: err?.message || String(err) }); });
+            proc.on("close", (code) => {
+                clearTimeout(killTimer);
+                const jsonLine = stdout.split(/\r?\n/).map(s => s.trim()).filter(Boolean).reverse()[0] || "";
+                let parsed = null;
+                try {
+                    parsed = JSON.parse(jsonLine);
+                }
+                catch { /* handled below */ }
+                if (parsed && typeof parsed === "object") {
+                    finish({ status: parsed.status, engine: parsed.engine, challenged: !!parsed.challenged, html: parsed.html || "", finalUrl: parsed.finalUrl });
+                }
+                else {
+                    finish({ error: `fetch exited ${code ?? "?"}${stderr ? `: ${stderr.slice(-200)}` : ""}` });
+                }
+            });
+            try {
+                proc.stdin.write(JSON.stringify({ url: u.href, budget_seconds: budget }));
+                proc.stdin.end();
+            }
+            catch { /* proc.on('error') handles it */ }
+        }
+        catch (e) {
+            reply({ error: e?.message || String(e) });
+        }
+    });
     socket.on("linkedin:start-login", async (req) => {
         const sid = req?.session_id;
         if (!sid)

package/dist/localBacklinkFetch.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Residential-IP page fetch for the Melaya backlink presence check.
+
+The Melaya cloud runs from a datacenter IP that Cloudflare "managed" Turnstile
+forces into an interactive challenge no headless engine clears. The user's
+runner is on a RESIDENTIAL IP, where the same real-browser fetch passes in
+seconds. So the server hands us ONE public https url; we fetch it with the
+local Scrapling stack and print the result as a single JSON line on stdout:
+
+    {"status": int, "engine": str|null, "challenged": bool,
+     "finalUrl": str, "html": str}   # html capped
+
+Input (stdin, JSON): {"url": "...", "budget_seconds": 40}
+
+Self-contained on purpose (mirrors localDbProbe.py): it imports only Scrapling
+from the runner venv, never the shared tool bundle. HTTPS-only; the server
+already validated the url, this is defence in depth. Never fetches anything but
+the url it is given — no internal hosts, no secrets, no catalog data.
+"""
+import asyncio
+import json
+import sys
+
+_HTML_CAP = 600_000
+_CHALLENGE_MARKERS = (
+    "just a moment", "cf_chl", "cf-challenge", "checking your browser",
+    "enable javascript and cookies", "attention required",
+)
+
+
+def _html_of(page) -> str:
+    for attr in ("html_content", "body", "content"):
+        try:
+            v = getattr(page, attr)
+            if callable(v):
+                v = v()
+            if v:
+                return v if isinstance(v, str) else str(v)
+        except Exception:
+            pass
+    try:
+        return str(page)
+    except Exception:
+        return ""
+
+
+def _challenged(h: str) -> bool:
+    h = (h or "").lower()
+    return any(m in h for m in _CHALLENGE_MARKERS)
+
+
+async def _fetch(url: str, budget: float) -> dict:
+    ms = int(max(6.0, min(75.0, budget)) * 1000)
+    # ── Tier 1: TLS-impersonating async GET (no browser) ──
+    try:
+        from scrapling.fetchers import AsyncFetcher
+        p = await AsyncFetcher.get(url, timeout=min(ms, 15000), impersonate="chrome",
+                                   stealthy_headers=True, follow_redirects=True)
+        st = int(getattr(p, "status", 0) or 0)
+        h = _html_of(p)
+        final = str(getattr(p, "url", "") or url)
+        if 200 <= st < 300 and h and not _challenged(h):
+            return {"status": st, "engine": "fast", "challenged": False,
+                    "finalUrl": final, "html": h[:_HTML_CAP]}
+    except Exception:
+        pass
+
+    # ── Tier 2/3: stealth browser (real Chrome), plain then solve_cloudflare ──
+    try:
+        from scrapling.fetchers import StealthyFetcher
+    except Exception:
+        return {"status": 0, "engine": None, "challenged": False, "finalUrl": url, "html": ""}
+    fn = getattr(StealthyFetcher, "async_fetch", None)
+    if fn is None:
+        return {"status": 0, "engine": None, "challenged": False, "finalUrl": url, "html": ""}
+
+    last = {"status": 0, "engine": "stealth", "challenged": True, "finalUrl": url, "html": ""}
+    for solve in (False, True):
+        base = {"headless": True, "timeout": 60000 if solve else min(ms, 25000)}
+        if solve:
+            base.update(solve_cloudflare=True, block_webrtc=True, disable_resources=False)
+        page = None
+        engine = ("real_chrome_cf" if solve else "real_chrome")
+        for use_real in (True, False):
+            kw = dict(base)
+            if use_real:
+                kw["real_chrome"] = True
+            else:
+                engine = ("chromium_cf" if solve else "chromium")
+            try:
+                page = await fn(url, **kw)
+                break
+            except Exception:
+                page = None
+                continue
+        if page is None:
+            continue
+        h = _html_of(page)
+        st = int(getattr(page, "status", 0) or 0)
+        final = str(getattr(page, "url", "") or url)
+        last = {"status": st, "engine": engine, "challenged": _challenged(h),
+                "finalUrl": final, "html": (h or "")[:_HTML_CAP]}
+        if h and not _challenged(h):
+            return last
+        if not solve and not _challenged(h):
+            break  # plain stealth returned something non-challenge-y but empty
+    return last
+
+
+async def _main() -> None:
+    try:
+        data = json.loads(sys.stdin.read() or "{}")
+    except Exception:
+        data = {}
+    url = str(data.get("url") or "").strip()
+    try:
+        budget = float(data.get("budget_seconds") or 40)
+    except Exception:
+        budget = 40.0
+    from urllib.parse import urlsplit
+    if not url or urlsplit(url).scheme != "https":
+        print(json.dumps({"status": 0, "engine": None, "challenged": False,
+                          "finalUrl": url, "html": "", "error": "https_only"}))
+        return
+    try:
+        out = await _fetch(url, budget)
+    except Exception as e:
+        out = {"status": 0, "engine": None, "challenged": False,
+               "finalUrl": url, "html": "", "error": str(e)[:200]}
+    print(json.dumps(out))
+
+
+if sys.platform.startswith("win"):
+    try:
+        asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+    except Exception:
+        pass
+asyncio.run(_main())

package/localBacklinkFetch.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Residential-IP page fetch for the Melaya backlink presence check.
+
+The Melaya cloud runs from a datacenter IP that Cloudflare "managed" Turnstile
+forces into an interactive challenge no headless engine clears. The user's
+runner is on a RESIDENTIAL IP, where the same real-browser fetch passes in
+seconds. So the server hands us ONE public https url; we fetch it with the
+local Scrapling stack and print the result as a single JSON line on stdout:
+
+    {"status": int, "engine": str|null, "challenged": bool,
+     "finalUrl": str, "html": str}   # html capped
+
+Input (stdin, JSON): {"url": "...", "budget_seconds": 40}
+
+Self-contained on purpose (mirrors localDbProbe.py): it imports only Scrapling
+from the runner venv, never the shared tool bundle. HTTPS-only; the server
+already validated the url, this is defence in depth. Never fetches anything but
+the url it is given — no internal hosts, no secrets, no catalog data.
+"""
+import asyncio
+import json
+import sys
+
+_HTML_CAP = 600_000
+_CHALLENGE_MARKERS = (
+    "just a moment", "cf_chl", "cf-challenge", "checking your browser",
+    "enable javascript and cookies", "attention required",
+)
+
+
+def _html_of(page) -> str:
+    for attr in ("html_content", "body", "content"):
+        try:
+            v = getattr(page, attr)
+            if callable(v):
+                v = v()
+            if v:
+                return v if isinstance(v, str) else str(v)
+        except Exception:
+            pass
+    try:
+        return str(page)
+    except Exception:
+        return ""
+
+
+def _challenged(h: str) -> bool:
+    h = (h or "").lower()
+    return any(m in h for m in _CHALLENGE_MARKERS)
+
+
+async def _fetch(url: str, budget: float) -> dict:
+    ms = int(max(6.0, min(75.0, budget)) * 1000)
+    # ── Tier 1: TLS-impersonating async GET (no browser) ──
+    try:
+        from scrapling.fetchers import AsyncFetcher
+        p = await AsyncFetcher.get(url, timeout=min(ms, 15000), impersonate="chrome",
+                                   stealthy_headers=True, follow_redirects=True)
+        st = int(getattr(p, "status", 0) or 0)
+        h = _html_of(p)
+        final = str(getattr(p, "url", "") or url)
+        if 200 <= st < 300 and h and not _challenged(h):
+            return {"status": st, "engine": "fast", "challenged": False,
+                    "finalUrl": final, "html": h[:_HTML_CAP]}
+    except Exception:
+        pass
+
+    # ── Tier 2/3: stealth browser (real Chrome), plain then solve_cloudflare ──
+    try:
+        from scrapling.fetchers import StealthyFetcher
+    except Exception:
+        return {"status": 0, "engine": None, "challenged": False, "finalUrl": url, "html": ""}
+    fn = getattr(StealthyFetcher, "async_fetch", None)
+    if fn is None:
+        return {"status": 0, "engine": None, "challenged": False, "finalUrl": url, "html": ""}
+
+    last = {"status": 0, "engine": "stealth", "challenged": True, "finalUrl": url, "html": ""}
+    for solve in (False, True):
+        base = {"headless": True, "timeout": 60000 if solve else min(ms, 25000)}
+        if solve:
+            base.update(solve_cloudflare=True, block_webrtc=True, disable_resources=False)
+        page = None
+        engine = ("real_chrome_cf" if solve else "real_chrome")
+        for use_real in (True, False):
+            kw = dict(base)
+            if use_real:
+                kw["real_chrome"] = True
+            else:
+                engine = ("chromium_cf" if solve else "chromium")
+            try:
+                page = await fn(url, **kw)
+                break
+            except Exception:
+                page = None
+                continue
+        if page is None:
+            continue
+        h = _html_of(page)
+        st = int(getattr(page, "status", 0) or 0)
+        final = str(getattr(page, "url", "") or url)
+        last = {"status": st, "engine": engine, "challenged": _challenged(h),
+                "finalUrl": final, "html": (h or "")[:_HTML_CAP]}
+        if h and not _challenged(h):
+            return last
+        if not solve and not _challenged(h):
+            break  # plain stealth returned something non-challenge-y but empty
+    return last
+
+
+async def _main() -> None:
+    try:
+        data = json.loads(sys.stdin.read() or "{}")
+    except Exception:
+        data = {}
+    url = str(data.get("url") or "").strip()
+    try:
+        budget = float(data.get("budget_seconds") or 40)
+    except Exception:
+        budget = 40.0
+    from urllib.parse import urlsplit
+    if not url or urlsplit(url).scheme != "https":
+        print(json.dumps({"status": 0, "engine": None, "challenged": False,
+                          "finalUrl": url, "html": "", "error": "https_only"}))
+        return
+    try:
+        out = await _fetch(url, budget)
+    except Exception as e:
+        out = {"status": 0, "engine": None, "challenged": False,
+               "finalUrl": url, "html": "", "error": str(e)[:200]}
+    print(json.dumps(out))
+
+
+if sys.platform.startswith("win"):
+    try:
+        asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+    except Exception:
+        pass
+asyncio.run(_main())

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.1.48",
+  "version": "1.1.50",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,
@@ -16,11 +16,12 @@
     "localRagIngest.py",
     "localRagRetrieve.py",
     "localDbProbe.py",
+    "localBacklinkFetch.py",
     "nltk_data/**",
     "README.md"
   ],
   "scripts": {
-    "build": "tsc && node -e \"const fs=require('fs'); fs.copyFileSync('localRagIngest.py','dist/localRagIngest.py'); fs.copyFileSync('localRagRetrieve.py','dist/localRagRetrieve.py'); fs.copyFileSync('localDbProbe.py','dist/localDbProbe.py'); fs.copyFileSync('src/assistantHost.py','dist/assistantHost.py')\"",
+    "build": "tsc && node -e \"const fs=require('fs'); fs.copyFileSync('localRagIngest.py','dist/localRagIngest.py'); fs.copyFileSync('localRagRetrieve.py','dist/localRagRetrieve.py'); fs.copyFileSync('localDbProbe.py','dist/localDbProbe.py'); fs.copyFileSync('localBacklinkFetch.py','dist/localBacklinkFetch.py'); fs.copyFileSync('src/assistantHost.py','dist/assistantHost.py')\"",
     "test": "node --test --import tsx \"src/**/*.test.ts\"",
     "prepublishOnly": "npm run build"
   },