@melaya/runner 1.1.49 → 1.1.51

package/dist/assistantHost.py

@@ -170,6 +170,8 @@ def _config_hash() -> str:
     pinned_tools = ",".join(sorted(t.lower() for t in raw_pinned.split(",") if t.strip()))
     if pinned_tools:
         fields.append(pinned_tools)
+    if os.environ.get("MEL_ASSISTANT_TELEMETRY_SURFACE") == "marketing":
+        fields.append("marketing-scoped-tools-v1")
     canon = "|".join(fields)
     return hashlib.sha256(canon.encode("utf-8")).hexdigest()[:32]
 
@@ -607,6 +609,7 @@ def _build_agent():
     # names are case-sensitive Python identifiers (unlike connector_services,
     # which are lowercased service ids). Registered ACTIVE at boot below so the
     # model calls them directly instead of search_tools/activate_tool.
+    marketing_tool_scope = os.environ.get("MEL_ASSISTANT_TELEMETRY_SURFACE") == "marketing"
     pinned_tool_names = [t.strip() for t in os.environ.get("MEL_ASSISTANT_PINNED_TOOLS", "").split(",") if t.strip()]
     # Self-arm the write-approval gate whenever connectors are active, so it can't
     # be silently OFF on an older runner build that didn't set this env. Writes
@@ -659,11 +662,12 @@ def _build_agent():
             include_categories=categories + core_categories + connector_services,
             budget=_budget,
             pinned_names=pinned_tool_names or None,
+            allowed_names=pinned_tool_names if marketing_tool_scope else None,
         )
     except Exception as exc:
         _log(f"toolkit build failed (connectors={connector_services}, core): {exc}; retrying melaya_agent only")
         try:
-            toolkit = build_toolkit(categories=["melaya_agent"])
+            toolkit = build_toolkit(names=[]) if marketing_tool_scope else build_toolkit(categories=["melaya_agent"])
         except Exception:
             toolkit = build_toolkit(names=[])
 
@@ -1302,6 +1306,7 @@ def main() -> int:
             except Exception:
                 _log("restore failed:\n" + traceback.format_exc())
             continue
+        os.environ["MEL_APPROVAL_CONVERSATION_ID"] = str(req.get("conversation_id") or "")
         turn_id = str(req.get("turnId") or "")
         message = str(req.get("message") or "")
         # Vision: base64 image blocks for THIS turn (validated server-side). Used

package/dist/browserBridge.js

@@ -214,6 +214,58 @@ async function humanType(page, text) {
         await page.waitForTimeout(Math.round(d));
     }
 }
+// Minimal, safe HTML rendering of plain text for the clipboard `text/html`
+// flavor: preserves paragraph and line breaks so a paste into Google Docs /
+// rich editors keeps structure instead of collapsing to one line.
+function htmlFromPlain(t) {
+    const esc = (s) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+    return t.split(/\n{2,}/).map((p) => `<p>${esc(p).replace(/\n/g, "<br>")}</p>`).join("") || "<p></p>";
+}
+// One-shot bulk text insert (the "paste" mode of input_text). Char-by-char
+// humanType is ~100ms/char (700s+ on an article), lands out of order in a live-
+// re-rendering editor, and never reaches a Google Docs <canvas> at all. This
+// writes the whole string to the clipboard (text/plain + text/html) and fires a
+// real Ctrl/Cmd+V, which every surface — <input>, <textarea>, contenteditable,
+// and the Docs canvas — imports in a single operation with formatting intact.
+// Returns "paste" on success, or null when the clipboard is unavailable
+// (permission / non-secure context) so the caller can fall back to CDP
+// Input.insertText. The caller has already focused the field and run the
+// secret-field / publish gates.
+async function pasteBulkText(page, text) {
+    try {
+        const origin = new URL(page.url()).origin;
+        if (origin && origin !== "null") {
+            await page.context().grantPermissions(["clipboard-read", "clipboard-write"], { origin }).catch(() => { });
+        }
+    }
+    catch { /* opaque / non-http origin: grantPermissions would throw */ }
+    const wrote = await page.evaluate(async (args) => {
+        try {
+            const nav = navigator;
+            const CI = globalThis.ClipboardItem;
+            if (nav.clipboard?.write && CI) {
+                await nav.clipboard.write([new CI({
+                        "text/plain": new Blob([args.t], { type: "text/plain" }),
+                        "text/html": new Blob([args.h], { type: "text/html" }),
+                    })]);
+                return true;
+            }
+            if (nav.clipboard?.writeText) {
+                await nav.clipboard.writeText(args.t);
+                return true;
+            }
+            return false;
+        }
+        catch {
+            return false;
+        }
+    }, { t: text, h: htmlFromPlain(text) }).catch(() => false);
+    if (wrote) {
+        await page.keyboard.press(process.platform === "darwin" ? "Meta+V" : "Control+V");
+        return "paste";
+    }
+    return null;
+}
 // Break a wheel scroll into a few smaller, jittered steps with pauses.
 async function humanScroll(page, dx, dy) {
     const steps = _randInt(3, 6);
@@ -1558,11 +1610,29 @@ export async function startBrowserBridge(opts) {
                             "Hand off to the human with browser_ask_user so they enter the secret; never type credentials.");
                     }
                     await humanClick(page, pt.x, pt.y);
-                    // Clear-then-type: select-all + humanized type preserves site key handlers.
+                    // Clear-then-type: select-all first so both paths REPLACE the field.
                     await page.keyboard.press(process.platform === "darwin" ? "Meta+A" : "Control+A").catch(() => { });
                     await humanPause(page, 40, 120);
-                    await humanType(page, text);
-                    return { typed: text.length };
+                    // Entry strategy. "human" = per-char cadence (anti-bot, short fields);
+                    // "paste" = one-shot bulk insert; omitted/"auto" = paste for long text
+                    // (char-by-char would blow the 20s tool timeout and scramble on a live
+                    // editor / never reach a Docs canvas). Short text keeps humanType so the
+                    // anti-bot cadence and site keystroke handlers still fire.
+                    const mode = String(args.mode ?? "auto");
+                    const usePaste = mode === "paste" || (mode !== "human" && text.length > 300);
+                    if (!usePaste) {
+                        await humanType(page, text);
+                        return { typed: text.length, via: "human" };
+                    }
+                    let via = await pasteBulkText(page, text);
+                    if (!via) {
+                        // Clipboard blocked: CDP whole-string insert (inputs/textarea/
+                        // contenteditable; not canvas). getCdp is in scope here.
+                        const cdp = await getCdp(rec, page);
+                        await cdpRace(cdp.send("Input.insertText", { text }), "Input.insertText");
+                        via = "insertText";
+                    }
+                    return { typed: text.length, via };
                 }
                 case "press_key": {
                     const key = String(args.key || "");

package/dist/browserGrantVerify.js

@@ -148,11 +148,11 @@ export const DEFAULT_CLOCK_TOLERANCE_SECONDS = 30;
  *  longer life than the issuer can mint is forged or corrupt, so it is
  *  rejected regardless of signature.
  *
- *  Raised 900s -> 4h alongside the server. Because this half ships in the
- *  runner, a server deployed ahead of a runner republish would mint grants
- *  this rejects with claims_invalid — publish the runner before, or at the
- *  same time as, the server change. */
-const MAX_LIFETIME_SECONDS = 4 * 60 * 60;
+ *  Raised 900s -> 4h -> 7d alongside the server (persistent week-long browser
+ *  sessions). Because this half ships in the runner, a server deployed ahead of
+ *  a runner republish would mint grants this rejects with claims_invalid —
+ *  publish the runner before, or at the same time as, the server change. */
+const MAX_LIFETIME_SECONDS = 7 * 24 * 60 * 60;
 function b64urlJson(part, what) {
     let buf;
     try {

package/dist/cli.js

@@ -15,6 +15,7 @@
  */
 import { Command } from "commander";
 import chalk from "chalk";
+import { reportInstallFailure } from "./installReport.js";
 import { connect } from "./connection.js";
 import { detectModels } from "./detect.js";
 // Logo colors: cyan → blue → indigo gradient (matching melaya.org)
@@ -63,6 +64,7 @@ async function main() {
         console.log(chalk.red(`  ✗ A Melaya runner is already running on this machine (pid ${lock.ownerPid}).`));
         console.log(chalk.gray("    Only one runner can run per machine. Stop the other one first"));
         console.log(chalk.gray(`    (e.g. \`kill ${lock.ownerPid}\`, or close its terminal), then start again.`));
+        await reportInstallFailure("single_instance_locked", { token: opts.token, server: opts.server });
         process.exit(1);
     }
     // Detect Python. Fast path: a supported host interpreter (3.10–3.12). If the
@@ -87,6 +89,10 @@ async function main() {
             console.log(chalk.gray("      macOS:   brew install python@3.12"));
             console.log(chalk.gray("      Linux:   sudo apt install python3.12 python3.12-venv    (or: pyenv install 3.12)"));
             console.log(chalk.gray("      Windows: winget install Python.Python.3.12"));
+            await reportInstallFailure("python_missing", {
+                token: opts.token, server: opts.server,
+                detail: py.foundVersion ? `host=${py.foundVersion}` : "none",
+            });
             process.exit(1);
         }
         python = provisioned;
@@ -185,7 +191,16 @@ async function bootstrap() {
     }
     await main();
 }
-bootstrap().catch((e) => {
+bootstrap().catch(async (e) => {
     console.error(chalk.red(`\n  Fatal: ${e.message}\n`));
+    // Token is parsed inside main(); recover it from argv so a crash BEFORE the
+    // socket still reports. Only the STAGE is sent — never the error text, which
+    // routinely carries file paths and usernames.
+    const tokArg = process.argv.find((a) => a.startsWith("--token"));
+    const srvArg = process.argv.find((a) => a.startsWith("--server"));
+    await reportInstallFailure("bootstrap_fatal", {
+        token: tokArg ? tokArg.replace(/^--token[= ]?/, "") : undefined,
+        server: srvArg ? srvArg.replace(/^--server[= ]?/, "") : undefined,
+    });
     process.exit(1);
 });

package/dist/connection.js

@@ -303,11 +303,20 @@ export async function connect(opts) {
         void _prewarmAssistantEnv();
         console.log(chalk.gray("  Waiting for pipeline runs...\n"));
     });
+    // A runner that never connects is invisible to the funnel: last_seen stays
+    // NULL and the reason dies on the user's machine. Report the stage before
+    // exiting — auth-rejected and unreachable are very different fixes.
+    let reportedConnectFailure = false;
     socket.on("connect_error", (err) => {
         spinner.fail(chalk.red(`Connection failed: ${err.message}`));
-        if (err.message.includes("Authentication")) {
+        const authRejected = err.message.includes("Authentication");
+        if (!reportedConnectFailure) {
+            reportedConnectFailure = true;
+            void import("./installReport.js").then((m) => m.reportInstallFailure(authRejected ? "socket_auth_rejected" : "socket_unreachable", { token: opts.token, server: opts.serverUrl })).catch(() => { });
+        }
+        if (authRejected) {
             console.log(chalk.yellow("  Token may be expired or revoked. Generate a new one in Settings → Connectors."));
-            process.exit(1);
+            setTimeout(() => process.exit(1), 300); // let the beacon leave the box
         }
     });
     socket.on("disconnect", (reason) => {
@@ -1884,6 +1893,7 @@ export async function connect(opts) {
                 message: payload.message,
                 hitl_mode: hitlMode,
                 static_context: typeof payload.staticContext === "string" ? payload.staticContext : "",
+                conversation_id: String(payload.sessionId || "").split(":").slice(1).join(":"),
                 telemetry_surface: typeof payload.telemetrySurface === "string" ? payload.telemetrySurface : "assistant",
                 // Per-turn browser grant: forwarded to the host as the raw compact
                 // JWS string so the Python browser toolkit can verify it again
@@ -2473,6 +2483,124 @@ export async function connect(opts) {
             reply(false, undefined, e?.message || String(e));
         }
     });
+    // ── Residential-IP page fetch (Melaya backlink presence check) ──────────────
+    // The Melaya cloud runs from a datacenter IP that Cloudflare "managed" Turnstile
+    // forces into an interactive challenge no headless engine clears. THIS machine
+    // is on a residential IP, where the same real-browser fetch passes in seconds.
+    // The server hands us ONE public https url; we fetch it with the local Scrapling
+    // ladder (localBacklinkFetch.py) and return the HTML for the server's presence
+    // scan. We receive only the url — no secrets, no internal hosts, no catalog data.
+    socket.on("runner:backlink-fetch", async (payload, ack) => {
+        const rid = payload?.requestId;
+        if (!rid)
+            return;
+        ack?.({ ok: true });
+        const reply = (r) => socket.emit("runner:backlink-fetch-result", {
+            requestId: rid,
+            ok: !r.error && !!r.html && !r.challenged,
+            status: r.status ?? 0,
+            engine: r.engine ?? null,
+            challenged: !!r.challenged,
+            html: r.html ?? "",
+            finalUrl: r.finalUrl ?? payload.url,
+            error: r.error,
+        });
+        // HTTPS-only, public url — defence in depth (the server already validated it).
+        let u = null;
+        try {
+            u = new URL(payload.url);
+        }
+        catch {
+            u = null;
+        }
+        if (!u || u.protocol !== "https:") {
+            reply({ error: "invalid_url" });
+            return;
+        }
+        try {
+            const { ensurePythonEnv } = await import("./pythonEnv.js");
+            const { getLocalSharedVersion } = await import("./sharedVendor.js");
+            const localVersion = getLocalSharedVersion();
+            if (!localVersion) {
+                reply({ error: "runner python runtime still setting up" });
+                return;
+            }
+            const env = await ensurePythonEnv(opts.pythonPath, localVersion, (m) => { if (opts.verbose)
+                console.log(chalk.gray(`  [backlink-fetch venv] ${m}`)); });
+            if (!env.ok) {
+                reply({ error: `venv bootstrap failed: ${env.reason}` });
+                return;
+            }
+            const { existsSync, copyFileSync, mkdirSync } = await import("fs");
+            const workDir = join(tmpdir(), `melaya-blf-${Date.now()}`);
+            mkdirSync(workDir, { recursive: true });
+            const candidates = [
+                join(__dirname, "localBacklinkFetch.py"),
+                join(__dirname, "..", "localBacklinkFetch.py"),
+            ];
+            let found = "";
+            for (const c of candidates) {
+                if (existsSync(c)) {
+                    found = c;
+                    break;
+                }
+            }
+            if (!found) {
+                reply({ error: "localBacklinkFetch.py not found — update @melaya/runner" });
+                return;
+            }
+            const staged = join(workDir, "localBacklinkFetch.py");
+            copyFileSync(found, staged);
+            const certBundle = (await import("./pythonEnv.js")).getCertBundlePath();
+            const sslEnv = certBundle
+                ? { SSL_CERT_FILE: certBundle, REQUESTS_CA_BUNDLE: certBundle } : {};
+            const budget = Math.max(6, Math.min(75, Math.floor(payload.budgetSeconds || 40)));
+            const proc = spawn(env.pythonPath, ["-u", staged], {
+                env: { ...process.env, ...sslEnv },
+                cwd: workDir,
+            });
+            let stdout = "";
+            let stderr = "";
+            let done = false;
+            const finish = (r) => { if (done)
+                return; done = true; reply(r); };
+            // Hard wall-clock cap slightly above the fetch budget so a wedged browser
+            // can't hold the session; the server has its own (budget+8s) timeout too.
+            const killTimer = setTimeout(() => {
+                try {
+                    proc.kill();
+                }
+                catch { /* already gone */ }
+                finish({ error: "fetch timed out" });
+            }, (budget + 10) * 1000);
+            proc.stdout.on("data", (d) => { stdout += d.toString("utf-8"); });
+            proc.stderr.on("data", (d) => { stderr += d.toString("utf-8"); });
+            proc.on("error", (err) => { clearTimeout(killTimer); finish({ error: err?.message || String(err) }); });
+            proc.on("close", (code) => {
+                clearTimeout(killTimer);
+                const jsonLine = stdout.split(/\r?\n/).map(s => s.trim()).filter(Boolean).reverse()[0] || "";
+                let parsed = null;
+                try {
+                    parsed = JSON.parse(jsonLine);
+                }
+                catch { /* handled below */ }
+                if (parsed && typeof parsed === "object") {
+                    finish({ status: parsed.status, engine: parsed.engine, challenged: !!parsed.challenged, html: parsed.html || "", finalUrl: parsed.finalUrl });
+                }
+                else {
+                    finish({ error: `fetch exited ${code ?? "?"}${stderr ? `: ${stderr.slice(-200)}` : ""}` });
+                }
+            });
+            try {
+                proc.stdin.write(JSON.stringify({ url: u.href, budget_seconds: budget }));
+                proc.stdin.end();
+            }
+            catch { /* proc.on('error') handles it */ }
+        }
+        catch (e) {
+            reply({ error: e?.message || String(e) });
+        }
+    });
     socket.on("linkedin:start-login", async (req) => {
         const sid = req?.session_id;
         if (!sid)

package/dist/installReport.d.ts

@@ -0,0 +1,12 @@
+/** The point at which the runner gave up. Keep in lockstep with the server's
+ *  META_ALLOW entry for `runner_install_failed`. */
+export type InstallFailStage = "single_instance_locked" | "python_missing" | "socket_auth_rejected" | "socket_unreachable" | "bootstrap_fatal";
+/**
+ * Best-effort telemetry for a runner that is about to exit without connecting.
+ * Awaited only so the process does not die mid-flight; it resolves either way.
+ */
+export declare function reportInstallFailure(stage: InstallFailStage, opts?: {
+    token?: string;
+    server?: string;
+    detail?: string;
+}): Promise<void>;

package/dist/installReport.js

@@ -0,0 +1,76 @@
+// packages/runner/src/installReport.ts
+//
+// Report WHY the runner never came up.
+//
+// The activation funnel could see that runner tokens were being issued and that
+// nobody ever connected — measured 2026-09-14 over 48h: **9 tokens issued, 0
+// connections, 0 local runs** — but not one reason, because every failure path
+// here ends in `process.exit(1)` on the user's own machine and the server never
+// hears about it. `runner_tokens.last_seen` stays NULL and that is the entire
+// signal. 9 → 0 is not actionable; "6 of 9 had no usable Python" is.
+//
+// This closes that gap with a single fire-and-forget POST before the exit.
+//
+// Rules it must obey, because it runs while something is already going wrong:
+//   - NEVER throws, never blocks shutdown, hard 3s timeout.
+//   - Sends a STAGE CODE, not a message. No file paths, no usernames, no
+//     hostnames, no error strings — those carry the user's directory layout and
+//     sometimes their name. The stage plus a coarse platform is what makes the
+//     number actionable; the rest is someone's home directory.
+//   - Honours MELAYA_RUNNER_NO_TELEMETRY=1.
+//   - Authenticated by the runner token the user already passed, so the server
+//     can attribute it to the account without any new credential.
+let alreadySent = false;
+/**
+ * Best-effort telemetry for a runner that is about to exit without connecting.
+ * Awaited only so the process does not die mid-flight; it resolves either way.
+ */
+export async function reportInstallFailure(stage, opts = {}) {
+    try {
+        if (alreadySent)
+            return; // one report per launch
+        alreadySent = true;
+        if (process.env.MELAYA_RUNNER_NO_TELEMETRY === "1")
+            return;
+        const token = String(opts.token || "").trim();
+        if (!token)
+            return; // nothing to attribute it to
+        const base = String(opts.server || "https://api.melaya.org").replace(/\/+$/, "");
+        // Version is read the same way connection.ts does; a dev tree without a
+        // package.json simply reports "0.0.0".
+        let version = "0.0.0";
+        try {
+            const { readFileSync } = await import("node:fs");
+            const { join, dirname } = await import("node:path");
+            const { fileURLToPath } = await import("node:url");
+            const here = dirname(fileURLToPath(import.meta.url));
+            version = String(JSON.parse(readFileSync(join(here, "..", "package.json"), "utf-8")).version || "0.0.0");
+        }
+        catch { /* non-fatal */ }
+        const ac = new AbortController();
+        const timer = setTimeout(() => ac.abort(), 3000);
+        try {
+            await fetch(`${base}/api/v1/runner/install-report`, {
+                method: "POST",
+                headers: { "content-type": "application/json", authorization: `Bearer ${token}` },
+                // `detail` is a SHORT, bounded, non-identifying hint (e.g. the
+                // Python version actually found) — never a path or an error dump.
+                body: JSON.stringify({
+                    stage,
+                    version,
+                    platform: process.platform, // darwin | win32 | linux
+                    arch: process.arch,
+                    node: process.versions.node,
+                    ...(opts.detail ? { detail: String(opts.detail).slice(0, 60) } : {}),
+                }),
+                signal: ac.signal,
+            });
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    catch {
+        /* The runner is already failing. Telemetry must never make that worse. */
+    }
+}

package/dist/localBacklinkFetch.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Residential-IP page fetch for the Melaya backlink presence check.
+
+The Melaya cloud runs from a datacenter IP that Cloudflare "managed" Turnstile
+forces into an interactive challenge no headless engine clears. The user's
+runner is on a RESIDENTIAL IP, where the same real-browser fetch passes in
+seconds. So the server hands us ONE public https url; we fetch it with the
+local Scrapling stack and print the result as a single JSON line on stdout:
+
+    {"status": int, "engine": str|null, "challenged": bool,
+     "finalUrl": str, "html": str}   # html capped
+
+Input (stdin, JSON): {"url": "...", "budget_seconds": 40}
+
+Self-contained on purpose (mirrors localDbProbe.py): it imports only Scrapling
+from the runner venv, never the shared tool bundle. HTTPS-only; the server
+already validated the url, this is defence in depth. Never fetches anything but
+the url it is given — no internal hosts, no secrets, no catalog data.
+"""
+import asyncio
+import json
+import sys
+
+_HTML_CAP = 600_000
+_CHALLENGE_MARKERS = (
+    "just a moment", "cf_chl", "cf-challenge", "checking your browser",
+    "enable javascript and cookies", "attention required",
+)
+
+
+def _html_of(page) -> str:
+    for attr in ("html_content", "body", "content"):
+        try:
+            v = getattr(page, attr)
+            if callable(v):
+                v = v()
+            if v:
+                return v if isinstance(v, str) else str(v)
+        except Exception:
+            pass
+    try:
+        return str(page)
+    except Exception:
+        return ""
+
+
+def _challenged(h: str) -> bool:
+    h = (h or "").lower()
+    return any(m in h for m in _CHALLENGE_MARKERS)
+
+
+async def _fetch(url: str, budget: float) -> dict:
+    ms = int(max(6.0, min(75.0, budget)) * 1000)
+    # ── Tier 1: TLS-impersonating async GET (no browser) ──
+    try:
+        from scrapling.fetchers import AsyncFetcher
+        p = await AsyncFetcher.get(url, timeout=min(ms, 15000), impersonate="chrome",
+                                   stealthy_headers=True, follow_redirects=True)
+        st = int(getattr(p, "status", 0) or 0)
+        h = _html_of(p)
+        final = str(getattr(p, "url", "") or url)
+        if 200 <= st < 300 and h and not _challenged(h):
+            return {"status": st, "engine": "fast", "challenged": False,
+                    "finalUrl": final, "html": h[:_HTML_CAP]}
+    except Exception:
+        pass
+
+    # ── Tier 2/3: stealth browser (real Chrome), plain then solve_cloudflare ──
+    try:
+        from scrapling.fetchers import StealthyFetcher
+    except Exception:
+        return {"status": 0, "engine": None, "challenged": False, "finalUrl": url, "html": ""}
+    fn = getattr(StealthyFetcher, "async_fetch", None)
+    if fn is None:
+        return {"status": 0, "engine": None, "challenged": False, "finalUrl": url, "html": ""}
+
+    last = {"status": 0, "engine": "stealth", "challenged": True, "finalUrl": url, "html": ""}
+    for solve in (False, True):
+        base = {"headless": True, "timeout": 60000 if solve else min(ms, 25000)}
+        if solve:
+            base.update(solve_cloudflare=True, block_webrtc=True, disable_resources=False)
+        page = None
+        engine = ("real_chrome_cf" if solve else "real_chrome")
+        for use_real in (True, False):
+            kw = dict(base)
+            if use_real:
+                kw["real_chrome"] = True
+            else:
+                engine = ("chromium_cf" if solve else "chromium")
+            try:
+                page = await fn(url, **kw)
+                break
+            except Exception:
+                page = None
+                continue
+        if page is None:
+            continue
+        h = _html_of(page)
+        st = int(getattr(page, "status", 0) or 0)
+        final = str(getattr(page, "url", "") or url)
+        last = {"status": st, "engine": engine, "challenged": _challenged(h),
+                "finalUrl": final, "html": (h or "")[:_HTML_CAP]}
+        if h and not _challenged(h):
+            return last
+        if not solve and not _challenged(h):
+            break  # plain stealth returned something non-challenge-y but empty
+    return last
+
+
+async def _main() -> None:
+    try:
+        data = json.loads(sys.stdin.read() or "{}")
+    except Exception:
+        data = {}
+    url = str(data.get("url") or "").strip()
+    try:
+        budget = float(data.get("budget_seconds") or 40)
+    except Exception:
+        budget = 40.0
+    from urllib.parse import urlsplit
+    if not url or urlsplit(url).scheme != "https":
+        print(json.dumps({"status": 0, "engine": None, "challenged": False,
+                          "finalUrl": url, "html": "", "error": "https_only"}))
+        return
+    try:
+        out = await _fetch(url, budget)
+    except Exception as e:
+        out = {"status": 0, "engine": None, "challenged": False,
+               "finalUrl": url, "html": "", "error": str(e)[:200]}
+    print(json.dumps(out))
+
+
+if sys.platform.startswith("win"):
+    try:
+        asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+    except Exception:
+        pass
+asyncio.run(_main())

package/localBacklinkFetch.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Residential-IP page fetch for the Melaya backlink presence check.
+
+The Melaya cloud runs from a datacenter IP that Cloudflare "managed" Turnstile
+forces into an interactive challenge no headless engine clears. The user's
+runner is on a RESIDENTIAL IP, where the same real-browser fetch passes in
+seconds. So the server hands us ONE public https url; we fetch it with the
+local Scrapling stack and print the result as a single JSON line on stdout:
+
+    {"status": int, "engine": str|null, "challenged": bool,
+     "finalUrl": str, "html": str}   # html capped
+
+Input (stdin, JSON): {"url": "...", "budget_seconds": 40}
+
+Self-contained on purpose (mirrors localDbProbe.py): it imports only Scrapling
+from the runner venv, never the shared tool bundle. HTTPS-only; the server
+already validated the url, this is defence in depth. Never fetches anything but
+the url it is given — no internal hosts, no secrets, no catalog data.
+"""
+import asyncio
+import json
+import sys
+
+_HTML_CAP = 600_000
+_CHALLENGE_MARKERS = (
+    "just a moment", "cf_chl", "cf-challenge", "checking your browser",
+    "enable javascript and cookies", "attention required",
+)
+
+
+def _html_of(page) -> str:
+    for attr in ("html_content", "body", "content"):
+        try:
+            v = getattr(page, attr)
+            if callable(v):
+                v = v()
+            if v:
+                return v if isinstance(v, str) else str(v)
+        except Exception:
+            pass
+    try:
+        return str(page)
+    except Exception:
+        return ""
+
+
+def _challenged(h: str) -> bool:
+    h = (h or "").lower()
+    return any(m in h for m in _CHALLENGE_MARKERS)
+
+
+async def _fetch(url: str, budget: float) -> dict:
+    ms = int(max(6.0, min(75.0, budget)) * 1000)
+    # ── Tier 1: TLS-impersonating async GET (no browser) ──
+    try:
+        from scrapling.fetchers import AsyncFetcher
+        p = await AsyncFetcher.get(url, timeout=min(ms, 15000), impersonate="chrome",
+                                   stealthy_headers=True, follow_redirects=True)
+        st = int(getattr(p, "status", 0) or 0)
+        h = _html_of(p)
+        final = str(getattr(p, "url", "") or url)
+        if 200 <= st < 300 and h and not _challenged(h):
+            return {"status": st, "engine": "fast", "challenged": False,
+                    "finalUrl": final, "html": h[:_HTML_CAP]}
+    except Exception:
+        pass
+
+    # ── Tier 2/3: stealth browser (real Chrome), plain then solve_cloudflare ──
+    try:
+        from scrapling.fetchers import StealthyFetcher
+    except Exception:
+        return {"status": 0, "engine": None, "challenged": False, "finalUrl": url, "html": ""}
+    fn = getattr(StealthyFetcher, "async_fetch", None)
+    if fn is None:
+        return {"status": 0, "engine": None, "challenged": False, "finalUrl": url, "html": ""}
+
+    last = {"status": 0, "engine": "stealth", "challenged": True, "finalUrl": url, "html": ""}
+    for solve in (False, True):
+        base = {"headless": True, "timeout": 60000 if solve else min(ms, 25000)}
+        if solve:
+            base.update(solve_cloudflare=True, block_webrtc=True, disable_resources=False)
+        page = None
+        engine = ("real_chrome_cf" if solve else "real_chrome")
+        for use_real in (True, False):
+            kw = dict(base)
+            if use_real:
+                kw["real_chrome"] = True
+            else:
+                engine = ("chromium_cf" if solve else "chromium")
+            try:
+                page = await fn(url, **kw)
+                break
+            except Exception:
+                page = None
+                continue
+        if page is None:
+            continue
+        h = _html_of(page)
+        st = int(getattr(page, "status", 0) or 0)
+        final = str(getattr(page, "url", "") or url)
+        last = {"status": st, "engine": engine, "challenged": _challenged(h),
+                "finalUrl": final, "html": (h or "")[:_HTML_CAP]}
+        if h and not _challenged(h):
+            return last
+        if not solve and not _challenged(h):
+            break  # plain stealth returned something non-challenge-y but empty
+    return last
+
+
+async def _main() -> None:
+    try:
+        data = json.loads(sys.stdin.read() or "{}")
+    except Exception:
+        data = {}
+    url = str(data.get("url") or "").strip()
+    try:
+        budget = float(data.get("budget_seconds") or 40)
+    except Exception:
+        budget = 40.0
+    from urllib.parse import urlsplit
+    if not url or urlsplit(url).scheme != "https":
+        print(json.dumps({"status": 0, "engine": None, "challenged": False,
+                          "finalUrl": url, "html": "", "error": "https_only"}))
+        return
+    try:
+        out = await _fetch(url, budget)
+    except Exception as e:
+        out = {"status": 0, "engine": None, "challenged": False,
+               "finalUrl": url, "html": "", "error": str(e)[:200]}
+    print(json.dumps(out))
+
+
+if sys.platform.startswith("win"):
+    try:
+        asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+    except Exception:
+        pass
+asyncio.run(_main())

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.1.49",
+  "version": "1.1.51",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,
@@ -16,11 +16,12 @@
     "localRagIngest.py",
     "localRagRetrieve.py",
     "localDbProbe.py",
+    "localBacklinkFetch.py",
     "nltk_data/**",
     "README.md"
   ],
   "scripts": {
-    "build": "tsc && node -e \"const fs=require('fs'); fs.copyFileSync('localRagIngest.py','dist/localRagIngest.py'); fs.copyFileSync('localRagRetrieve.py','dist/localRagRetrieve.py'); fs.copyFileSync('localDbProbe.py','dist/localDbProbe.py'); fs.copyFileSync('src/assistantHost.py','dist/assistantHost.py')\"",
+    "build": "tsc && node -e \"const fs=require('fs'); fs.copyFileSync('localRagIngest.py','dist/localRagIngest.py'); fs.copyFileSync('localRagRetrieve.py','dist/localRagRetrieve.py'); fs.copyFileSync('localDbProbe.py','dist/localDbProbe.py'); fs.copyFileSync('localBacklinkFetch.py','dist/localBacklinkFetch.py'); fs.copyFileSync('src/assistantHost.py','dist/assistantHost.py')\"",
     "test": "node --test --import tsx \"src/**/*.test.ts\"",
     "prepublishOnly": "npm run build"
   },