@melaya/runner 1.1.16 → 1.1.18

package/dist/assistantHost.py

@@ -272,17 +272,25 @@ def _apply_browser_turn_grant(req) -> bool:
     present (injected by the runner's per-turn bridge registration). These are
     purged at turn end alongside the grant (see _purge_browser_turn_grant)."""
     grant = str((req or {}).get("browser_grant") or "") if isinstance(req, dict) else ""
+    # Front 3: EXTENSION-target proxy turn — no client-side grant; the host's
+    # browser tools call the server bridge (MEL_BROWSER_URL/TOKEN) which holds the
+    # grant and runs the action against the user's real tab.
+    ext_proxy = bool((req or {}).get("browser_ext_proxy")) if isinstance(req, dict) else False
     target_ref = str((req or {}).get("browser_target_ref") or "") if isinstance(req, dict) else ""
-    if grant and _browser_capable():
-        os.environ["MEL_BROWSER_TURN_GRANT"] = grant
+    bridge_url = str((req or {}).get("mel_browser_url") or "") if isinstance(req, dict) else ""
+    bridge_token = str((req or {}).get("mel_browser_token") or "") if isinstance(req, dict) else ""
+    if (grant or ext_proxy) and _browser_capable():
+        if grant:
+            os.environ["MEL_BROWSER_TURN_GRANT"] = grant
+        else:
+            os.environ.pop("MEL_BROWSER_TURN_GRANT", None)  # proxy: grant stays server-side
         if target_ref:
             os.environ["MEL_BROWSER_TURN_TARGET_REF"] = target_ref
         else:
             os.environ.pop("MEL_BROWSER_TURN_TARGET_REF", None)
-        # Per-turn bridge URL + token (assistant path only; pipeline path has
-        # these in the spawn env already). Apply only when present on the frame.
-        bridge_url = str((req or {}).get("mel_browser_url") or "") if isinstance(req, dict) else ""
-        bridge_token = str((req or {}).get("mel_browser_token") or "") if isinstance(req, dict) else ""
+        # Per-turn bridge URL + token (assistant path). For the runner target this
+        # is the runner's local bridge; for the extension proxy it is the server's
+        # /ext-bridge. Apply only when present on the frame.
         if bridge_url:
             os.environ["MEL_BROWSER_URL"] = bridge_url
         if bridge_token:
@@ -704,7 +712,18 @@ def _register_stream_hooks(agent) -> None:
                 return
             prev = _stream["lens"].get(mid, 0)
             if len(text) > prev:
-                _emit(tid, "delta", content=text[prev:])
+                # ReAct emits one message PER iteration: the running chain-of-thought
+                # ("Let me...", "Actually...", the plan before each tool call). That
+                # is REASONING, not the answer. Streaming it as `delta` dumped every
+                # round's raw thought into the chat bubble with no separator between
+                # rounds ("...page content.I notice..."), which read as broken run-on
+                # sentences. Route it to `think` instead (the ephemeral thinking
+                # strip) so it MIRRORS the cloud provider mapping (thinking_delta ->
+                # think, answer -> text). The clean authoritative answer still lands
+                # in the bubble via the end-of-turn `text` event (_run_turn), so the
+                # message reads like one professional reply, reasoning stays in its
+                # own channel, and tool chips show the live steps.
+                _emit(tid, "think", content=text[prev:])
                 _stream["lens"][mid] = len(text)
         except Exception:
             pass

package/dist/browserBridge.d.ts

@@ -10,6 +10,8 @@ export interface BrowserRunSpec {
     space?: SpaceSpec;
     codeMode?: boolean;
     headless?: boolean;
+    serverBase?: string;
+    runAuthHeader?: string;
 }
 export interface BrowserBridge {
     url: string;

package/dist/browserBridge.js

@@ -38,6 +38,7 @@ import { buildOriginPolicy, evaluateUrlResolved, checkEffect, enforceOnContext,
 import { SessionManager, SessionError, spaceUserDataDir, } from "./sessionManager.js";
 import { ensureEngine } from "./browserProvisioner.js";
 import { runSandboxedScript } from "./codeWorker.js";
+import { classifyControlEffect } from "./publishGate.js";
 // ---------------------------------------------------------------------
 // Limits and constants
 // ---------------------------------------------------------------------
@@ -914,6 +915,165 @@ export async function startBrowserBridge(opts) {
             return true;
         return false;
     }
+    // ── Route 1 publish gate (phone detectPublish mirror) ───────────────
+    // The bridge is the only place that reads the clicked control's TRUSTED
+    // accessible name. For click/dblclick/tap we read that name, classify it
+    // into a consequential effect ("publish"|"purchase"), and, when the grant
+    // does not permit the effect unattended, stage a server approval and refuse
+    // to execute until it is approved (browser.py's poll loop re-issues the same
+    // act, which re-detects and re-calls the idempotent endpoint).
+    /** Read the accessible name of the element at (x,y) via page.evaluate. Walks
+     *  from the element at the point up to its closest clickable ancestor
+     *  (button, a, [role=button], [type=submit]) and reads, in precedence order:
+     *  aria-label, else innerText/textContent, else value, else title. Returns a
+     *  lowercased + trimmed string (empty when nothing is readable). Never
+     *  throws: on any page error it returns "" so the caller treats it as
+     *  "no match" and runs the action normally. */
+    async function readAccessibleNameAtPoint(page, x, y) {
+        try {
+            const raw = await page.evaluate(([px, py]) => {
+                const clickableSel = "button, a, [role=button], [type=submit]";
+                let el = document.elementFromPoint(px, py);
+                if (!el)
+                    return "";
+                // Prefer the closest clickable ancestor (or self) so a click on an
+                // inner <span> still resolves the button's own label.
+                const clickable = el.closest(clickableSel) || el;
+                el = clickable;
+                const aria = el.getAttribute && el.getAttribute("aria-label");
+                if (aria && aria.trim())
+                    return aria;
+                const txt = (el.innerText || el.textContent || "").trim();
+                if (txt)
+                    return txt;
+                const val = el.value;
+                if (val && String(val).trim())
+                    return String(val);
+                const title = el.getAttribute && el.getAttribute("title");
+                if (title && title.trim())
+                    return title;
+                return "";
+            }, [x, y]);
+            return String(raw || "").toLowerCase().trim();
+        }
+        catch {
+            return "";
+        }
+    }
+    /** Stage a human approval on the server for a consequential click/tap and
+     *  return the decision. Reuses the SAME server base URL + Bearer run token
+     *  frame posting uses (spec.serverBase + spec.runAuthHeader, derived in
+     *  connection.ts from opts.serverUrl + opts.token). Fails closed: any
+     *  missing wiring, network error, non-2xx status, or unparseable body maps
+     *  to "fail_closed" so the caller refuses to run the action. */
+    async function stagePublishApproval(reg, args) {
+        const serverBase = String(reg.spec.serverBase || "").replace(/\/$/, "");
+        const authHeader = String(reg.spec.runAuthHeader || "");
+        if (!serverBase || !authHeader)
+            return "fail_closed";
+        const grant = reg.spec.grant;
+        const body = JSON.stringify({
+            session_id: grant.browserSession,
+            run_id: reg.spec.runId,
+            target_ref: grant.target.ref,
+            top_origin: args.origin,
+            effect: args.effect,
+            kind: args.kind,
+            ref: args.ref,
+        });
+        return await new Promise((resolve) => {
+            try {
+                const u = new URL(serverBase + "/api/v1/browser/runner/approval");
+                const isHttps = u.protocol === "https:";
+                const req = (isHttps ? httpsRequest : httpRequest)({
+                    hostname: u.hostname,
+                    port: u.port || (isHttps ? 443 : 80),
+                    path: u.pathname + u.search,
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json",
+                        "Content-Length": Buffer.byteLength(body),
+                        "Authorization": authHeader,
+                    },
+                }, (res) => {
+                    const status = res.statusCode || 0;
+                    const chunks = [];
+                    res.on("data", (c) => chunks.push(Buffer.from(c)));
+                    res.on("end", () => {
+                        if (status < 200 || status >= 300) {
+                            resolve("fail_closed");
+                            return;
+                        }
+                        try {
+                            const parsed = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+                            const decision = String(parsed?.decision || "");
+                            if (decision === "approved" ||
+                                decision === "pending" ||
+                                decision === "rejected" ||
+                                decision === "canceled") {
+                                resolve(decision);
+                            }
+                            else {
+                                resolve("fail_closed");
+                            }
+                        }
+                        catch {
+                            resolve("fail_closed");
+                        }
+                    });
+                });
+                req.on("error", () => resolve("fail_closed"));
+                req.end(body);
+            }
+            catch {
+                resolve("fail_closed");
+            }
+        });
+    }
+    /** Escalate a resolved click/tap from its trusted accessible name and, when
+     *  the effect is consequential and not permitted unattended, stage an
+     *  approval and throw the contract BridgeError until it is approved. Returns
+     *  normally (falls through to execution) when there is nothing to gate or
+     *  the approval came back "approved". Applies to click/dblclick/tap only;
+     *  callers must not invoke it for hover. */
+    async function enforcePublishGate(reg, page, kind, name, ref) {
+        const escalated = classifyControlEffect(name);
+        if (!escalated)
+            return; // not a publish/purchase control: run normally.
+        // If the grant permits this effect unattended, do not gate.
+        const permitted = checkEffect(escalated, reg.spec.grant);
+        if (permitted.allowed)
+            return;
+        // SAFE mode: not permitted unattended. Stage a server approval.
+        let origin = "";
+        try {
+            origin = new URL(page.url()).origin;
+        }
+        catch {
+            origin = page.url();
+        }
+        const approvalKind = kind === "tap" ? "tap" : "click";
+        const decision = await stagePublishApproval(reg, {
+            effect: escalated,
+            kind: approvalKind,
+            ref,
+            origin,
+        });
+        if (decision === "approved")
+            return; // fall through to execution.
+        if (decision === "pending") {
+            throw new BridgeError("approval_required", "Waiting for your approval to " + escalated + " on " + origin +
+                ". A card is showing in Melaya; approve it and this will proceed.");
+        }
+        if (decision === "rejected") {
+            throw new BridgeError("approval_required", "You declined this action, so it was not run. Ask the user what to do instead.");
+        }
+        if (decision === "canceled") {
+            throw new BridgeError("canceled", "The approval was canceled.");
+        }
+        // fail_closed (missing wiring, network/HTTP error, bad body).
+        throw new BridgeError("approval_required", "Could not reach the approval service; not running this consequential action until it is approved.");
+    }
     async function performAct(reg, args) {
         const kind = String(args.kind || "");
         if (!(kind in KIND_MIN_EFFECT)) {
@@ -989,6 +1149,12 @@ export async function startBrowserBridge(opts) {
                                 "with a ref instead of a text label.");
                         }
                         const pt = { x: box.x + box.width / 2, y: box.y + box.height / 2 };
+                        // Publish gate: when text located the element, that text IS the
+                        // control's accessible name (no extra page read needed). hover is
+                        // never gated.
+                        if (kind !== "hover") {
+                            await enforcePublishGate(reg, page, kind, label, label);
+                        }
                         if (kind === "hover")
                             await humanMove(page, pt.x, pt.y);
                         else
@@ -996,6 +1162,12 @@ export async function startBrowserBridge(opts) {
                         return { at: { x: Math.round(pt.x), y: Math.round(pt.y) }, matched_text: label };
                     }
                     const pt = await resolveActionPoint(reg, rec, lease, args);
+                    // Publish gate for the ref/coords branch: read the trusted accessible
+                    // name at the resolved point and escalate. hover is never gated.
+                    if (kind !== "hover") {
+                        const name = await readAccessibleNameAtPoint(page, pt.x, pt.y);
+                        await enforcePublishGate(reg, page, kind, name, String(args.ref ?? `${Math.round(pt.x)},${Math.round(pt.y)}`));
+                    }
                     if (kind === "hover")
                         await humanMove(page, pt.x, pt.y);
                     else
@@ -1007,6 +1179,9 @@ export async function startBrowserBridge(opts) {
                     // browser.py sends {x, y} as 0..1 fractions or CSS px.
                     // resolveActionPoint already handles the fraction/px heuristic.
                     const pt = await resolveActionPoint(reg, rec, lease, args);
+                    // Publish gate: read the trusted accessible name at the tapped point.
+                    const name = await readAccessibleNameAtPoint(page, pt.x, pt.y);
+                    await enforcePublishGate(reg, page, kind, name, String(args.ref ?? `${Math.round(pt.x)},${Math.round(pt.y)}`));
                     await humanClick(page, pt.x, pt.y);
                     return { at: { x: Math.round(pt.x), y: Math.round(pt.y) } };
                 }

package/dist/connection.js

@@ -454,6 +454,10 @@ export async function connect(opts) {
                         space: payload.browser?.space ?? { kind: "ephemeral" },
                         codeMode: payload.browser?.codeMode === true,
                         headless: payload.browser?.headless === true,
+                        // Route 1 publish gate: same server base + Bearer run token the
+                        // frame POST path uses (ws/wss -> http/https).
+                        serverBase: opts.serverUrl.replace(/^wss?:\/\//, (m) => (m === "wss://" ? "https://" : "http://")),
+                        runAuthHeader: `Bearer ${opts.token}`,
                     };
                     const { token } = bridge.registerRun(spec);
                     browserToken = token;
@@ -1541,6 +1545,10 @@ export async function connect(opts) {
                             space: { kind: "ephemeral" },
                             codeMode: false,
                             headless: false,
+                            // Route 1 publish gate: same server base + Bearer run token the
+                            // frame POST path uses (ws/wss -> http/https).
+                            serverBase: opts.serverUrl.replace(/^wss?:\/\//, (m) => (m === "wss://" ? "https://" : "http://")),
+                            runAuthHeader: `Bearer ${opts.token}`,
                         };
                         const { token } = bridge.registerRun(spec);
                         turnBridgeRunId = turnRunId;
@@ -1569,6 +1577,16 @@ export async function connect(opts) {
                 return;
             }
         }
+        else if (payload.browserExtProxy && payload.browserExtProxy.url && payload.browserExtProxy.token) {
+            // Front 3: EXTENSION target. No grant verification / local bridge run — the
+            // host's browser_* tools POST to the server bridge (Bearer proxy token) and
+            // the SERVER runs the action against the extension using the grant it holds.
+            turnBridgeUrl = String(payload.browserExtProxy.url);
+            turnBridgeToken = String(payload.browserExtProxy.token);
+            browserTargetRef = String(payload.browserExtProxy.targetRef || "");
+            if (opts.verbose)
+                console.log(chalk.gray(`  [browser-bridge] ext-proxy turn (session=${payload.sessionId.slice(0, 10)} url=${turnBridgeUrl})`));
+        }
         // Capture for the stdout "done" handler closure below.
         const _turnBridgeRunId = turnBridgeRunId;
         const _turnId = payload.turnId;
@@ -1617,9 +1635,13 @@ export async function connect(opts) {
                 // JWS string so the Python browser toolkit can verify it again
                 // (defence-in-depth) and install MEL_BROWSER_TURN_GRANT in the
                 // host env for THIS turn only. Absent when no grant was provided.
-                ...(verifiedGrantToken ? { browser_grant: verifiedGrantToken, browser_target_ref: browserTargetRef ?? "" } : {}),
+                ...(verifiedGrantToken
+                    ? { browser_grant: verifiedGrantToken, browser_target_ref: browserTargetRef ?? "" }
+                    // Front 3 EXTENSION proxy: no grant, but the host must still apply
+                    // MEL_BROWSER_URL/TOKEN + register the browser toolkit for the turn.
+                    : (turnBridgeUrl ? { browser_ext_proxy: true, browser_target_ref: browserTargetRef ?? "" } : {})),
                 // Bridge URL + token for the host process env (applied per-turn by
-                // _apply_browser_turn_grant alongside the grant token).
+                // _apply_browser_turn_grant, alongside the grant token OR the proxy flag).
                 ...(turnBridgeUrl && turnBridgeToken
                     ? { mel_browser_url: turnBridgeUrl, mel_browser_token: turnBridgeToken }
                     : {}),

package/dist/publishGate.d.ts

@@ -0,0 +1,5 @@
+/** Classify a control's accessible name into a consequential effect.
+ *  Purchase wins on overlap. Returns null when nothing matches (the caller
+ *  then runs the action normally, no gate). The name is lowercased + trimmed
+ *  here so callers can pass a raw accessible name. */
+export declare function classifyControlEffect(name: string): "publish" | "purchase" | null;

package/dist/publishGate.js

@@ -0,0 +1,87 @@
+// packages/runner/src/publishGate.ts
+//
+// Melaya Browser, Route 1 HITL publish gate: the runner-side mirror of the
+// phone's on-device `detectPublish` gate. The bridge is the only place that
+// sees the clicked control's TRUSTED accessible name (page text is untrusted,
+// so the server can only classify a raw "Post/Comment/Buy" click as
+// "message"). This module classifies that trusted name into a consequential
+// effect ("publish" | "purchase") so the bridge can escalate the effect from
+// its own read and stage a human approval when the grant does not permit the
+// action unattended.
+//
+// Pure classification only: no I/O, no Playwright. The bridge owns the
+// accessible-name read and the server approval call; this file just maps a
+// name string to an effect class using the frozen word lists.
+// ---------------------------------------------------------------------
+// Frozen effect vocabularies (phone detectPublish mirror)
+// ---------------------------------------------------------------------
+//
+// Each phrase is matched word-aware (word boundaries) so "reporter" does not
+// match "report" and "password" does not match "pass". Purchase wins on any
+// overlap with publish. Keep these lists in sync with the phone's
+// detectPublish word lists.
+const PURCHASE_PHRASES = [
+    "buy",
+    "buy now",
+    "pay",
+    "pay now",
+    "place order",
+    "order",
+    "checkout",
+    "check out",
+    "purchase",
+    "subscribe",
+    "donate",
+    "confirm and pay",
+    "complete purchase",
+    "place bid",
+];
+const PUBLISH_PHRASES = [
+    "post",
+    "comment",
+    "reply",
+    "send",
+    "publish",
+    "share",
+    "tweet",
+    "retweet",
+    "repost",
+    "submit",
+    "post comment",
+    "add comment",
+    "leave a comment",
+    "send message",
+];
+/** Escape a phrase for use inside a RegExp source. */
+function escapeRe(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/** Word-aware containment: true when `phrase` appears in `name` bounded by
+ *  non-word characters (or string ends) on both sides. Both are assumed
+ *  already lowercased + trimmed. A phrase with internal spaces matches the
+ *  same run of whitespace-separated words. */
+function containsPhrase(name, phrase) {
+    // \b would not fire around non-ASCII, but the frozen phrases are all ASCII
+    // words, so an explicit non-word-char boundary on each side is sufficient
+    // and avoids the "reporter" != "report" / "password" != "pass" traps.
+    const re = new RegExp(`(?:^|[^a-z0-9])${escapeRe(phrase)}(?:$|[^a-z0-9])`, "i");
+    return re.test(name);
+}
+/** Classify a control's accessible name into a consequential effect.
+ *  Purchase wins on overlap. Returns null when nothing matches (the caller
+ *  then runs the action normally, no gate). The name is lowercased + trimmed
+ *  here so callers can pass a raw accessible name. */
+export function classifyControlEffect(name) {
+    const n = String(name || "").toLowerCase().trim();
+    if (!n)
+        return null;
+    for (const phrase of PURCHASE_PHRASES) {
+        if (containsPhrase(n, phrase))
+            return "purchase";
+    }
+    for (const phrase of PUBLISH_PHRASES) {
+        if (containsPhrase(n, phrase))
+            return "publish";
+    }
+    return null;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.1.16",
+  "version": "1.1.18",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,