@melaya/runner 1.1.68 → 1.1.70

package/dist/browserFast.js

@@ -0,0 +1,363 @@
+// packages/runner/src/browserFast.ts
+//
+// browser_fast on the runner's OWN browser (the desktop app's dedicated
+// Playwright browser, browserBridge.ts). Design: Obsidian/37 System One
+// Models/System One Model & Browser Extension.md, section 2.
+//
+// The fast EXECUTOR stays on the server (server/src/services/fast/fastRun.ts):
+// one plan call, deterministic grounding first, Laya as a gated fallback with
+// the per-user and box budgets, verification after every step, STOP, no blind
+// retry. Only the transport differs from the extension path. The server
+// cannot reach this machine's loopback bridge, so the runner opens the
+// channel itself:
+//
+//   browser.py browser_fast --POST /browser/fast--> bridge (per-run bearer,
+//     grant expiry) --POST /api/v1/browser/runner/fast {op:"start"}--> server
+//     (runner token, session ownership, grant not revoked) runs fastToolRun
+//     with runnerBrowserFastHost (server/src/services/fast/fastRunnerBrowser.ts)
+//
+//   meanwhile the bridge pumps commands: POST /api/v1/browser/runner/fast/pump
+//     carries the previous command's outcome and long-polls the next one.
+//     Every command runs HERE, under the SAME run registration (the signed
+//     grant the bridge verified), through performAct: effect ceiling, origin
+//     policy, takeover pause, the publish gate and its approval card.
+//
+// This module holds the parts that need no browser: stable refs, the fast
+// screen rendering, command execution over an injected ops interface, the
+// pump loop and the HTTP client. browserBridge.ts supplies the real ops.
+import { request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
+// ---------------------------------------------------------------------
+// Stable refs
+// ---------------------------------------------------------------------
+//
+// The bridge renumbers @eN on every snapshot and refuses a ref from an older
+// one. The executor compares refs ACROSS reads (a control that appeared since
+// the last action wins a tie between identical labels; a picker's options are
+// the elements that were not there before), so it needs refs that stay put
+// while the element lives, as the extension's do. A stable ref is keyed on
+// the DOM node (frame + backendNodeId), numbered per lease and never reused:
+// a ref from an earlier document can never land on a node of a later one.
+// They carry their own prefix ("r@e12") so the per-step tools, whose @eN are
+// per snapshot, refuse them instead of clicking whatever @e12 is now.
+export const STABLE_REF_PREFIX = "r";
+const STABLE_REF_RE = /^r@e(\d+)$/;
+const SNAPSHOT_REF_RE = /^@e\d+$/;
+export class StableRefs {
+    documentGeneration = -1;
+    next = 0;
+    byKey = new Map();
+    byNum = new Map();
+    static keyOf(b) {
+        return `${b.frameKey}:${b.backendNodeId}`;
+    }
+    sync(documentGeneration) {
+        if (documentGeneration === this.documentGeneration)
+            return;
+        // A new document: forget every node, keep counting (never reuse a number).
+        this.documentGeneration = documentGeneration;
+        this.byKey.clear();
+        this.byNum.clear();
+    }
+    /** Stable ref for every binding of the current snapshot (snapshot ref ->
+     *  stable ref). Records each node's latest role and label. */
+    assign(documentGeneration, bindings) {
+        this.sync(documentGeneration);
+        const out = new Map();
+        for (const b of bindings) {
+            const key = StableRefs.keyOf(b);
+            let num = this.byKey.get(key);
+            if (num === undefined) {
+                num = ++this.next;
+                this.byKey.set(key, num);
+            }
+            this.byNum.set(num, { key, frameKey: b.frameKey, backendNodeId: b.backendNodeId, role: b.role, name: b.name });
+            out.set(b.ref, `${STABLE_REF_PREFIX}@e${num}`);
+        }
+        return out;
+    }
+    /** Stable ref of a binding already numbered (no new number). */
+    stableOf(documentGeneration, b) {
+        if (documentGeneration !== this.documentGeneration)
+            return null;
+        const num = this.byKey.get(StableRefs.keyOf(b));
+        return num === undefined ? null : `${STABLE_REF_PREFIX}@e${num}`;
+    }
+    /** The current snapshot's ref for a stable ref, or null when the element is
+     *  gone, the document changed, or the node no longer carries the role and
+     *  label it was read with. Never a guess. */
+    resolve(stableRef, documentGeneration, current) {
+        const m = STABLE_REF_RE.exec(String(stableRef || "").trim());
+        if (!m || documentGeneration !== this.documentGeneration)
+            return null;
+        const e = this.byNum.get(Number(m[1]));
+        if (!e)
+            return null;
+        for (const b of current) {
+            if (b.frameKey !== e.frameKey || b.backendNodeId !== e.backendNodeId)
+                continue;
+            if (b.role !== e.role || b.name !== e.name)
+                return null;
+            return b.ref;
+        }
+        return null;
+    }
+}
+export function isStableRef(ref) {
+    return typeof ref === "string" && STABLE_REF_RE.test(ref.trim());
+}
+const clean = (s, max) => String(s || "").replace(/\s+/g, " ").replace(/"/g, "'").trim().slice(0, max);
+/** The screen in the line format the server's parseBrowserTree reads:
+ *  `r@e12 <role> [click] [edit] [COVERED] "name" ="value" ~aria #id @href <l,t,r,b>`.
+ *  A COVERED element (something else is painted over its centre) is not
+ *  offered as clickable: a click there lands on whatever covers it. */
+export function renderFastLines(items, toStable) {
+    const lines = [];
+    for (const n of items) {
+        const ref = toStable.get(n.ref);
+        if (!ref)
+            continue;
+        const flags = `${n.click && !n.covered ? " click" : ""}${n.edit ? " edit" : ""}${n.covered ? " COVERED" : ""}`;
+        const name = ` "${clean(n.name, 240)}"`;
+        const value = n.edit && n.value ? ` ="${clean(n.value, 200)}"` : "";
+        const aria = n.aria ? ` ~${clean(n.aria, 40).replace(/\s@/g, " ")}` : "";
+        const domId = n.domId ? ` #${clean(n.domId, 40).replace(/\s/g, "")}` : "";
+        const href = n.href ? ` @${clean(n.href, 200).replace(/\s/g, "")}` : "";
+        lines.push(`${ref} <${n.role || "node"}>${flags}${name}${value}${aria}${domId}${href} <${Math.round(n.l)},${Math.round(n.t)},${Math.round(n.r)},${Math.round(n.b)}>`);
+    }
+    return lines;
+}
+// ---------------------------------------------------------------------
+// Steps that quote the per-step tree's refs
+// ---------------------------------------------------------------------
+/** The LLM usually read the page with browser_get_screen_tree before planning,
+ *  so its steps may quote that snapshot's @eN (a target's `ref`, a list's
+ *  `in`). The run's first read takes a new snapshot, which would orphan them:
+ *  translate them to stable refs now, while that snapshot is still current.
+ *  A ref that no longer resolves is dropped; the step falls back to its label. */
+export function translateStepRefs(steps, toStable) {
+    if (!Array.isArray(steps))
+        return steps;
+    return steps.map((raw) => {
+        if (!raw || typeof raw !== "object" || Array.isArray(raw))
+            return raw;
+        const s = { ...raw };
+        for (const k of ["ref", "in"]) {
+            const v = s[k];
+            if (typeof v === "string" && SNAPSHOT_REF_RE.test(v.trim())) {
+                const stable = toStable(v.trim());
+                if (stable)
+                    s[k] = stable;
+                else
+                    delete s[k];
+            }
+        }
+        if (Array.isArray(s.steps))
+            s.steps = translateStepRefs(s.steps, toStable);
+        return s;
+    });
+}
+// ---------------------------------------------------------------------
+// Autonomy
+// ---------------------------------------------------------------------
+const MODE_RANK = { safe: 0, payments_only: 1, autonomous: 2 };
+const asMode = (m) => m === "autonomous" || m === "payments_only" ? m : "safe";
+/** The run's autonomy: the registered (server-clamped) mode, which a caller
+ *  holding the bridge token may TIGHTEN but never loosen (same rule as the
+ *  bridge's publish gate). No registered mode: the caller's, default safe. */
+export function clampHitlMode(specMode, bodyMode) {
+    if (specMode == null || specMode === "")
+        return asMode(bodyMode);
+    const spec = asMode(specMode);
+    if (bodyMode == null || bodyMode === "")
+        return spec;
+    const body = asMode(bodyMode);
+    return MODE_RANK[body] < MODE_RANK[spec] ? body : spec;
+}
+const ACT_KINDS = new Set(["click", "input_text", "press_key", "scroll", "select_option", "back", "navigate"]);
+function errOf(e) {
+    const x = e;
+    return {
+        code: typeof x?.code === "string" && x.code ? x.code : "bridge_error",
+        message: String(x?.message ?? e ?? "failed").slice(0, 600),
+    };
+}
+export class FastCommandError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+    }
+}
+async function screenOf(ops) {
+    const obs = await ops.observe();
+    const refs = await ops.stableRefs();
+    const toStable = refs.assign(obs.documentGeneration, obs.items);
+    const lines = renderFastLines(obs.items, toStable);
+    return { url: obs.url, title: obs.title, generation: obs.generation, tree: lines.join("\n"), nodes: lines.length };
+}
+async function snapshotRefOf(ops, ref) {
+    const r = typeof ref === "string" ? ref.trim() : "";
+    if (!isStableRef(r))
+        throw new FastCommandError("stale_ref", `ref '${r.slice(0, 40)}' is not a fast-mode ref of this page`);
+    const [b, refs] = await Promise.all([ops.bindings(), ops.stableRefs()]);
+    const snap = refs.resolve(r, b.documentGeneration, b.refs);
+    if (!snap)
+        throw new FastCommandError("stale_ref", `the element ${r} is no longer on the page as it was read`);
+    return snap;
+}
+/** Run one command the server's executor sent, under this run's grant and
+ *  autonomy (`mode`, clamped on this side: the server cannot loosen it). */
+export async function execFastCommand(ops, cmd, mode) {
+    try {
+        const st = ops.state();
+        if (st.cancelled)
+            return { ok: false, code: "canceled", message: "the browser run was stopped or torn down" };
+        if (st.expired)
+            return { ok: false, code: "grant_expired", message: "the browser grant for this run has expired" };
+        const body = cmd.body ?? {};
+        switch (cmd.kind) {
+            case "observe":
+                return { ok: true, data: await screenOf(ops) };
+            case "location":
+                return { ok: true, data: await ops.location() };
+            case "text": {
+                const ref = body.ref ? await snapshotRefOf(ops, body.ref) : undefined;
+                const out = (await ops.act({ kind: "get_text", ...(ref ? { ref } : {}), hitl_mode: mode }));
+                return { ok: true, data: { text: String(out?.text ?? "") } };
+            }
+            case "act": {
+                const action = String(body.action ?? "");
+                if (!ACT_KINDS.has(action))
+                    return { ok: false, code: "act_kind_unknown", message: `fast mode does not send '${action.slice(0, 40)}'` };
+                // The user took over: stop here, before anything is acted on.
+                if (st.paused)
+                    return { ok: false, code: "paused_by_user", message: "the user has taken manual control of the browser" };
+                const args = { kind: action, hitl_mode: mode };
+                if (body.ref !== undefined)
+                    args.ref = await snapshotRefOf(ops, body.ref);
+                let result;
+                switch (action) {
+                    case "input_text": {
+                        const text = String(body.text ?? "");
+                        // A submitting type is a commit (Enter): its approval is settled
+                        // before a single character is typed, so a pending or declined
+                        // card leaves the field untouched.
+                        if (body.submit === true)
+                            await ops.gateSubmit(mode);
+                        result = await ops.act({ ...args, text, ...(body.mode === "human" ? { mode: "human" } : {}) });
+                        if (body.submit === true)
+                            await ops.act({ kind: "press_key", key: "Enter", hitl_mode: mode });
+                        break;
+                    }
+                    case "press_key":
+                        result = await ops.act({ ...args, key: String(body.key ?? "") });
+                        break;
+                    case "scroll":
+                        result = await ops.act({ ...args, text: body.direction === "up" ? "up" : "down", value: "medium" });
+                        // Let an infinite feed load its next batch before the page is read.
+                        if (body.direction !== "up" && body.feed !== false)
+                            await ops.wait(700);
+                        break;
+                    case "select_option":
+                        result = await ops.act({ ...args, value: String(body.option ?? ""), match_label: true });
+                        break;
+                    case "navigate":
+                        result = await ops.act({ ...args, url: String(body.url ?? "") });
+                        break;
+                    default:
+                        result = await ops.act(args);
+                }
+                return { ok: true, data: { result: result ?? null, ...(await screenOf(ops)) } };
+            }
+            default:
+                return { ok: false, code: "unknown_command", message: `unknown fast command '${String(cmd.kind).slice(0, 40)}'` };
+        }
+    }
+    catch (e) {
+        return { ok: false, ...errOf(e) };
+    }
+}
+/** Pull the server executor's commands and run them, one at a time, until the
+ *  server ends the run or this run is torn down. A command is executed once:
+ *  its outcome is carried by the next poll, resent unchanged if that poll
+ *  fails, and the server ignores an outcome it already has. Nothing is ever
+ *  re-run here. */
+export async function runFastPump(o) {
+    const sleep = o.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    const maxFailures = o.maxFailures ?? 5;
+    let outcome = null;
+    let failures = 0;
+    const executed = new Set();
+    for (;;) {
+        const cancelled = o.isCancelled();
+        let reply;
+        try {
+            reply = await o.post({ fast_id: o.fastId, ...(outcome ? { result: outcome } : {}), ...(cancelled ? { abort: true } : {}) });
+        }
+        catch (e) {
+            if (cancelled || ++failures >= maxFailures) {
+                o.log?.(`[fast] pump ${o.fastId.slice(0, 8)} gave up: ${e?.message || e}`);
+                return;
+            }
+            await sleep(Math.min(5_000, 500 * failures));
+            continue;
+        }
+        failures = 0;
+        outcome = null;
+        if (cancelled || reply.end)
+            return;
+        const cmd = reply.command;
+        if (!cmd || typeof cmd.seq !== "number")
+            continue;
+        if (executed.has(cmd.seq)) {
+            // Never run a command twice, whatever the channel did.
+            outcome = { seq: cmd.seq, outcome: { ok: false, code: "unknown_outcome", message: "this command was already delivered once; it is not run again" } };
+            continue;
+        }
+        executed.add(cmd.seq);
+        outcome = { seq: cmd.seq, outcome: await o.exec(cmd) };
+    }
+}
+// ---------------------------------------------------------------------
+// HTTP to the server (same base URL + runner token the publish gate uses)
+// ---------------------------------------------------------------------
+export function postServerJson(serverBase, path, authHeader, body, timeoutMs) {
+    return new Promise((resolve, reject) => {
+        let u;
+        try {
+            u = new URL(serverBase.replace(/\/$/, "") + path);
+        }
+        catch (e) {
+            reject(e);
+            return;
+        }
+        const isHttps = u.protocol === "https:";
+        const data = JSON.stringify(body);
+        const req = (isHttps ? httpsRequest : httpRequest)({
+            hostname: u.hostname,
+            port: u.port || (isHttps ? 443 : 80),
+            path: u.pathname + u.search,
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "Content-Length": Buffer.byteLength(data),
+                "Authorization": authHeader,
+            },
+        }, (res) => {
+            const chunks = [];
+            res.on("data", (c) => chunks.push(Buffer.from(c)));
+            res.on("end", () => {
+                let json = {};
+                try {
+                    json = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+                }
+                catch { /* non-JSON body */ }
+                resolve({ status: res.statusCode || 0, json });
+            });
+        });
+        req.setTimeout(timeoutMs, () => req.destroy(new Error(`no answer from the server in ${Math.round(timeoutMs / 1000)}s`)));
+        req.on("error", reject);
+        req.end(data);
+    });
+}

package/dist/browserProvisioner.d.ts

@@ -31,3 +31,42 @@ export declare function ensureEngine(engine: BrowserEngineId, opts?: {
     allowDownload?: boolean;
     log?: (m: string) => void;
 }): Promise<InstalledEngine>;
+/** Preference order for headed flows. */
+export declare const INSTALLED_BROWSER_PREFERENCE: readonly BrowserEngineId[];
+/** Actionable error when no supported browser is installed. */
+export declare function noInstalledBrowserError(): ProvisionError;
+/** `npx` on PATH (Windows: npx.cmd). File checks only, no subprocess. */
+export declare function npxOnPath(env?: NodeJS.ProcessEnv): boolean;
+/** May this process download Playwright's Chromium? Only the CLI, with npx present. */
+export declare function canDownloadBundledChromium(opts?: {
+    hosted?: boolean;
+    npx?: boolean;
+}): boolean;
+/** Installed engines in preference order (Chrome, Edge, Brave, on-disk Chromium). */
+export declare function installedBrowsersByPreference(engines?: InstalledEngine[]): Promise<InstalledEngine[]>;
+type LaunchFn<B> = (o: {
+    headless: boolean;
+    channel?: string;
+    executablePath?: string;
+    args?: string[];
+}) => Promise<B>;
+/**
+ * Launch a headed browser for a sign-in flow from the installed set. Tries each
+ * installed engine in preference order, then Playwright's own Chrome / Edge
+ * channel lookup. When nothing launches: in the CLI with npx, `downloadChromium`
+ * (if given) installs Playwright's Chromium and it is launched; otherwise a
+ * ProvisionError with install instructions is thrown. Never spawns npx in the
+ * desktop app.
+ */
+export declare function launchInstalledBrowser<B>(launch: LaunchFn<B>, opts?: {
+    args?: string[];
+    progress?: (m: string) => void;
+    downloadChromium?: () => Promise<void>;
+    engines?: InstalledEngine[];
+    hosted?: boolean;
+    npx?: boolean;
+}): Promise<{
+    browser: B;
+    engine: BrowserEngineId | "channel";
+}>;
+export {};

package/dist/browserProvisioner.js

@@ -239,6 +239,12 @@ export async function ensureEngine(engine, opts = {}) {
         };
         throw new ProvisionError("engine_not_installed", `${engine} is not installed on this machine`, `Install ${engine} from ${urls[engine]} and reconnect the runner.`);
     }
+    // Desktop app: process.execPath is the Electron binary, which cannot run
+    // Playwright's CLI (Electron-as-Node is fused off), and there is no npx.
+    // Never shell out; ask for an installed browser instead (audit P1.4).
+    if (hostedByDesktopApp()) {
+        throw noInstalledBrowserError();
+    }
     if (!opts.allowDownload) {
         throw new ProvisionError("download_not_authorized", "bundled Chromium is not on disk and download was not authorized", "Re-run with the download flag enabled, or run: npx playwright install chromium");
     }
@@ -258,6 +264,96 @@ export async function ensureEngine(engine, opts = {}) {
         throw new ProvisionError("download_failed", "Chromium installed but executable not found");
     return post;
 }
+// ---------------------------------------------------------------------
+// Installed-browser selection (one provisioner for every headed flow)
+// ---------------------------------------------------------------------
+//
+// LinkedIn / Luma sign-in and any other headed flow pick a browser here:
+// installed Chrome, then Edge, then Brave, then a Playwright Chromium that is
+// already on disk. Downloading Chromium is a CLI-only last resort (needs npx
+// or a real Node), never in the desktop app.
+/** Preference order for headed flows. */
+export const INSTALLED_BROWSER_PREFERENCE = ["chrome", "edge", "brave", "chromium"];
+/** Actionable error when no supported browser is installed. */
+export function noInstalledBrowserError() {
+    return new ProvisionError("engine_not_installed", "No supported browser found on this computer (Chrome, Edge or Brave).", "Install Google Chrome (https://www.google.com/chrome/) or Microsoft Edge (https://www.microsoft.com/edge), then try again.");
+}
+function hostedByDesktopApp() {
+    const p = process.parentPort;
+    return !!p && typeof p.postMessage === "function";
+}
+/** `npx` on PATH (Windows: npx.cmd). File checks only, no subprocess. */
+export function npxOnPath(env = process.env) {
+    const sep = process.platform === "win32" ? ";" : ":";
+    const names = process.platform === "win32" ? ["npx.cmd", "npx.exe", "npx"] : ["npx"];
+    for (const dir of String(env.PATH ?? env.Path ?? "").split(sep)) {
+        if (!dir)
+            continue;
+        for (const n of names) {
+            try {
+                if (existsSync(join(dir, n)))
+                    return true;
+            }
+            catch { /* unreadable */ }
+        }
+    }
+    return false;
+}
+/** May this process download Playwright's Chromium? Only the CLI, with npx present. */
+export function canDownloadBundledChromium(opts = {}) {
+    const hosted = opts.hosted ?? hostedByDesktopApp();
+    if (hosted)
+        return false;
+    return opts.npx ?? npxOnPath();
+}
+/** Installed engines in preference order (Chrome, Edge, Brave, on-disk Chromium). */
+export async function installedBrowsersByPreference(engines) {
+    const list = engines ?? await discoverEngines();
+    return INSTALLED_BROWSER_PREFERENCE
+        .map((id) => list.find((e) => e.engine === id))
+        .filter((e) => !!e);
+}
+/**
+ * Launch a headed browser for a sign-in flow from the installed set. Tries each
+ * installed engine in preference order, then Playwright's own Chrome / Edge
+ * channel lookup. When nothing launches: in the CLI with npx, `downloadChromium`
+ * (if given) installs Playwright's Chromium and it is launched; otherwise a
+ * ProvisionError with install instructions is thrown. Never spawns npx in the
+ * desktop app.
+ */
+export async function launchInstalledBrowser(launch, opts = {}) {
+    const args = opts.args ?? [];
+    const tried = new Set();
+    for (const e of await installedBrowsersByPreference(opts.engines)) {
+        const key = e.channel ? `channel:${e.channel}` : `path:${e.executablePath}`;
+        if (tried.has(key))
+            continue;
+        tried.add(key);
+        try {
+            const browser = e.channel
+                ? await launch({ headless: false, channel: e.channel, args })
+                : await launch({ headless: false, executablePath: e.executablePath, args });
+            return { browser, engine: e.engine };
+        }
+        catch (err) {
+            opts.progress?.(`${e.engine} did not start (${String(err?.message || err).split("\n")[0]}); trying the next browser.`);
+        }
+    }
+    // Installs outside the well-known paths: Playwright's own channel lookup.
+    for (const channel of ["chrome", "msedge"]) {
+        if (tried.has(`channel:${channel}`))
+            continue;
+        try {
+            return { browser: await launch({ headless: false, channel, args }), engine: "channel" };
+        }
+        catch { /* not installed */ }
+    }
+    if (opts.downloadChromium && canDownloadBundledChromium({ hosted: opts.hosted, npx: opts.npx })) {
+        await opts.downloadChromium();
+        return { browser: await launch({ headless: false, args }), engine: "chromium" };
+    }
+    throw noInstalledBrowserError();
+}
 function packageRootGuess() {
     // dist/ layout: this file compiles to dist/browserProvisioner.js; the
     // package root (with node_modules/playwright) is one level up. In the