@melaya/runner 1.1.68 → 1.1.69

package/dist/browserBridge.js

@@ -24,6 +24,9 @@
 //   POST /browser/act             -> {ok, result, tree, generation} |
 //                                    {ok:false, error:{code, message}}
 //   POST /browser/run             -> {ok, result, console[], tree}
+//   POST /browser/fast            -> {ok, result: text} (browser_fast: the
+//                                    server's fast executor drives this run
+//                                    over a pump, see browserFast.ts)
 //
 // Every act is ACT-AND-OBSERVE: it returns a fresh snapshot so the model
 // never acts twice on a stale world.
@@ -41,6 +44,7 @@ import { runSandboxedScript } from "./codeWorker.js";
 import { classifyControlEffect, writeGatedInMode } from "./publishGate.js";
 import { applyUploadResize, parseUploadResize, resolveDataUrlSource, resolveHandleSource, resolvePathSource, resolveUrlSource, UploadSourceError, } from "./uploadFetch.js";
 import { unlink } from "node:fs/promises";
+import { StableRefs, clampHitlMode, execFastCommand, postServerJson, runFastPump, translateStepRefs, } from "./browserFast.js";
 // A wall-clock around a CDP command. Playwright's CDPSession.send has NO default
 // timeout, so a wedged/busy renderer means the promise NEVER settles (not a
 // rejection - no catch fires) and the lease op-chain queues every later op
@@ -812,6 +816,10 @@ export async function startBrowserBridge(opts) {
         }
         return rec;
     }
+    // The last snapshot's rendered nodes per lease, for fast mode (browserFast.ts),
+    // which renders its own screen with stable refs from them.
+    const fastSnapItems = new WeakMap();
+    const fastStableRefs = new WeakMap();
     async function captureSnapshot(rec, lease, scope) {
         const page = lease.page;
         const generation = sessions.beginSnapshot(lease);
@@ -874,10 +882,17 @@ export async function startBrowserBridge(opts) {
             .sort((a, b) => a.order - b.order);
         // Bind refs and render one line per node.
         const lines = [];
+        const fastItems = [];
         let refNo = 0;
         for (const n of candidates) {
             refNo += 1;
             const ref = `@e${refNo}`;
+            fastItems.push({
+                ref, frameKey: n.frameKey, backendNodeId: n.backendNodeId, role: n.role, name: n.name,
+                aria: n.aria, domId: n.domId, href: n.href, click: n.click, edit: n.edit, covered: n.covered,
+                l: n.l, t: n.t, r: n.r, b: n.b,
+                ...(n.edit && n.value ? { value: n.value } : {}),
+            });
             sessions.bindRef(lease, {
                 ref,
                 targetRef: lease.ref,
@@ -895,6 +910,7 @@ export async function startBrowserBridge(opts) {
             const href = n.href ? ` @${n.href.slice(0, 100)}` : "";
             lines.push(`${ref} <${n.role}>${flags}${name}${aria}${domId}${href} <${Math.round(n.l)},${Math.round(n.t)},${Math.round(n.r)},${Math.round(n.b)}>`);
         }
+        fastSnapItems.set(lease, { generation, items: fastItems });
         const url = page.url();
         const title = await page.title().catch(() => "");
         const header = `url=${url} title=${title.slice(0, 120)} gen=${generation} nodes=${lines.length}/${total}`;
@@ -1048,7 +1064,19 @@ export async function startBrowserBridge(opts) {
                                     }
                                 }
                                 catch { /* unknown -> leave uncovered (conservative) */ }
+                                // A field's value, for fast mode's check that typed text
+                                // landed. Never a secret field's (password, one-time code,
+                                // card number): those are left out entirely.
+                                let value = "";
+                                if (edit) {
+                                    const type = attr(dom.doc, dom.nodeIdx, "type").toLowerCase();
+                                    const ac = attr(dom.doc, dom.nodeIdx, "autocomplete").toLowerCase();
+                                    const secret = type === "password" || /(^|\s)(current-password|new-password|one-time-code|cc-number|cc-csc)(\s|$)/.test(ac);
+                                    if (!secret)
+                                        value = String(n.value?.value ?? "").slice(0, 400);
+                                }
                                 out.push({
+                                    ...(value ? { value } : {}),
                                     frameKey,
                                     backendNodeId: backendId,
                                     role: role || "node",
@@ -1728,6 +1756,25 @@ export async function startBrowserBridge(opts) {
                     const value = String(args.value ?? "");
                     const cdp = await getCdp(rec, page);
                     const resolved = await cdpRace(cdp.send("DOM.resolveNode", { backendNodeId: binding.backendNodeId }), "DOM.resolveNode");
+                    if (args.match_label) {
+                        // Fast mode names an option by what the dropdown SHOWS (its text)
+                        // or by its value; pick the one that matches, and say so when none
+                        // does instead of setting a value the <select> does not have.
+                        const r = await cdpRace(cdp.send("Runtime.callFunctionOn", {
+                            objectId: resolved.object.objectId,
+                            functionDeclaration: "function(v){ var w = String(v).replace(/\\s+/g,' ').trim().toLowerCase(); var opts = this.options ? Array.prototype.slice.call(this.options) : [];" +
+                                " var hit = opts.find(function(o){ return String(o.value).trim().toLowerCase() === w; }) ||" +
+                                " opts.find(function(o){ return String(o.text).replace(/\\s+/g,' ').trim().toLowerCase() === w; });" +
+                                " if (!hit) return null; this.value = hit.value; this.dispatchEvent(new Event('input',{bubbles:true}));" +
+                                " this.dispatchEvent(new Event('change',{bubbles:true})); return hit.value; }",
+                            arguments: [{ value }],
+                            returnByValue: true,
+                        }), "Runtime.callFunctionOn");
+                        if (r.result?.value === null || r.result?.value === undefined) {
+                            throw new BridgeError("act_args_invalid", `the dropdown has no option "${value.slice(0, 80)}"`);
+                        }
+                        return { selected: String(r.result.value) };
+                    }
                     await cdpRace(cdp.send("Runtime.callFunctionOn", {
                         objectId: resolved.object.objectId,
                         functionDeclaration: `function(v){ this.value = v; this.dispatchEvent(new Event('input',{bubbles:true})); this.dispatchEvent(new Event('change',{bubbles:true})); }`,
@@ -2039,6 +2086,132 @@ export async function startBrowserBridge(opts) {
             },
         };
     }
+    // -- Fast mode (browser_fast) on this browser -----------------------------
+    //
+    // The server runs the fast executor (fastRun.ts) and sends this bridge one
+    // command at a time over a pump the bridge itself opens (browserFast.ts).
+    // Every command runs through performAct on THIS run's registration, so the
+    // grant's effect ceiling and origin policy, the takeover pause and the
+    // publish gate apply exactly as for the per-step tools.
+    const FAST_CONTROL_TIMEOUT_MS = 195_000; // browser.py holds the call 200 s; the server answers by 170 s
+    const FAST_PUMP_TIMEOUT_MS = 45_000; // the server long-polls 25 s
+    function fastStable(lease) {
+        let s = fastStableRefs.get(lease);
+        if (!s) {
+            s = new StableRefs();
+            fastStableRefs.set(lease, s);
+        }
+        return s;
+    }
+    function currentBindings(lease) {
+        return [...lease.refs.values()].filter((b) => b.snapshotGeneration === lease.snapshotGeneration && b.documentGeneration === lease.documentGeneration);
+    }
+    function fastOps(reg) {
+        return {
+            state: () => ({
+                cancelled: reg.cancelled,
+                expired: Date.now() / 1000 > reg.spec.grant.exp + 30,
+                paused: isActPaused(reg),
+            }),
+            observe: async () => {
+                const { rec, lease } = await getLease(reg);
+                const snap = await captureSnapshot(rec, lease);
+                const kept = fastSnapItems.get(lease);
+                return {
+                    url: snap.url, title: snap.title, generation: snap.generation,
+                    documentGeneration: lease.documentGeneration,
+                    items: kept && kept.generation === snap.generation ? kept.items : [],
+                };
+            },
+            bindings: async () => {
+                const { lease } = await getLease(reg);
+                return { documentGeneration: lease.documentGeneration, refs: currentBindings(lease) };
+            },
+            act: async (args) => {
+                const result = await performAct(reg, args);
+                if (args.kind !== "get_text") {
+                    // The same settle actAndObserve gives an act before the page is read.
+                    const { lease } = await getLease(reg);
+                    await lease.page.waitForLoadState("domcontentloaded", { timeout: 5_000 }).catch(() => { });
+                    await lease.page.waitForTimeout(ACT_SETTLE_MS);
+                }
+                return result;
+            },
+            gateSubmit: async (mode) => {
+                const { lease } = await getLease(reg);
+                // Exactly the gate performAct's press_key Enter raises (same digest),
+                // so the approved card is the one that later lets the Enter through.
+                await enforcePublishGate(reg, lease.page, "press_key", "focused field", "press_key:Enter", mode, "publish");
+            },
+            location: async () => {
+                const { lease } = await getLease(reg);
+                return { url: lease.page.url(), title: (await lease.page.title().catch(() => "")).slice(0, 200) };
+            },
+            wait: async (ms) => {
+                const { lease } = await getLease(reg);
+                await lease.page.waitForTimeout(Math.max(0, Math.min(5_000, ms)));
+            },
+            stableRefs: async () => fastStable((await getLease(reg)).lease),
+        };
+    }
+    async function handleFast(reg, payload) {
+        const serverBase = String(reg.spec.serverBase || "").replace(/\/$/, "");
+        const authHeader = String(reg.spec.runAuthHeader || "");
+        if (!serverBase || !authHeader) {
+            throw new BridgeError("source_unavailable", "fast mode needs this run's server connection, which it was registered without; use the per-step browser_* tools.");
+        }
+        const mode = clampHitlMode(reg.spec.hitlMode, payload["hitl_mode"]);
+        const control = async (body) => {
+            const { status, json } = await postServerJson(serverBase, "/api/v1/browser/runner/fast", authHeader, body, FAST_CONTROL_TIMEOUT_MS);
+            if (status >= 200 && status < 300 && typeof json.result === "string")
+                return json.result;
+            const err = (json.error && typeof json.error === "object" ? json.error : {});
+            if (status === 404 && !err.code) {
+                throw new BridgeError("fast_unavailable", "this Melaya server does not run fast mode on the runner's browser yet; use the per-step browser_* tools (or browser_batch).");
+            }
+            throw new BridgeError(String(err.code || "fast_failed"), String(err.message || `fast mode failed (HTTP ${status})`).slice(0, 600));
+        };
+        const resumeId = typeof payload["fast_run_id"] === "string" ? String(payload["fast_run_id"]).trim() : "";
+        if (resumeId)
+            return control({ op: "resume", fast_run_id: resumeId, ...(payload["stop"] === true ? { stop: true } : {}) });
+        const { run_id: _r, pipeline_name: _p, hitl_mode: _h, fast_run_id: _f, stop: _s, ...args } = payload;
+        // Steps may quote the @eN of the caller's last browser_get_screen_tree:
+        // turn them into stable refs while that snapshot is still the current one.
+        try {
+            const { lease } = await getLease(reg);
+            const toStable = fastStable(lease).assign(lease.documentGeneration, currentBindings(lease));
+            args.steps = translateStepRefs(args.steps, (r) => toStable.get(r) ?? null);
+        }
+        catch { /* no session yet: the run's own first read decides */ }
+        const fastId = randomBytes(18).toString("base64url");
+        const ops = fastOps(reg);
+        void runFastPump({
+            fastId,
+            post: async (body) => {
+                const { status, json } = await postServerJson(serverBase, "/api/v1/browser/runner/fast/pump", authHeader, body, FAST_PUMP_TIMEOUT_MS);
+                // Refused or unknown: nothing more will come for this run.
+                if (status === 401 || status === 403 || status === 404 || status === 410)
+                    return { end: true };
+                if (status < 200 || status >= 300)
+                    throw new Error(`pump HTTP ${status}`);
+                return json;
+            },
+            exec: (cmd) => execFastCommand(ops, cmd, mode),
+            isCancelled: () => reg.cancelled,
+            log,
+        }).catch((e) => log(`[fast] pump stopped: ${e?.message || e}`));
+        const grant = reg.spec.grant;
+        return control({
+            op: "start",
+            fast_id: fastId,
+            run_id: reg.spec.runId,
+            session_id: grant.browserSession,
+            jti: grant.jti,
+            target_ref: grant.target.ref,
+            hitl_mode: mode,
+            args,
+        });
+    }
     // Post an (unchanged) heartbeat frame at least this often so the viewer's
     // frame watchdog never trips on a STATIC page. Without it, a page that is not
     // animating (the agent is reasoning, or the page is idle) produced ZERO frames
@@ -2330,6 +2503,11 @@ export async function startBrowserBridge(opts) {
                     const out = await actAndObserve(reg, payload);
                     return respond(200, out);
                 }
+                case "/browser/fast": {
+                    // browser_fast (browser.py): the server's fast executor drives this
+                    // run's browser through a pump the bridge opens (handleFast).
+                    return respond(200, { ok: true, result: await handleFast(reg, payload) });
+                }
                 case "/browser/run": {
                     if (!reg.spec.codeMode) {
                         return fail(403, "code_mode_disabled", "code mode is not enabled for this run (Phase 2 capability)");

package/dist/browserFast.d.ts

@@ -0,0 +1,144 @@
+export declare const STABLE_REF_PREFIX = "r";
+export interface RefBinding {
+    ref: string;
+    frameKey: string;
+    backendNodeId: number;
+    role: string;
+    name: string;
+    snapshotGeneration?: number;
+    documentGeneration?: number;
+}
+export declare class StableRefs {
+    private documentGeneration;
+    private next;
+    private byKey;
+    private byNum;
+    private static keyOf;
+    private sync;
+    /** Stable ref for every binding of the current snapshot (snapshot ref ->
+     *  stable ref). Records each node's latest role and label. */
+    assign(documentGeneration: number, bindings: Iterable<RefBinding>): Map<string, string>;
+    /** Stable ref of a binding already numbered (no new number). */
+    stableOf(documentGeneration: number, b: {
+        frameKey: string;
+        backendNodeId: number;
+    }): string | null;
+    /** The current snapshot's ref for a stable ref, or null when the element is
+     *  gone, the document changed, or the node no longer carries the role and
+     *  label it was read with. Never a guess. */
+    resolve(stableRef: string, documentGeneration: number, current: Iterable<RefBinding>): string | null;
+}
+export declare function isStableRef(ref: unknown): boolean;
+/** One rendered node of the bridge's last snapshot (captureSnapshot keeps
+ *  them per lease for this module). */
+export interface SnapItem extends RefBinding {
+    aria: string;
+    domId: string;
+    href: string;
+    click: boolean;
+    edit: boolean;
+    covered: boolean;
+    l: number;
+    t: number;
+    r: number;
+    b: number;
+    /** Editable fields only, never a secret field's. */
+    value?: string;
+}
+/** The screen in the line format the server's parseBrowserTree reads:
+ *  `r@e12 <role> [click] [edit] [COVERED] "name" ="value" ~aria #id @href <l,t,r,b>`.
+ *  A COVERED element (something else is painted over its centre) is not
+ *  offered as clickable: a click there lands on whatever covers it. */
+export declare function renderFastLines(items: SnapItem[], toStable: Map<string, string>): string[];
+/** The LLM usually read the page with browser_get_screen_tree before planning,
+ *  so its steps may quote that snapshot's @eN (a target's `ref`, a list's
+ *  `in`). The run's first read takes a new snapshot, which would orphan them:
+ *  translate them to stable refs now, while that snapshot is still current.
+ *  A ref that no longer resolves is dropped; the step falls back to its label. */
+export declare function translateStepRefs(steps: unknown, toStable: (snapshotRef: string) => string | null): unknown;
+/** The run's autonomy: the registered (server-clamped) mode, which a caller
+ *  holding the bridge token may TIGHTEN but never loosen (same rule as the
+ *  bridge's publish gate). No registered mode: the caller's, default safe. */
+export declare function clampHitlMode(specMode: unknown, bodyMode: unknown): "safe" | "payments_only" | "autonomous";
+export interface FastObservation {
+    url: string;
+    title: string;
+    generation: number;
+    documentGeneration: number;
+    items: SnapItem[];
+}
+/** What the bridge gives a fast session. Every write goes through act, which
+ *  is the bridge's performAct for this run's registration. */
+export interface FastBridgeOps {
+    /** The run is torn down, its grant expired, or the user took over. */
+    state(): {
+        cancelled: boolean;
+        expired: boolean;
+        paused: boolean;
+    };
+    /** Take a snapshot (the bridge binds its @eN refs) and return its nodes. */
+    observe(): Promise<FastObservation>;
+    /** The current snapshot's bindings, for resolving a stable ref. */
+    bindings(): Promise<{
+        documentGeneration: number;
+        refs: RefBinding[];
+    }>;
+    /** One act through performAct. Throws {code, message} on refusal. */
+    act(args: Record<string, unknown>): Promise<unknown>;
+    /** Stage (or correlate) the approval a submitting Enter needs, BEFORE
+     *  anything is typed. Throws approval_required while it is pending. */
+    gateSubmit(mode: string): Promise<void>;
+    /** URL and title of the controlled tab. */
+    location(): Promise<{
+        url: string;
+        title: string;
+    }>;
+    wait(ms: number): Promise<void>;
+    /** Stable refs of this lease (kept across fast runs, per document). */
+    stableRefs(): Promise<StableRefs>;
+}
+export interface FastCommand {
+    seq: number;
+    kind: string;
+    body: Record<string, unknown>;
+}
+export type FastOutcome = {
+    ok: true;
+    data: Record<string, unknown>;
+} | {
+    ok: false;
+    code: string;
+    message: string;
+};
+export declare class FastCommandError extends Error {
+    code: string;
+    constructor(code: string, message: string);
+}
+/** Run one command the server's executor sent, under this run's grant and
+ *  autonomy (`mode`, clamped on this side: the server cannot loosen it). */
+export declare function execFastCommand(ops: FastBridgeOps, cmd: FastCommand, mode: string): Promise<FastOutcome>;
+export interface PumpReply {
+    command?: FastCommand;
+    end?: boolean;
+    idle?: boolean;
+}
+/** Pull the server executor's commands and run them, one at a time, until the
+ *  server ends the run or this run is torn down. A command is executed once:
+ *  its outcome is carried by the next poll, resent unchanged if that poll
+ *  fails, and the server ignores an outcome it already has. Nothing is ever
+ *  re-run here. */
+export declare function runFastPump(o: {
+    fastId: string;
+    post: (body: Record<string, unknown>) => Promise<PumpReply>;
+    exec: (cmd: FastCommand) => Promise<FastOutcome>;
+    isCancelled: () => boolean;
+    log?: (msg: string) => void;
+    /** Consecutive failed polls before giving up (the server's own command
+     *  deadline then reports the step as unknown). */
+    maxFailures?: number;
+    sleep?: (ms: number) => Promise<void>;
+}): Promise<void>;
+export declare function postServerJson(serverBase: string, path: string, authHeader: string, body: Record<string, unknown>, timeoutMs: number): Promise<{
+    status: number;
+    json: Record<string, unknown>;
+}>;