@proagentstore/cli 0.3.8 → 0.4.1

package/dist/browser-runner/mcp-runtime.js

@@ -0,0 +1,48 @@
+import { createConnection } from "@playwright/mcp";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+/**
+ * Hosts the INDUSTRY-STANDARD `@playwright/mcp` server in the runner and an in-process
+ * MCP client to drive it. Every browser action goes through the standard Playwright MCP
+ * tools (browser_snapshot, browser_click, browser_type, browser_select_option,
+ * browser_fill_form, browser_file_upload, …) — no hand-rolled browser code. When the lib
+ * updates, we get its fixes for free. The cloud brain calls these same tools over the relay.
+ */
+export class McpRuntime {
+    server;
+    client;
+    async start(opts) {
+        if (this.client)
+            return;
+        const browser = opts.cdpEndpoint
+            ? { cdpEndpoint: opts.cdpEndpoint }
+            : { userDataDir: opts.userDataDir, isolated: opts.isolated, launchOptions: { headless: opts.headless ?? false } };
+        // The runner is a trusted local process uploading the user's OWN résumé, which
+        // lives under the runner's data dir (outside the CWD). Lift the file-root guard
+        // (meant to stop an LLM reading arbitrary host files) so browser_file_upload can
+        // attach it — the runner, not the model, chooses the path.
+        this.server = await createConnection({ browser, allowUnrestrictedFileAccess: true });
+        const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+        await this.server.connect(serverTransport);
+        this.client = new Client({ name: "pags-runner", version: "1.0.0" });
+        await this.client.connect(clientTransport);
+    }
+    /** The standard Playwright MCP tool schemas — advertised to the cloud brain verbatim. */
+    async listTools() {
+        const res = await this.client.listTools();
+        return res.tools;
+    }
+    async callTool(name, args = {}) {
+        return (await this.client.callTool({ name, arguments: args }));
+    }
+    /** Text of the last tool result (the standard tools return their output as text content). */
+    textOf(res) {
+        return (res.content || []).map((c) => c.text ?? "").join("\n");
+    }
+    async stop() {
+        await this.client?.close().catch(() => undefined);
+        await this.server?.close().catch(() => undefined);
+        this.client = undefined;
+        this.server = undefined;
+    }
+}

package/dist/browser-runner/runner.js

@@ -1,11 +1,11 @@
 import { spawnSync } from "node:child_process";
-import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { createRequire } from "node:module";
 import { basename, dirname, join, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
 import { captureScreenshotDataUrl, challengeSolved, detectHumanChallenge } from "./challenge.js";
 import { resolveRealChromeProfileDir, seedProfileCopy } from "./browser-profile.js";
-import { inspectField, performBrowserAction } from "./browser-actions.js";
+import { McpRuntime } from "./mcp-runtime.js";
 import { HumanHandoffError, RunnerInputError } from "./errors.js";
 import { RunnerStore } from "./store.js";
 import { CodingRuntime } from "./coding/runtime.js";
@@ -42,6 +42,16 @@ export class LocalRunner {
     activePage = null;
     fileAttachArmed = new WeakSet();
     applyResumePath = null;
+    /**
+     * The INDUSTRY-STANDARD `@playwright/mcp` server, attached over CDP to the same
+     * real-profile Chrome the takeover machinery drives. Every brain-issued browser
+     * action runs through its standard tools — no hand-rolled action code. Lazily
+     * started once the browser is up (see {@link getMcp}).
+     */
+    mcp = null;
+    cdpEndpoint = null;
+    /** The profile dir Chrome was actually launched into (holds DevToolsActivePort). */
+    launchedProfileDir = null;
     store;
     /** Local tmux coding sessions (the second runtime — AgentCoder port). */
     coding;
@@ -145,8 +155,12 @@ export class LocalRunner {
     }
     async close() {
         this.coding.closeAll();
+        await this.mcp?.stop().catch(() => undefined);
+        this.mcp = null;
+        this.cdpEndpoint = null;
         await this.browserContext?.close();
         this.browserContext = null;
+        this.launchedProfileDir = null;
     }
     async runTask(id) {
         const task = this.requireTask(id);
@@ -517,7 +531,10 @@ export class LocalRunner {
             viewport: null,
             // Light anti-detection: drop the most obvious automation tell so fewer
             // CAPTCHAs trigger in the first place (the human still solves the rest).
-            args: ["--disable-blink-features=AutomationControlled", "--start-maximized", "--window-size=1512,982"],
+            // --remote-debugging-port=0 → Chrome picks a free CDP port and writes it to
+            // DevToolsActivePort in the profile dir; the standard @playwright/mcp server
+            // attaches to that endpoint so it drives THIS same real-profile browser.
+            args: ["--disable-blink-features=AutomationControlled", "--start-maximized", "--window-size=1512,982", "--remote-debugging-port=0"],
         };
         // Prefer the real Chrome build (better TLS/fingerprint → fewer CAPTCHAs);
         // fall back to bundled Chromium if Chrome isn't installed. Disable with
@@ -548,10 +565,13 @@ export class LocalRunner {
                     channel: "chrome",
                     args: [...baseOpts.args, "--profile-directory=Default"],
                 });
+                this.launchedProfileDir = seededDir;
                 console.log(`[runner] launched a private copy of your real Chrome profile (signed-in sessions seeded from "${realProfileDir.profile}")`);
             }
             else {
-                this.browserContext = await playwright.chromium.launchPersistentContext(join(this.config.dataDir, "chrome-profile"), { ...baseOpts, channel: "chrome" });
+                const dedicated = join(this.config.dataDir, "chrome-profile");
+                this.browserContext = await playwright.chromium.launchPersistentContext(dedicated, { ...baseOpts, channel: "chrome" });
+                this.launchedProfileDir = dedicated;
             }
         }
         catch (err) {
@@ -560,6 +580,7 @@ export class LocalRunner {
                 console.warn(`[runner] real-profile launch failed (${msg}); falling back to a dedicated profile. Fully quit Chrome (Cmd+Q) to use your real profile.`);
             }
             this.browserContext = await playwright.chromium.launchPersistentContext(profileDir, baseOpts);
+            this.launchedProfileDir = profileDir;
         }
         await this.browserContext
             .addInitScript(() => {
@@ -608,10 +629,102 @@ export class LocalRunner {
         this.activePage = page;
         return page;
     }
+    /** The CDP endpoint of the launched Chrome (from DevToolsActivePort in its profile). */
+    async readCdpEndpoint() {
+        if (this.cdpEndpoint)
+            return this.cdpEndpoint;
+        const dir = this.launchedProfileDir;
+        if (!dir)
+            throw new Error("browser context not launched yet");
+        const portFile = join(dir, "DevToolsActivePort");
+        for (let i = 0; i < 50; i++) {
+            if (existsSync(portFile)) {
+                const line = readFileSync(portFile, "utf8").split("\n")[0]?.trim();
+                if (line) {
+                    this.cdpEndpoint = `http://127.0.0.1:${line}`;
+                    return this.cdpEndpoint;
+                }
+            }
+            await new Promise((r) => setTimeout(r, 100));
+        }
+        throw new Error("could not read Chrome CDP port (DevToolsActivePort missing)");
+    }
+    /**
+     * The standard `@playwright/mcp` server, attached over CDP to the runner's own
+     * real-profile Chrome. Lazily started; every browser action + snapshot goes
+     * through its standard tools. Because it shares the browser over CDP, actions it
+     * performs land on the exact page the takeover/screencast machinery drives.
+     */
+    async getMcp() {
+        if (this.mcp)
+            return this.mcp;
+        await this.getBrowserContext();
+        const cdpEndpoint = await this.readCdpEndpoint();
+        const mcp = new McpRuntime();
+        await mcp.start({ cdpEndpoint });
+        this.mcp = mcp;
+        return mcp;
+    }
+    /** Pull the aria tree (with [ref=eNN]) out of a standard browser_snapshot result. */
+    extractSnapshot(text) {
+        const yaml = text.match(/```yaml\n([\s\S]*?)```/);
+        if (yaml)
+            return yaml[1].trimEnd();
+        const i = text.indexOf("Page Snapshot:");
+        return i >= 0 ? text.slice(i + "Page Snapshot:".length).trim() : text.trim();
+    }
+    /** Concise feedback from a tool result — drop the big page snapshot (the brain
+     *  gets a fresh one next turn); keep the action confirmation / any error text. */
+    conciseFeedback(text) {
+        const cut = text.search(/###\s*Page state|- Page Snapshot:|```yaml/);
+        const head = (cut >= 0 ? text.slice(0, cut) : text).replace(/```[a-z]*\n?|```/g, " ").replace(/\s+/g, " ").trim();
+        return head.slice(0, 400);
+    }
+    /** The snapshot ref the brain must target the element by (standard-tool `target`). */
+    refOf(action) {
+        const ref = (action.ref || "").trim();
+        if (!ref)
+            throw new RunnerInputError(`the "${action.action}" action needs the element's snapshot ref (e.g. "e42")`);
+        return ref;
+    }
+    /** Map one brain BrowserAction onto the industry-standard @playwright/mcp tool. */
+    async callBrowserTool(mcp, a) {
+        const label = a.name || a.role || a.action;
+        switch (a.action) {
+            case "navigate":
+                return mcp.callTool("browser_navigate", { url: a.url });
+            case "type":
+                return mcp.callTool("browser_type", { element: label, target: this.refOf(a), text: a.text ?? "" });
+            // A checkbox/radio is just a click at the standard-tool level.
+            case "click":
+            case "check":
+                return mcp.callTool("browser_click", { element: label, target: this.refOf(a) });
+            case "upload": {
+                // Standard two-step: clicking the upload control opens the file chooser,
+                // which puts the @playwright/mcp server into its file-chooser modal state;
+                // browser_file_upload then resolves that state with the résumé path.
+                await mcp.callTool("browser_click", { element: label, target: this.refOf(a) });
+                if (!this.applyResumePath)
+                    throw new RunnerInputError("no résumé file available to upload");
+                return mcp.callTool("browser_file_upload", { paths: [this.applyResumePath] });
+            }
+            case "select":
+                return mcp.callTool("browser_select_option", { element: label, target: this.refOf(a), values: [a.text ?? ""] });
+            case "key":
+                return mcp.callTool("browser_press_key", { key: a.key || "Enter" });
+            case "scroll":
+                return mcp.callTool("browser_evaluate", { function: `() => window.scrollBy(0, ${Math.round(a.dy ?? 600)})` });
+            case "wait":
+                return mcp.callTool("browser_wait_for", { time: Math.min(5, Math.max(0.1, (a.ms ?? 1000) / 1000)) });
+            default:
+                throw new RunnerInputError(`unknown action: ${a.action}`);
+        }
+    }
     /**
-     * What the remote brain "sees": a compact ARIA tree (roles + accessible names
-     * + values + states) of the active page — the same representation the brain
-     * acts on via {@link browserAct}. No raw HTML/screenshots → cheap on tokens.
+     * What the remote brain "sees": the standard `@playwright/mcp` page snapshot — a
+     * compact ARIA tree with roles, accessible names, values, states, and a stable
+     * [ref=eNN] on each interactive element. The brain targets elements by that ref;
+     * because {@link browserAct} routes through the SAME mcp server, the refs resolve.
      */
     async browserSnapshot(taskId) {
         // If the user pressed Stop, the task is cancelled — tell the brain to halt.
@@ -620,15 +733,9 @@ export class LocalRunner {
         }
         const page = await this.getActivePage();
         await page.waitForLoadState("domcontentloaded", { timeout: 5_000 }).catch(() => undefined);
-        // Match what a direct Claude + Playwright-MCP agent sees: `ariaSnapshot({mode:"ai"})`
-        // carries element STATE (disabled/checked/expanded/selected/value) and a stable
-        // [ref=eNN] on each interactive element, so the brain targets them unambiguously via
-        // the aria-ref selector (instead of role+name, which collides when two fields share a
-        // label). This is exactly the call Playwright's own MCP uses. Falls back to the plain
-        // aria snapshot if the option is ever unavailable.
-        const aria = page;
-        const full = (await aria.ariaSnapshot({ mode: "ai" }).catch(() => "")) ||
-            (await page.locator("body").ariaSnapshot().catch(() => ""));
+        const mcp = await this.getMcp();
+        const res = await mcp.callTool("browser_snapshot");
+        const full = this.extractSnapshot(mcp.textOf(res));
         const MAX = 16_000;
         const snapshot = full.length > MAX ? `${full.slice(0, MAX)}\n… [snapshot truncated]` : full;
         return {
@@ -640,9 +747,10 @@ export class LocalRunner {
         };
     }
     /**
-     * Perform one brain-issued action on the active page. Elements are addressed
-     * by ARIA role + accessible name (from the snapshot) via Playwright's
-     * getByRole — robust and selector-free. Returns the resulting page state.
+     * Perform one brain-issued action via the standard `@playwright/mcp` tools
+     * (browser_click / browser_type / browser_select_option / …), targeting the
+     * element by its snapshot ref. A tool-level failure is thrown so the workflow
+     * surfaces it to the brain as an `error` (which drives its self-correction).
      */
     async browserAct(action, resumePath) {
         const page = await this.getActivePage();
@@ -656,9 +764,31 @@ export class LocalRunner {
         }
         if (this.applyResumePath)
             this.armFileAutoAttach(page, this.applyResumePath);
-        // The selector-free action switch lives in browser-actions.ts (kept this
-        // file from ballooning); the post-action settle + result stay here.
-        await performBrowserAction(page, action, (p, loc) => this.clickRobustly(p, loc), this.applyResumePath);
+        const mcp = await this.getMcp();
+        const res = await this.callBrowserTool(mcp, action);
+        const text = mcp.textOf(res).trim();
+        // A native page dialog (alert/confirm/beforeunload) puts the standard server
+        // into a modal state that blocks EVERY other tool until it's handled. The brain
+        // has no dialog vocabulary (the old runtime relied on Playwright's default
+        // auto-dismiss, which @playwright/mcp overrides), so clear it transparently and
+        // report it as a successful, informative step — the brain proceeds on the next
+        // snapshot instead of thrashing into a needless human handoff.
+        if (res.isError && /modal state/i.test(text) && /dialog/i.test(text)) {
+            const dialogMsg = (text.match(/dialog with message "([^"]*)"/)?.[1] || "").slice(0, 160);
+            await mcp.callTool("browser_handle_dialog", { accept: true }).catch(() => undefined);
+            await page.waitForTimeout(400).catch(() => undefined);
+            const settled = await this.getActivePage();
+            await settled.waitForLoadState("domcontentloaded", { timeout: 6_000 }).catch(() => undefined);
+            return {
+                ok: true,
+                url: settled.url(),
+                title: await settled.title().catch(() => ""),
+                challenge: await detectHumanChallenge(settled),
+                feedback: dialogMsg ? `a native dialog was accepted: "${dialogMsg}"` : "a native dialog was accepted",
+            };
+        }
+        if (res.isError)
+            throw new RunnerInputError(this.conciseFeedback(text) || `${action.action} failed`);
         // A click may trigger SPA navigation or open a new tab/popup. Give it a beat
         // to settle and follow the (possibly new) active page, so the next snapshot
         // reflects the new state — not the element the brain just clicked.
@@ -672,36 +802,14 @@ export class LocalRunner {
         if (action.action === "click" || action.action === "navigate" || action.action === "key") {
             await active.waitForLoadState("networkidle", { timeout: 3_500 }).catch(() => undefined);
         }
-        // Read back what a write action actually did (real field value + any validation
-        // error) so the brain gets semantic feedback, not just "Playwright didn't throw".
-        const feedback = await inspectField(active, action).catch(() => "");
         return {
             ok: true,
             url: active.url(),
             title: await active.title().catch(() => ""),
             challenge: await detectHumanChallenge(active),
-            feedback: feedback || undefined,
+            feedback: this.conciseFeedback(text) || undefined,
         };
     }
-    /**
-     * Click an element through escalating fallbacks: normal → force (bypass
-     * actionability) → scroll-into-view + click the element's CENTER COORDINATES
-     * with the mouse. The coordinate click defeats custom-styled controls (hidden
-     * inputs, overlays) that swallow locator clicks. Returns false only if the
-     * element can't be located/positioned at all.
-     */
-    async clickRobustly(page, loc) {
-        if (await loc.click({ timeout: 6_000 }).then(() => true).catch(() => false))
-            return true;
-        if (await loc.click({ force: true, timeout: 4_000 }).then(() => true).catch(() => false))
-            return true;
-        await loc.scrollIntoViewIfNeeded({ timeout: 3_000 }).catch(() => undefined);
-        const box = await loc.boundingBox().catch(() => null);
-        if (!box)
-            return false;
-        await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2).catch(() => undefined);
-        return true;
-    }
     // ── Agent-driven application lifecycle (called by the remote Workflow brain) ──
     /** Append a decision/event to the agent task's activity (the console trace). */
     browserEvent(taskId, type, message, data) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@proagentstore/cli",
-  "version": "0.3.8",
+  "version": "0.4.1",
   "description": "CLI for creating, publishing, and running ProAgentStore agents",
   "license": "MIT",
   "type": "module",
@@ -30,6 +30,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@modelcontextprotocol/sdk": "1.29.0",
+    "@playwright/mcp": "0.0.77",
     "chalk": "^5.6.2",
     "commander": "^13.0.0",
     "playwright": "^1.60.0"

package/dist/browser-runner/browser-actions.js

@@ -1,357 +0,0 @@
-import { existsSync } from "node:fs";
-import { resolve } from "node:path";
-import { RunnerInputError } from "./errors.js";
-/**
- * Enter text and VERIFY it actually took, escalating through the best-practice ladder
- * for masked / framework-controlled inputs (intl-tel, React-controlled, etc.):
- *   1) fill()  2) pressSequentially() + blur()  3) native value setter + dispatch events.
- * Returns true once the field holds the value. Generic — no site/widget-specific logic.
- */
-async function typeRobustly(loc, text) {
-    const norm = (s) => s.replace(/\s+/g, "");
-    const took = async () => {
-        const v = await loc.inputValue({ timeout: 1_500 }).catch(() => null);
-        if (v == null)
-            return false;
-        return v === text || norm(v) === norm(text) || (norm(text).length > 3 && norm(v).includes(norm(text)));
-    };
-    // 1. Playwright fill — fires input/change for most inputs.
-    if (await loc.fill(text, { timeout: 6_000 }).then(() => true).catch(() => false)) {
-        if (await took())
-            return true;
-    }
-    // 2. Real per-character typing + blur — for masks/typeaheads that only validate on key events.
-    try {
-        await loc.click({ timeout: 3_000 });
-        await loc.fill("", { timeout: 2_000 }).catch(() => undefined); // clear first
-        await loc.pressSequentially(text, { delay: 25, timeout: 8_000 });
-        await loc.blur().catch(() => undefined);
-        if (await took())
-            return true;
-    }
-    catch {
-        /* fall through */
-    }
-    // 3. Native value setter + dispatched events — for React/Vue-controlled inputs where
-    //    fill() sets the DOM value but the framework's onChange never fires.
-    try {
-        await loc.evaluate((el, val) => {
-            const input = el;
-            const proto = input instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype;
-            const setter = Object.getOwnPropertyDescriptor(proto, "value")?.set;
-            if (setter)
-                setter.call(input, val);
-            else
-                input.value = val;
-            input.dispatchEvent(new Event("input", { bubbles: true }));
-            input.dispatchEvent(new Event("change", { bubbles: true }));
-            input.dispatchEvent(new Event("blur", { bubbles: true }));
-        }, text);
-        if (await took())
-            return true;
-    }
-    catch {
-        /* fall through */
-    }
-    return false;
-}
-/**
- * Execute one selector-free browser action — address by ARIA role + accessible
- * name, with robust fallbacks for custom comboboxes, typeaheads, fuzzy <select>
- * matches, hidden checkboxes, and hidden file inputs. Pure over `page`; the
- * runner passes its clickRobustly helper. The caller handles post-action settle.
- */
-export async function performBrowserAction(page, action, clickRobustly, resumeFile) {
-    const locate = () => {
-        // Prefer the stable snapshot ref (aria-ref) — points at the exact element, so two
-        // fields sharing a label (e.g. a phone "Country" code and an address "Country")
-        // are never confused. Fall back to role+name for older snapshots.
-        if (action.ref)
-            return page.locator(`aria-ref=${action.ref}`);
-        const role = action.role;
-        let loc = role
-            ? page.getByRole(role, action.name ? { name: action.name } : undefined)
-            : page.getByText(action.name ?? "", { exact: false });
-        loc = typeof action.nth === "number" ? loc.nth(action.nth) : loc.first();
-        return loc;
-    };
-    switch (action.action) {
-        case "navigate":
-            if (!action.url || !/^https?:\/\//.test(action.url))
-                throw new RunnerInputError("navigate requires an http(s) url");
-            await page.goto(action.url, { waitUntil: "domcontentloaded", timeout: 30_000 });
-            break;
-        case "type": {
-            const text = String(action.text ?? "");
-            const loc = locate();
-            // Verified ladder (fill → pressSequentially+blur → setter+dispatch) — makes the
-            // value actually take on masked / framework-controlled inputs.
-            if (await typeRobustly(loc, text))
-                break;
-            // The field may be a combobox or typeahead rather than a bare textbox.
-            if (action.name && (await page.getByRole("combobox", { name: action.name }).fill(text, { timeout: 3_000 }).then(() => true).catch(() => false)))
-                break;
-            if (action.name && (await page.getByLabel(action.name).fill(text, { timeout: 3_000 }).then(() => true).catch(() => false)))
-                break;
-            await clickRobustly(page, loc);
-            await page.keyboard.type(text, { delay: 15 }).catch(() => undefined);
-            break;
-        }
-        case "click":
-            if (!(await clickRobustly(page, locate())))
-                throw new RunnerInputError("could not click the target");
-            break;
-        case "select": {
-            const value = String(action.text ?? "");
-            const loc = locate();
-            const pickOption = async () => {
-                // Custom dropdowns render options async/with animation — WAIT for the list
-                // to appear before clicking (else you click an overlay or nothing).
-                await page.locator('[role="option"], [role="listbox"] li, ul[role="listbox"] > *').first().waitFor({ state: "visible", timeout: 2_000 }).catch(() => undefined);
-                const opt = page.getByRole("option", { name: value, exact: false }).first();
-                if ((await opt.count().catch(() => 0)) > 0)
-                    return opt.click({ timeout: 2_500 }).then(() => true).catch(() => false);
-                // Fallback: any visible option/li whose text contains the value.
-                const alt = page.locator('[role="option"], li').filter({ hasText: value }).first();
-                if ((await alt.count().catch(() => 0)) > 0)
-                    return alt.click({ timeout: 2_500 }).then(() => true).catch(() => false);
-                return false;
-            };
-            // 1. Native <select>: exact label/value, then a fuzzy (case/punctuation
-            //    -insensitive) match so "Decline to self-identify" hits the option
-            //    "Decline To Self Identify".
-            if (await loc.selectOption({ label: value }, { timeout: 4_000 }).then(() => true).catch(() => false))
-                break;
-            if (await loc.selectOption(value, { timeout: 2_500 }).then(() => true).catch(() => false))
-                break;
-            const fuzzy = await loc.evaluate((el, want) => {
-                if (!(el instanceof HTMLSelectElement))
-                    return false;
-                const norm = (s) => s.toLowerCase().replace(/[^a-z0-9]/g, "");
-                const w = norm(want);
-                const opt = Array.from(el.options).find((o) => { const t = norm(o.textContent || ""); return t === w || (w.length > 3 && (t.includes(w) || w.includes(t))); });
-                if (!opt)
-                    return false;
-                el.value = opt.value;
-                el.dispatchEvent(new Event("input", { bubbles: true }));
-                el.dispatchEvent(new Event("change", { bubbles: true }));
-                return true;
-            }, value).catch(() => false);
-            if (fuzzy)
-                break;
-            // 2. Custom combobox / typeahead: open it, try a visible matching option.
-            await clickRobustly(page, loc);
-            await page.waitForTimeout(350).catch(() => undefined);
-            if (await pickOption())
-                break;
-            // 3. Typeahead: type to filter, then pick the suggestion — by click, else
-            //    keyboard (ArrowDown+Enter), which beats clicking a dropdown that
-            //    closes on blur.
-            await page.keyboard.type(value, { delay: 15 }).catch(() => undefined);
-            await page.waitForTimeout(550).catch(() => undefined);
-            if (await pickOption())
-                break;
-            await page.keyboard.press("ArrowDown").catch(() => undefined);
-            await page.keyboard.press("Enter").catch(() => undefined);
-            break;
-        }
-        case "check": {
-            const loc = locate();
-            // Custom checkboxes hide the real <input> (opacity:0 / behind a label /
-            // a div[role=checkbox]). Try Playwright's check, then force, then a
-            // direct DOM tick matched by the checkbox's label text.
-            if (await loc.check({ timeout: 5_000 }).then(() => true).catch(() => false))
-                break;
-            if (await loc.check({ force: true, timeout: 3_000 }).then(() => true).catch(() => false))
-                break;
-            const ticked = await page
-                .evaluate((rawName) => {
-                const norm = (s) => (s || "").toLowerCase().replace(/\s+/g, " ").trim();
-                const needle = norm(rawName).slice(0, 30);
-                const boxes = Array.from(document.querySelectorAll('input[type="checkbox"], [role="checkbox"]'));
-                const labelOf = (b) => {
-                    const forId = b.id ? document.querySelector(`label[for="${b.id}"]`)?.textContent : "";
-                    return norm(b.closest("label")?.textContent || b.getAttribute("aria-label") || forId || b.parentElement?.textContent || "");
-                };
-                const tick = (b) => {
-                    if (b instanceof HTMLInputElement) {
-                        b.checked = true;
-                        b.dispatchEvent(new Event("input", { bubbles: true }));
-                        b.dispatchEvent(new Event("change", { bubbles: true }));
-                    }
-                    else {
-                        b.setAttribute("aria-checked", "true");
-                    }
-                    b.click?.();
-                };
-                const match = boxes.find((b) => { const l = labelOf(b); return needle && (l.includes(needle) || (l.length > 12 && needle.includes(l.slice(0, 12)))); });
-                const target = match || (boxes.length === 1 ? boxes[0] : undefined);
-                if (!target)
-                    return false;
-                tick(target);
-                return true;
-            }, action.name ?? "")
-                .catch(() => false);
-            if (ticked)
-                break;
-            if (!(await clickRobustly(page, locate())))
-                throw new RunnerInputError("could not find/tick the checkbox");
-            break;
-        }
-        case "upload": {
-            // Always attach via Playwright — never a native dialog. Handles both a
-            // direct <input type=file> (set files on it) and a styled "Upload"
-            // button that opens a native chooser (intercept the filechooser).
-            // The LLM doesn't know the runner's local path, so prefer the résumé the
-            // runner already resolved (downloaded from the platform); fall back to an
-            // explicit action.file only if it's a real local file.
-            const explicit = action.file ? resolve(String(action.file)) : "";
-            const file = resumeFile && existsSync(resumeFile)
-                ? resumeFile
-                : explicit && existsSync(explicit) ? explicit : "";
-            if (!file)
-                throw new RunnerInputError("no résumé available to upload — upload one in the console (Knowledge → Résumé)");
-            let done = false;
-            // 1. Label-associated input (some forms).
-            if (action.name) {
-                done = await page.getByLabel(action.name).setInputFiles(file, { timeout: 4_000 }).then(() => true).catch(() => false);
-            }
-            // 2. The real <input type=file> directly — most ATS (Greenhouse, Lever…)
-            //    hide it behind a styled "Attach" button; setInputFiles works on a
-            //    hidden input and fires the change event, no native dialog.
-            if (!done) {
-                done = await page.locator('input[type="file"]').first().setInputFiles(file, { timeout: 4_000 }).then(() => true).catch(() => false);
-            }
-            // 3. Styled uploader with no input at all: click the trigger + intercept
-            //    the native file chooser.
-            if (!done) {
-                const chooserP = page.waitForEvent("filechooser", { timeout: 8_000 }).catch(() => null);
-                await locate().click({ timeout: 8_000 }).catch(() => undefined);
-                const chooser = await chooserP;
-                if (chooser)
-                    await chooser.setFiles(file);
-                else
-                    throw new RunnerInputError("no file upload control found for upload action");
-            }
-            break;
-        }
-        case "key":
-            await page.keyboard.press(String(action.key ?? "Enter"));
-            break;
-        case "scroll":
-            await page.mouse.wheel(0, action.dy ?? 600);
-            break;
-        case "wait":
-            await page.waitForTimeout(Math.min(5_000, action.ms ?? 1_000));
-            break;
-        default: {
-            const name = String(action.action || "");
-            // The brain sometimes invents click variants (triple_click / double_click
-            // to select a field's text before retyping). Handle them generically
-            // rather than failing the whole task — clickCount selects the text.
-            if (/click/i.test(name)) {
-                const loc = locate();
-                const clickCount = /triple/i.test(name) ? 3 : /double|dbl/i.test(name) ? 2 : 1;
-                if (!(await loc.click({ clickCount, timeout: 6_000 }).then(() => true).catch(() => false))) {
-                    await clickRobustly(page, loc);
-                }
-                break;
-            }
-            throw new RunnerInputError(`Unsupported action "${name}". Use one of: click, type, select, check, upload, navigate, scroll, key, wait. To clear or replace a field, use "type" — it overwrites the existing value.`);
-        }
-    }
-}
-/**
- * After a WRITE action (type/select/check), read back what the field actually
- * holds now + any validation error rendered near it. This is the semantic
- * feedback the brain otherwise lacks: `fill()` succeeding doesn't mean the value
- * "took" (masked/intl widgets concatenate; validators reject a format). Returns a
- * short human string for the action log, or "" when there's nothing notable.
- * Never throws.
- */
-export async function inspectField(page, action) {
-    if (action.action !== "type" && action.action !== "select" && action.action !== "check")
-        return "";
-    if (!action.name && !action.ref)
-        return "";
-    try {
-        const role = action.role;
-        let loc = action.ref
-            ? page.locator(`aria-ref=${action.ref}`)
-            : role
-                ? page.getByRole(role, action.name ? { name: action.name } : undefined)
-                : page.getByText(action.name ?? "", { exact: false });
-        if (!action.ref)
-            loc = typeof action.nth === "number" ? loc.nth(action.nth) : loc.first();
-        const el = await loc.elementHandle({ timeout: 1_500 }).catch(() => null);
-        if (!el)
-            return "";
-        const info = await el
-            .evaluate((node) => {
-            const e = node;
-            const raw = typeof e.value === "string" ? e.value : (e.getAttribute("aria-checked") ?? e.textContent ?? "");
-            const value = String(raw).trim().slice(0, 120);
-            const invalid = e.getAttribute("aria-invalid") === "true" || (typeof e.matches === "function" && e.matches(":invalid"));
-            let err = "";
-            const rx = /invalid|required|must|valid|error|format|enter a|please/i;
-            const describedby = e.getAttribute("aria-describedby");
-            if (describedby) {
-                for (const id of describedby.split(/\s+/)) {
-                    const d = document.getElementById(id);
-                    const t = (d?.textContent || "").trim();
-                    if (t && rx.test(t)) {
-                        err = t;
-                        break;
-                    }
-                }
-            }
-            if (!err) {
-                const scope = e.closest("[class*=field], [class*=form-group], [class*=form-item], fieldset") || e.parentElement;
-                const cand = scope?.querySelector('[role="alert"], [class*=error i], [class*=invalid i], [class*=danger i], [class*=help-block i]');
-                const t = (cand?.textContent || "").trim();
-                if (t && rx.test(t) && t.length < 180)
-                    err = t;
-            }
-            const disabled = e.hasAttribute("disabled") || e.getAttribute("aria-disabled") === "true" || e.hasAttribute("readonly");
-            return { value, invalid, err: err.replace(/\s+/g, " ").slice(0, 180), tag: e.tagName.toLowerCase(), html: (e.outerHTML || "").replace(/\s+/g, " ").slice(0, 500), disabled };
-        })
-            .catch(() => null);
-        if (!info)
-            return "";
-        const typed = String(action.text ?? "").trim();
-        const stuck = !!typed && info.value === typed; // our value actually took
-        const parts = [];
-        // Report the field's real value when it differs from what we sent, or when the
-        // field is authoritatively invalid.
-        if (info.value && (info.invalid || !stuck)) {
-            parts.push(`"${action.name}" now reads "${info.value}"${typed && info.value !== typed ? ` (you sent "${typed}")` : ""}`);
-        }
-        // Flag REJECTED ONLY on an authoritative per-field signal (aria-invalid / :invalid)
-        // or when our value clearly did NOT take. A nearby error while the value stuck is
-        // often stale (widgets show a "format" error mid-type that clears on blur/submit)
-        // or belongs to another field — reporting it as REJECTED sent the brain into a
-        // false-retry loop on a value that was actually accepted.
-        if (info.invalid || (typed && !stuck)) {
-            if (info.disabled) {
-                // A disabled/read-only control can't be changed — it's almost always already
-                // set correctly. Tell the brain to move on instead of fixating on it.
-                parts.unshift(`ℹ "${action.name}" is DISABLED / read-only — it is likely already set correctly; do NOT keep trying, move on to other fields.`);
-            }
-            else {
-                if (info.err)
-                    parts.unshift(`⚠ "${action.name}" REJECTED: "${info.err}"`);
-                else if (info.invalid)
-                    parts.unshift(`⚠ "${action.name}" is marked invalid`);
-                // Value didn't take / field invalid → show the widget's real DOM so the brain
-                // can pick the right interaction (custom dropdown → click to open + click the
-                // option by text; masked input → a different shape) instead of blindly retrying.
-                if (info.html)
-                    parts.push(`DOM: <${info.tag}> ${info.html}`);
-            }
-        }
-        return parts.join("; ");
-    }
-    catch {
-        return "";
-    }
-}

package/dist/browser-runner/human-mouse.js

@@ -1,71 +0,0 @@
-const rand = (min, max) => min + Math.random() * (max - min);
-const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-// easeInOutQuad — slow start, fast middle, slow approach (how a hand decelerates onto a target).
-const ease = (t) => (t < 0.5 ? 2 * t * t : 1 - (-2 * t + 2) ** 2 / 2);
-function cubicBezier(p0, p1, p2, p3, t) {
-    const u = 1 - t;
-    return {
-        x: u * u * u * p0.x + 3 * u * u * t * p1.x + 3 * u * t * t * p2.x + t * t * t * p3.x,
-        y: u * u * u * p0.y + 3 * u * u * t * p1.y + 3 * u * t * t * p2.y + t * t * t * p3.y,
-    };
-}
-/** Move the cursor from `from` to `to` along a randomized human-like arc. */
-export async function humanMoveTo(cdp, from, to) {
-    const dist = Math.hypot(to.x - from.x, to.y - from.y);
-    const steps = Math.max(12, Math.min(42, Math.round(dist / 9)));
-    // Control points pushed off the straight line so the path bows like a real
-    // hand's arc — bigger arc for longer travel.
-    const off = Math.min(90, dist * 0.3);
-    const c1 = {
-        x: from.x + (to.x - from.x) * 0.33 + rand(-off, off),
-        y: from.y + (to.y - from.y) * 0.33 + rand(-off, off),
-    };
-    const c2 = {
-        x: from.x + (to.x - from.x) * 0.66 + rand(-off, off),
-        y: from.y + (to.y - from.y) * 0.66 + rand(-off, off),
-    };
-    for (let i = 1; i <= steps; i++) {
-        const p = cubicBezier(from, c1, c2, to, ease(i / steps));
-        await cdp.send("Input.dispatchMouseEvent", {
-            type: "mouseMoved",
-            x: p.x + rand(-0.6, 0.6),
-            y: p.y + rand(-0.6, 0.6),
-        });
-        await sleep(rand(5, 15));
-    }
-}
-/** Click `to` with a human approach + natural press/release dwell. */
-export async function humanClickAt(cdp, from, to) {
-    await humanMoveTo(cdp, from, to);
-    await sleep(rand(40, 130));
-    await cdp.send("Input.dispatchMouseEvent", { type: "mousePressed", x: to.x, y: to.y, button: "left", buttons: 1, clickCount: 1 });
-    await sleep(rand(40, 100));
-    await cdp.send("Input.dispatchMouseEvent", { type: "mouseReleased", x: to.x, y: to.y, button: "left", buttons: 0, clickCount: 1 });
-}
-/** Pick a natural target point inside a box (not dead-center) + an off-screen-ish start. */
-export function targetIn(box) {
-    const to = {
-        x: box.x + box.width / 2 + rand(-box.width * 0.2, box.width * 0.2),
-        y: box.y + box.height / 2 + rand(-box.height * 0.25, box.height * 0.25),
-    };
-    const from = { x: to.x + rand(-260, -40), y: to.y - rand(60, 260) };
-    return { from, to };
-}
-/**
- * Best-effort humanized approach to an element before the caller's real click,
- * so the cursor arrives the way a hand would. Never throws — humanization must
- * not break the underlying action.
- */
-export async function humanApproach(page, box) {
-    if (!box)
-        return;
-    try {
-        const cdp = await page.context().newCDPSession(page);
-        const { from, to } = targetIn(box);
-        await humanMoveTo(cdp, from, to);
-        await cdp.detach().catch(() => undefined);
-    }
-    catch {
-        // humanization is an enhancement, never a hard dependency
-    }
-}