@trusty-squire/mcp 0.5.9 → 0.6.0

package/dist/bot/browser.js

@@ -25,6 +25,7 @@ import { chromium as baseChromium } from "playwright";
 import { createRequire } from "node:module";
 import { detectAsn } from "./asn.js";
 import { CHROME_PROFILE_DIR } from "./profile.js";
+import { startXvfb, xvfbAvailable } from "./xvfb.js";
 // Lazy registration: installing the plugin mutates the chromium singleton
 // from playwright-extra so we only do it once per process. We require()
 // the CJS modules lazily (the stealth toolchain only ships CJS) and treat
@@ -170,6 +171,19 @@ export class BrowserController {
     // parked here so settleAfterOAuth() can switch back to it once the
     // Google handshake completes.
     oauthProductPage = null;
+    // F13 — on-demand Xvfb. Set when start() determined the host has no
+    // display surface but Xvfb is available, so Chrome can run with
+    // `headless: false` against a virtual display (Cloudflare/Stytch et
+    // al. detect Chromium-headless and block their signup forms). Torn
+    // down by close().
+    xvfb = null;
+    // F13 — which launch path start() took. Surfaced via .launchMode so
+    // the agent can push it into the run's step trail and we can see
+    // (from outside the box) whether the bot ran headed.
+    launchedMode = "unknown";
+    get launchMode() {
+        return this.launchedMode;
+    }
     constructor(opts = {}) {
         this.humanize = opts.humanize ?? true;
         this.profileDir = opts.profileDir ?? CHROME_PROFILE_DIR;
@@ -214,13 +228,71 @@ export class BrowserController {
                     ? ` loc=${geo.geolocation.latitude},${geo.geolocation.longitude}`
                     : ""));
         }
+        // F13 — decide whether to spin up Xvfb and run Chrome headed.
+        // Modern SaaS signups (Cloudflare/Stytch, Clerk, Auth0) detect
+        // Chromium-headless via JS fingerprints and gate their forms
+        // behind the check. Running headed against Xvfb defeats the gate
+        // — the user never sees the display.
+        //
+        // The decision matrix:
+        //   - UNIVERSAL_BOT_HEADLESS=true (explicit opt-in): keep true
+        //     headless. CI / Codespaces that lack Xvfb.
+        //   - UNIVERSAL_BOT_HEADLESS=false (explicit opt-out): the
+        //     current pre-F13 behavior — DISPLAY must exist already.
+        //   - default + DISPLAY set: run headed against the existing
+        //     display (laptop/desktop install).
+        //   - default + no DISPLAY + Xvfb on PATH: spawn Xvfb, run
+        //     headed against it (the headless-server install — what
+        //     Cloudflare needed).
+        //   - default + no DISPLAY + no Xvfb: fall back to true
+        //     headless with a clear stderr warning.
+        let chromeEnv;
+        let chromeHeadless;
+        const explicitHeadless = process.env.UNIVERSAL_BOT_HEADLESS;
+        const hostHasDisplay = process.platform === "darwin" ||
+            process.platform === "win32" ||
+            (typeof process.env.DISPLAY === "string" && process.env.DISPLAY.length > 0);
+        if (explicitHeadless === "true") {
+            chromeHeadless = true;
+            this.launchedMode = "headless";
+        }
+        else if (explicitHeadless === "false") {
+            chromeHeadless = false;
+            this.launchedMode = "display";
+        }
+        else if (hostHasDisplay) {
+            chromeHeadless = false;
+            this.launchedMode = "display";
+        }
+        else if (xvfbAvailable()) {
+            try {
+                this.xvfb = await startXvfb({ width: 1280, height: 720 });
+                chromeEnv = { ...process.env, DISPLAY: this.xvfb.display };
+                chromeHeadless = false;
+                this.launchedMode = "xvfb";
+                console.error(`[universal-bot] no DISPLAY — spawned Xvfb at ${this.xvfb.display} for headed Chrome`);
+            }
+            catch (err) {
+                console.error(`[universal-bot] Xvfb failed (${err instanceof Error ? err.message : String(err)}) — ` +
+                    `falling back to true headless; Cloudflare/Stytch-class signups may fail`);
+                chromeHeadless = true;
+                this.launchedMode = "headless";
+            }
+        }
+        else {
+            console.error(`[universal-bot] no DISPLAY and Xvfb not installed — running true headless. ` +
+                `For Cloudflare/Stytch-class signups install xvfb: apt-get install -y xvfb`);
+            chromeHeadless = true;
+            this.launchedMode = "headless";
+        }
         // T3: a PERSISTENT context. The profile dir carries the user's
         // Google session (established by `mcp login` — see google-login.ts),
         // so the OAuth-first signup path reuses it instead of starting
         // logged-out. launchPersistentContext takes launch + context
         // options in one call.
         const context = await getChromium().launchPersistentContext(this.profileDir, {
-            headless: process.env.UNIVERSAL_BOT_HEADLESS !== "false",
+            headless: chromeHeadless,
+            ...(chromeEnv !== undefined ? { env: chromeEnv } : {}),
             // `channel:` selects a real installed browser over the bundled
             // binary; omitted entirely when null.
             ...(channel !== null ? { channel } : {}),
@@ -241,9 +313,20 @@ export class BrowserController {
             // timezone + geolocation track the real egress (T3.1); a fixed
             // default when the probe failed.
             timezoneId: geo?.timezoneId ?? "America/New_York",
-            ...(geo?.geolocation !== undefined
-                ? { geolocation: geo.geolocation, permissions: ["geolocation"] }
-                : {}),
+            // F10: `clipboard-read` is what makes `navigator.clipboard.readText()`
+            // return the user's just-clicked Copy-button value, which is how
+            // every modern API-key modal (OpenRouter, Anthropic, OpenAI,
+            // Stripe) reveals the full secret — the visible display is
+            // masked / truncated and only the clipboard has the whole key.
+            // `clipboard-write` is a freebie; some Copy buttons no-op without
+            // it. Granting both at context-creation time so we don't have to
+            // re-grant on every nav.
+            permissions: [
+                ...(geo?.geolocation !== undefined ? ["geolocation"] : []),
+                "clipboard-read",
+                "clipboard-write",
+            ],
+            ...(geo?.geolocation !== undefined ? { geolocation: geo.geolocation } : {}),
         });
         this.context = context;
         // Patch the navigator.webdriver flag — most anti-bot heuristics look here.
@@ -578,25 +661,149 @@ export class BrowserController {
             await this.page.check(selector, { force: true });
         }
     }
-    // Pick a valid option for a <select> (F3 T6). The bot must not call
-    // type() on a <select> (Sentry: "Element is not an <input>").
-    // Signup <select>s are country / region / role pickers — any
-    // non-placeholder option satisfies the form. Throws (caught by the
-    // executor) when the select has no selectable option.
-    async selectOption(selector) {
+    // Pick a valid option for either a native <select> OR a custom
+    // ARIA combobox (Radix, Headless UI, React Aria, cmdk — F11). The
+    // bot must not call type() on a select-shaped element (Sentry,
+    // legacy form path: "Element is not an <input>"); modern dashboards
+    // increasingly render permission / role / region pickers as
+    // <button role="combobox"> that open a <ul role="listbox"> with
+    // <li role="option"> children, so Playwright's selectOption fails
+    // with "no selectable option" on them.
+    //
+    // Dispatch: read the element's tag. <select> → native path
+    // (existing behavior, picks the first non-placeholder option).
+    // Anything else → combobox path (click to open, find role=option,
+    // click the chosen one).
+    //
+    // `optionMatcher` is the planner-supplied text of the option to
+    // pick (e.g. "Project: Read"). Case-insensitive substring match
+    // against the option's visible text. When undefined, picks the
+    // first option — preserves the existing behavior for native
+    // selects whose contents are interchangeable (country pickers).
+    async selectOption(selector, optionMatcher) {
         if (!this.page)
             throw new Error("Browser not started");
         await this.page.waitForSelector(selector, { state: "attached", timeout: 10000 });
-        const values = await this.page
-            .locator(`${selector} option`)
-            .evaluateAll((opts) => opts
-            .map((o) => (o instanceof HTMLOptionElement ? o.value : ""))
-            .filter((v) => v.length > 0));
-        const first = values[0];
-        if (first === undefined) {
-            throw new Error(`<select> ${selector} has no selectable option`);
-        }
-        await this.page.selectOption(selector, first);
+        const tagName = await this.page
+            .locator(selector)
+            .first()
+            .evaluate((node) => node.tagName.toLowerCase());
+        if (tagName === "select") {
+            // Native path — unchanged.
+            const values = await this.page
+                .locator(`${selector} option`)
+                .evaluateAll((opts) => opts
+                .map((o) => (o instanceof HTMLOptionElement ? o.value : ""))
+                .filter((v) => v.length > 0));
+            const first = values[0];
+            if (first === undefined) {
+                throw new Error(`<select> ${selector} has no selectable option`);
+            }
+            // When the planner specified an option, prefer the one whose
+            // visible text matches it; fall back to first.
+            let chosenValue = first;
+            if (optionMatcher !== undefined) {
+                const matcherLower = optionMatcher.toLowerCase();
+                const matched = await this.page
+                    .locator(`${selector} option`)
+                    .evaluateAll((opts, needle) => opts
+                    .filter((o) => o instanceof HTMLOptionElement)
+                    .find((o) => o.textContent?.toLowerCase().includes(needle))
+                    ?.value ?? null, matcherLower);
+                if (typeof matched === "string" && matched.length > 0) {
+                    chosenValue = matched;
+                }
+            }
+            await this.page.selectOption(selector, chosenValue);
+            return;
+        }
+        // Custom combobox path. Sentry, Radix, Headless UI, React Aria
+        // — every modern React picker emits role=option on its items.
+        await this.selectFromCombobox(selector, optionMatcher);
+    }
+    // F11 (+rc.7 hardening): click a combobox trigger, wait for the
+    // listbox to open, click an option.
+    //
+    // Tries option-selector patterns in priority order — each tier
+    // targets one combobox-library convention. The text-based final
+    // tier catches libraries that ship NO ARIA roles at all (Sentry's
+    // permissions picker is the canonical case: it uses `<div>`s with
+    // plain text for options and pure JS click handlers).
+    //
+    //   1. [role=option]            — Radix, Headless UI, React Aria, cmdk
+    //   2. [role=menuitem]          — ARIA menu pattern (libs that model
+    //                                 a dropdown as a menu)
+    //   3. [role=listbox] li        — listbox container without role
+    //                                 attribute on its children
+    //   4. text-based (matcher only) — after the trigger click, any newly-
+    //                                 visible element whose text matches
+    //                                 the planner-supplied label is
+    //                                 almost certainly the option. Only
+    //                                 enabled when a matcher exists,
+    //                                 since "first text on the page"
+    //                                 with no matcher would catch
+    //                                 unrelated UI text.
+    async selectFromCombobox(triggerSelector, optionMatcher) {
+        if (!this.page)
+            throw new Error("Browser not started");
+        await this.humanClick(triggerSelector);
+        const patternSelectors = [
+            '[role="option"]:visible',
+            '[role="menuitem"]:visible',
+            '[role="listbox"]:visible li:visible',
+        ];
+        const triedDescriptors = [];
+        for (const sel of patternSelectors) {
+            triedDescriptors.push(sel);
+            const locator = this.page.locator(sel);
+            try {
+                await locator.first().waitFor({ state: "visible", timeout: 1500 });
+            }
+            catch {
+                continue;
+            }
+            const count = await locator.count();
+            if (count === 0)
+                continue;
+            await this.pickComboboxOption(locator, optionMatcher);
+            return;
+        }
+        // ARIA tiers all empty. Text-based fallback, only if the planner
+        // told us WHICH option to pick — without a matcher, "first text
+        // on the page" would click unrelated UI.
+        if (optionMatcher !== undefined) {
+            const byText = this.page.getByText(optionMatcher, { exact: false }).first();
+            triedDescriptors.push(`text="${optionMatcher}"`);
+            try {
+                await byText.waitFor({ state: "visible", timeout: 2000 });
+                await this.humanClickLocator(byText);
+                await this.wait(0.5);
+                return;
+            }
+            catch {
+                // not found — fall through to error
+            }
+        }
+        throw new Error(`combobox ${triggerSelector}: no options found after click. ` +
+            `Tried: ${triedDescriptors.join(", ")}. ` +
+            `The trigger may not have opened a popover, or the popover uses ` +
+            `an option pattern this executor doesn't recognize.`);
+    }
+    // F11: pick an option from a Playwright Locator already-narrowed to
+    // candidates. Matcher → filter by hasText (case-insensitive by
+    // default in Playwright). No matcher → first.
+    async pickComboboxOption(options, matcher) {
+        if (matcher !== undefined) {
+            const filtered = options.filter({ hasText: matcher });
+            const filteredCount = await filtered.count();
+            if (filteredCount > 0) {
+                await this.humanClickLocator(filtered.first());
+                await this.wait(0.5);
+                return;
+            }
+        }
+        await this.humanClickLocator(options.first());
+        await this.wait(0.5);
     }
     // ───────────── humanization internals ─────────────
     // Click that mimics a real user: locate element, bezier-path the
@@ -874,6 +1081,23 @@ export class BrowserController {
         return buffer.toString("base64");
     }
     async getState() {
+        if (!this.page)
+            throw new Error("Browser not started");
+        // page.content() / page.title() / screenshot() all throw
+        // "Execution context was destroyed" when the page is mid-
+        // navigation — common after an OAuth-button click that kicks off
+        // a 3-5 hop redirect chain (sentry.io → accounts.google.com →
+        // consent → callback → onboarding). Retry once after a short
+        // settle: most navigations finish in <500ms even on slow links.
+        try {
+            return await this.snapshotState();
+        }
+        catch {
+            await this.wait(0.8);
+            return await this.snapshotState();
+        }
+    }
+    async snapshotState() {
         if (!this.page)
             throw new Error("Browser not started");
         return {
@@ -902,6 +1126,46 @@ export class BrowserController {
     //      immediate children, excluding descendants. A key in a
     //      <code>/<span>/<div> yields its clean value here even when a
     //      sibling button shares the same parent.
+    // F10: read the clipboard contents (typically populated by the
+    // user-modal's Copy button — every modern API-key reveal modal puts
+    // the full secret here while displaying a masked stub). Requires
+    // `clipboard-read` permission, granted at context creation. Returns
+    // an empty string if the clipboard is empty; throws on permission
+    // failure (caller catches and falls through to other paths).
+    async readClipboard() {
+        if (!this.page)
+            throw new Error("Browser not started");
+        return await this.page.evaluate(async () => {
+            try {
+                return await navigator.clipboard.readText();
+            }
+            catch {
+                return "";
+            }
+        });
+    }
+    // F10 fallback: ALL <input> / <textarea> values, ignoring
+    // visibility and type filters. extractCredentialCandidates
+    // deliberately skips `type=hidden` / `type=password` / invisible
+    // elements (correct for general candidate scanning), but some
+    // API-key modals stash the full key in a hidden input the masked
+    // display reads from — and that needs to be reachable when the
+    // visible extraction comes back truncated.
+    async extractAllInputValues() {
+        if (!this.page)
+            throw new Error("Browser not started");
+        return await this.page.evaluate(() => {
+            const out = [];
+            document.querySelectorAll("input, textarea").forEach((el) => {
+                if (!(el instanceof HTMLInputElement) && !(el instanceof HTMLTextAreaElement))
+                    return;
+                const value = el.value;
+                if (value.trim().length > 0)
+                    out.push(value.trim());
+            });
+            return out;
+        });
+    }
     async extractCredentialCandidates() {
         if (!this.page)
             throw new Error("Browser not started");
@@ -956,6 +1220,14 @@ export class BrowserController {
             // networkidle never settles on pages with analytics sockets or
             // long-poll — not fatal, fall through to the element wait.
         }
+        // F13 follow-up — if we landed on a full-page anti-bot interstitial
+        // (Cloudflare "Just a moment..." / Turnstile pre-clear / similar),
+        // wait for it to clear and the real page to render. networkidle
+        // sometimes fires DURING the interstitial because Cloudflare keeps
+        // the connection quiet between the verify-handshake and the
+        // redirect to the real page. Without this, the bot snapshots a
+        // 2-element interstitial inventory and bails.
+        await this.waitForAntiBotInterstitialToClear(timeoutMs);
         try {
             await this.page.waitForSelector("input, button", {
                 state: "visible",
@@ -967,6 +1239,72 @@ export class BrowserController {
             // anyway; it fails cleanly rather than hanging.
         }
     }
+    // Cloudflare and similar gateways serve a full-page interstitial
+    // ("Just a moment..." / Turnstile pre-clear) before the real page.
+    // The challenge usually clears within ~5-10s — the bot just needs
+    // to wait. Detected from page text patterns rather than URL: the
+    // URL stays the same; the body replaces.
+    //
+    // Returns when the interstitial is gone, or after `timeoutMs` if it
+    // never cleared. Best-effort: any unexpected error returns early
+    // rather than failing the whole signup.
+    async waitForAntiBotInterstitialToClear(timeoutMs) {
+        if (!this.page)
+            return;
+        let detected = await this.pollUntilInterstitialClears(timeoutMs);
+        if (!detected) {
+            // We either never saw an interstitial, or we saw one and it
+            // cleared on its own. Nothing more to do.
+            return;
+        }
+        // The interstitial outlived the wait. Cloudflare frequently shows
+        // "Verification successful. Wait" but then never fires the JS
+        // redirect — the challenge passed, but the redirect script got
+        // stuck or the cookie set is racing the navigation. A single
+        // reload, now that the cf_clearance cookie is set, often lets the
+        // real page render. (If the issue is a server-side risk-score
+        // block — fingerprint/IP — reload won't help, but the caller's
+        // inventory diagnostic will still surface the block.)
+        try {
+            await this.page.reload({ waitUntil: "networkidle", timeout: 10_000 });
+        }
+        catch {
+            // reload failed — proceed with what's there
+        }
+        await this.pollUntilInterstitialClears(Math.max(5000, timeoutMs / 2));
+    }
+    // One poll loop. Returns true if an interstitial was ever observed
+    // (cleared or still there at timeout), false if never seen.
+    async pollUntilInterstitialClears(timeoutMs) {
+        if (!this.page)
+            return false;
+        const deadline = Date.now() + timeoutMs;
+        let detected = false;
+        while (Date.now() < deadline) {
+            let title = "";
+            let bodyText = "";
+            try {
+                title = await this.page.title();
+                bodyText = await this.page.evaluate(() => (document.body?.innerText ?? "").slice(0, 500));
+            }
+            catch {
+                await new Promise((r) => setTimeout(r, 500));
+                continue;
+            }
+            const onInterstitial = /just a moment|performing security verification|verifying you are human|checking your browser|attention required/i.test(title + " " + bodyText);
+            if (!onInterstitial) {
+                if (detected) {
+                    // Give the freshly-revealed page a tick to hydrate before
+                    // the inventory scan.
+                    await new Promise((r) => setTimeout(r, 800));
+                }
+                return detected;
+            }
+            detected = true;
+            await new Promise((r) => setTimeout(r, 1000));
+        }
+        return detected;
+    }
     // Walk the live DOM (piercing open shadow roots) and return every
     // visible interactive element with a bot-computed selector (F3 T1).
     // The planner picks from this inventory instead of inventing
@@ -1271,6 +1609,13 @@ export class BrowserController {
         // Closing the persistent context shuts the browser down too.
         if (this.context)
             await this.context.close();
+        // F13 — release the on-demand Xvfb if we spawned one. Order
+        // matters: kill Chrome (context.close) first so it has its
+        // display until it exits, THEN kill Xvfb.
+        if (this.xvfb !== null) {
+            this.xvfb.stop();
+            this.xvfb = null;
+        }
     }
 }
 // Random integer in [min, max]. We use Math.random() (not crypto)