@tea-agent/loop-agent 0.43.0-next.9 → 0.43.0

package/dist/worker/console/chat/browser-automation.js

@@ -0,0 +1,523 @@
+/**
+ * Workspace-scoped Playwright browser used by both Operator Chat tools and
+ * the Browser panel. The browser profile lives outside the repository and is
+ * reused across Console restarts, so login state and downloads remain
+ * isolated from the user's normal browser without contaminating the repo.
+ */
+import { randomUUID } from "node:crypto";
+import { mkdir } from "node:fs/promises";
+import path from "node:path";
+import { chromium, } from "playwright-core";
+import { normalizeBrowserUrl } from "./browser-policy.js";
+export const BROWSER_VIEWPORT_WIDTH = 1280;
+export const BROWSER_VIEWPORT_HEIGHT = 900;
+const SEARCH_INPUT_SELECTORS = [
+    "input[name='wd']",
+    "input[name='q']",
+    "input[name='query']",
+    "input[type='search']",
+    "input[type='text']",
+];
+const SEARCH_SUBMIT_SELECTORS = [
+    "button[type='submit']",
+    "input[type='submit']",
+    "button:not([type])",
+];
+const DEFAULT_SEARCH_ENGINE_URL = "https://www.baidu.com/";
+const SEARCH_TYPING_DELAY_MS = 45;
+const DEFAULT_ACTION_TIMEOUT_MS = 15_000;
+const MAX_SNAPSHOT_CHARS = 24_000;
+const MAX_SCREENSHOT_BYTES = 4 * 1024 * 1024;
+export class BrowserAutomationService {
+    options;
+    browserPromise;
+    activePage;
+    tabs = new Map();
+    nextTabId = 1;
+    disposed = false;
+    dialogQueue = [];
+    downloadQueue = [];
+    constructor(options) {
+        this.options = options;
+    }
+    async navigate(rawUrl) {
+        const normalized = normalizeBrowserUrl(rawUrl);
+        if (normalized.kind !== "ok") {
+            return {
+                ok: false,
+                code: normalized.kind === "blocked" ? "BAD_SCHEME" : "INVALID_URL",
+                error: normalized.kind === "blocked"
+                    ? "browser urls must use http(s)"
+                    : "invalid browser url",
+            };
+        }
+        try {
+            const page = await this.page();
+            await page.goto(normalized.url, { waitUntil: "domcontentloaded", timeout: 30_000 });
+            const snapshot = await this.snapshot({ screenshot: false });
+            return { ok: true, page: snapshot };
+        }
+        catch (error) {
+            return { ok: false, code: "NAVIGATION_FAILED", error: errorMessage(error) };
+        }
+    }
+    async reload() {
+        const page = await this.page();
+        await page.reload({ waitUntil: "domcontentloaded", timeout: 30_000 });
+        return this.snapshot({ screenshot: false });
+    }
+    async goBack() {
+        const page = await this.page();
+        await page.goBack({ waitUntil: "domcontentloaded", timeout: 30_000 });
+        return this.snapshot({ screenshot: false });
+    }
+    async goForward() {
+        const page = await this.page();
+        await page.goForward({ waitUntil: "domcontentloaded", timeout: 30_000 });
+        return this.snapshot({ screenshot: false });
+    }
+    async snapshot(options) {
+        const page = await this.page();
+        const dialog = this.dialogFor(page);
+        if (dialog) {
+            return {
+                url: page.url(),
+                title: "浏览器弹窗",
+                tabs: await this.tabSummaries(),
+                dialog,
+                ariaSnapshot: "",
+                ...(options?.screenshot === false
+                    ? {}
+                    : { screenshot: await this.captureScreenshot(page) }),
+            };
+        }
+        const [title, aria] = await Promise.all([
+            page.title(),
+            page.ariaSnapshot({ mode: "ai", timeout: 10_000 }),
+        ]);
+        const download = await this.nextDownloadResult();
+        const screenshot = options?.screenshot === false
+            ? undefined
+            : await this.captureScreenshot(page);
+        return {
+            url: page.url(),
+            title,
+            tabs: await this.tabSummaries(),
+            dialog,
+            ariaSnapshot: truncateText(aria, MAX_SNAPSHOT_CHARS),
+            ...(screenshot ? { screenshot } : {}),
+            ...(download ? { download } : {}),
+        };
+    }
+    /**
+     * Lightweight state for iframe-driven UI. Avoids aria capture and PNG
+     * encoding on every panel poll.
+     */
+    async state() {
+        if (!this.browserPromise && !this.disposed)
+            return emptyBrowserState();
+        const page = await this.page();
+        const state = {
+            running: true,
+            url: page.url(),
+            title: await page.title().catch(() => ""),
+            tabs: await this.tabSummaries(),
+            dialog: this.dialogFor(page),
+            ariaSnapshot: "",
+        };
+        return state;
+    }
+    async search(rawQuery, rawUrl) {
+        const query = rawQuery.trim();
+        if (!query)
+            throw new BrowserAutomationError("EMPTY_QUERY", "search query is empty");
+        const target = rawUrl?.trim() || DEFAULT_SEARCH_ENGINE_URL;
+        const normalized = normalizeBrowserUrl(target);
+        if (normalized.kind !== "ok") {
+            throw new BrowserAutomationError("INVALID_URL", "search engine url must use http(s)");
+        }
+        const page = await this.page();
+        await page.bringToFront();
+        await page.goto(normalized.url, { waitUntil: "domcontentloaded", timeout: 30_000 });
+        const input = await findSearchInput(page);
+        if (!input) {
+            throw new BrowserAutomationError("SEARCH_INPUT_NOT_FOUND", "the search page did not expose a recognizable search input");
+        }
+        await input.fill("", { timeout: DEFAULT_ACTION_TIMEOUT_MS });
+        await input.pressSequentially(query, {
+            delay: SEARCH_TYPING_DELAY_MS,
+            timeout: DEFAULT_ACTION_TIMEOUT_MS,
+        });
+        await Promise.all([
+            page.waitForNavigation({ waitUntil: "domcontentloaded", timeout: 30_000 }),
+            submitSearchForm(page, input),
+        ]);
+        return this.snapshot({ screenshot: false });
+    }
+    async act(request) {
+        const page = await this.page();
+        const ref = request.ref.trim();
+        if (!/^(f\d+)?e\d+$/.test(ref)) {
+            throw new BrowserAutomationError("INVALID_REF", "element ref must come from a fresh browser snapshot");
+        }
+        const locator = page.locator(`aria-ref=${ref}`);
+        try {
+            if (request.type === "click") {
+                await locator.click({ timeout: DEFAULT_ACTION_TIMEOUT_MS });
+            }
+            else if (request.type === "fill") {
+                await locator.fill(request.value, { timeout: DEFAULT_ACTION_TIMEOUT_MS });
+            }
+            else if (request.type === "press") {
+                await locator.press(request.key, { timeout: DEFAULT_ACTION_TIMEOUT_MS });
+            }
+            else {
+                await locator.hover({ timeout: DEFAULT_ACTION_TIMEOUT_MS });
+            }
+        }
+        catch (error) {
+            if (error instanceof Error && /ref .* not found/i.test(error.message)) {
+                throw new BrowserAutomationError("STALE_REF", "element ref is stale; take a fresh browser snapshot", { cause: error });
+            }
+            throw new BrowserAutomationError("ACTION_FAILED", errorMessage(error), { cause: error });
+        }
+        await page.waitForLoadState("domcontentloaded", { timeout: 3_000 }).catch(() => undefined);
+        return { action: request.type, ref, page: await this.snapshot({ screenshot: false }) };
+    }
+    async scroll(input) {
+        const page = await this.page();
+        const amount = clamp(input.amount ?? 600, 50, 3000);
+        await page.mouse.wheel(0, input.direction === "down" ? amount : -amount);
+        await page.waitForTimeout(150);
+        return this.snapshot({ screenshot: false });
+    }
+    async pressKey(key) {
+        const page = await this.page();
+        await page.keyboard.press(key);
+        await page.waitForLoadState("domcontentloaded", { timeout: 3_000 }).catch(() => undefined);
+        return this.snapshot({ screenshot: false });
+    }
+    async selectTab(tabId) {
+        const page = this.tabs.get(tabId);
+        if (!page)
+            throw new BrowserAutomationError("TAB_NOT_FOUND", `tab ${tabId} does not exist`);
+        this.activePage = page;
+        await page.bringToFront();
+        return this.snapshot({ screenshot: false });
+    }
+    async newTab(rawUrl) {
+        const context = await this.context();
+        const page = await context.newPage();
+        this.trackPage(page);
+        this.activePage = page;
+        if (rawUrl) {
+            const result = await this.navigate(rawUrl);
+            if (!result.ok)
+                throw new BrowserAutomationError(result.code, result.error);
+            return result.page;
+        }
+        return this.snapshot({ screenshot: false });
+    }
+    async closeTab(tabId) {
+        const page = tabId === undefined ? await this.page() : this.tabs.get(tabId);
+        if (!page)
+            throw new BrowserAutomationError("TAB_NOT_FOUND", `tab ${tabId} does not exist`);
+        this.tabs.delete(tabIdForValue(this.tabs, page) ?? 0);
+        await page.close();
+        if (this.activePage === page)
+            this.activePage = undefined;
+        return this.snapshot({ screenshot: false });
+    }
+    async handleDialog(action, promptText) {
+        const page = await this.page();
+        const index = this.dialogQueue.findIndex((entry) => entry.page === page);
+        const entry = index >= 0 ? this.dialogQueue[index] : undefined;
+        if (!entry)
+            throw new BrowserAutomationError("NO_DIALOG", "the active page has no pending dialog");
+        this.dialogQueue.splice(index, 1);
+        if (action === "accept")
+            await entry.dialog.accept(promptText);
+        else
+            await entry.dialog.dismiss();
+        return this.snapshot({ screenshot: false });
+    }
+    async dispatchPointer(input) {
+        const page = await this.page();
+        if (input.type === "mousemove")
+            await page.mouse.move(input.x, input.y);
+        else if (input.type === "wheel")
+            await page.mouse.wheel(input.deltaX, input.deltaY);
+        else if (input.type === "mousedown")
+            await page.mouse.down({ button: input.button ?? "left" });
+        else if (input.type === "mouseup")
+            await page.mouse.up({ button: input.button ?? "left" });
+        return { ok: true };
+    }
+    async dispatchKeyboard(input) {
+        const page = await this.page();
+        if (input.type === "keydown")
+            await page.keyboard.down(input.key);
+        else
+            await page.keyboard.up(input.key);
+        return { ok: true };
+    }
+    async inputText(text) {
+        const page = await this.page();
+        await page.keyboard.type(text);
+        return { ok: true };
+    }
+    async screenshot() {
+        return this.snapshot({ screenshot: true });
+    }
+    async focus() {
+        const page = await this.page();
+        await page.bringToFront();
+        return { ok: true };
+    }
+    async waitFor(ms) {
+        const page = await this.page();
+        await page.waitForTimeout(clamp(ms, 0, 10_000));
+        return this.snapshot({ screenshot: false });
+    }
+    async dispose() {
+        if (this.disposed)
+            return;
+        this.disposed = true;
+        const pending = this.browserPromise;
+        this.browserPromise = undefined;
+        this.tabs.clear();
+        this.activePage = undefined;
+        if (!pending)
+            return;
+        const context = await pending.catch(() => undefined);
+        await context?.close().catch(() => undefined);
+    }
+    async page() {
+        const context = await this.context();
+        if (this.activePage && !this.activePage.isClosed())
+            return this.activePage;
+        const existing = context.pages().find((page) => this.tabs.has(tabIdForValue(this.tabs, page) ?? -1));
+        if (existing) {
+            this.activePage = existing;
+            return existing;
+        }
+        const page = await context.newPage();
+        this.trackPage(page);
+        this.activePage = page;
+        return page;
+    }
+    async context() {
+        if (this.disposed)
+            throw new BrowserAutomationError("BROWSER_CLOSED", "browser service is disposed");
+        this.browserPromise ??= (async () => {
+            await Promise.all([
+                mkdir(this.options.profileDir, { recursive: true }),
+                mkdir(this.options.downloadsDir, { recursive: true }),
+                mkdir(this.options.screenshotsDir, { recursive: true }),
+            ]);
+            const browser = await this.launchBrowser();
+            this.wireContext(browser);
+            const page = browser.pages()[0] ?? await browser.newPage();
+            this.trackPage(page);
+            this.activePage = page;
+            return browser;
+        })();
+        try {
+            return await this.browserPromise;
+        }
+        catch (error) {
+            this.browserPromise = undefined;
+            throw new BrowserAutomationError("BROWSER_START_FAILED", errorMessage(error), {
+                cause: error,
+            });
+        }
+    }
+    wireContext(context) {
+        context.on("page", (page) => {
+            this.trackPage(page);
+            if (!this.activePage || this.activePage.isClosed())
+                this.activePage = page;
+        });
+        context.on("close", () => {
+            if (this.browserPromise) {
+                void this.browserPromise.then((current) => {
+                    if (current === context) {
+                        this.browserPromise = undefined;
+                        this.activePage = undefined;
+                        this.tabs.clear();
+                    }
+                });
+            }
+        });
+        context.on("dialog", (dialog) => {
+            const page = dialog.page();
+            if (page)
+                this.dialogQueue.push({ page, dialog });
+        });
+        context.on("download", (download) => {
+            this.downloadQueue.push({ download, consumed: false });
+        });
+    }
+    async launchBrowser() {
+        const candidates = chromiumLaunchCandidates();
+        const launchOptions = chromiumLaunchOptions(this.options.headless);
+        let lastError;
+        for (const candidate of candidates) {
+            try {
+                return await chromium.launchPersistentContext(this.options.profileDir, { ...launchOptions, ...candidate });
+            }
+            catch (error) {
+                lastError = error;
+            }
+        }
+        if (candidates.length === 0) {
+            return await chromium.launchPersistentContext(this.options.profileDir, launchOptions);
+        }
+        throw lastError instanceof Error
+            ? lastError
+            : new Error("no Chromium executable was available");
+    }
+    trackPage(page) {
+        for (const [id, existing] of this.tabs) {
+            if (existing === page)
+                return id;
+        }
+        const id = this.nextTabId;
+        this.nextTabId += 1;
+        this.tabs.set(id, page);
+        page.on("close", () => {
+            this.tabs.delete(id);
+            if (this.activePage === page)
+                this.activePage = undefined;
+        });
+        return id;
+    }
+    dialogFor(page) {
+        const index = this.dialogQueue.findIndex((entry) => entry.page === page);
+        const dialog = index >= 0 ? this.dialogQueue[index]?.dialog : undefined;
+        if (!dialog)
+            return null;
+        return {
+            type: dialog.type(),
+            message: dialog.message(),
+            ...(dialog.defaultValue() ? { defaultText: dialog.defaultValue() } : {}),
+        };
+    }
+    async tabSummaries() {
+        return Promise.all([...this.tabs.entries()].map(async ([id, page]) => ({
+            id,
+            url: page.url(),
+            title: page.isClosed() ? "" : await page.title().catch(() => ""),
+            active: !page.isClosed() && page === this.activePage,
+        })));
+    }
+    async captureScreenshot(page) {
+        const buffer = await page.screenshot({
+            type: "png",
+            fullPage: false,
+            timeout: 15_000,
+        });
+        if (buffer.byteLength > MAX_SCREENSHOT_BYTES)
+            return undefined;
+        return `data:image/png;base64,${buffer.toString("base64")}`;
+    }
+    async nextDownloadResult() {
+        const index = this.downloadQueue.findIndex((entry) => !entry.consumed);
+        if (index < 0)
+            return undefined;
+        const entry = this.downloadQueue[index];
+        if (!entry)
+            return undefined;
+        entry.consumed = true;
+        const download = entry.download;
+        const target = path.join(this.options.downloadsDir, `${randomUUID()}-${sanitizeFilename(download.suggestedFilename())}`);
+        try {
+            await download.saveAs(target);
+            this.downloadQueue.splice(index, 1);
+            return {
+                url: download.url(),
+                suggestedFilename: download.suggestedFilename(),
+                path: target,
+            };
+        }
+        catch (error) {
+            this.downloadQueue.splice(index, 1);
+            throw error;
+        }
+    }
+}
+function emptyBrowserState() {
+    return {
+        running: false,
+        url: "about:blank",
+        title: "",
+        tabs: [],
+        dialog: null,
+        ariaSnapshot: "",
+    };
+}
+function chromiumLaunchOptions(headless) {
+    return {
+        headless: headless ?? false,
+        viewport: undefined,
+        acceptDownloads: true,
+        locale: "zh-CN",
+        timezoneId: "Asia/Shanghai",
+        args: ["--start-maximized", "--disable-blink-features=AutomationControlled"],
+        ignoreDefaultArgs: ["--enable-automation"],
+    };
+}
+async function findSearchInput(page) {
+    for (const selector of SEARCH_INPUT_SELECTORS) {
+        const locator = page.locator(selector).first();
+        if (await locator.isVisible().catch(() => false))
+            return locator;
+    }
+    return undefined;
+}
+async function submitSearchForm(page, input) {
+    for (const selector of SEARCH_SUBMIT_SELECTORS) {
+        const locator = page.locator(selector).first();
+        if (!await locator.isVisible().catch(() => false))
+            continue;
+        await locator.click({ timeout: DEFAULT_ACTION_TIMEOUT_MS });
+        return;
+    }
+    await input.press("Enter", { timeout: DEFAULT_ACTION_TIMEOUT_MS });
+}
+export class BrowserAutomationError extends Error {
+    code;
+    constructor(code, message, options) {
+        super(message, options);
+        this.code = code;
+        this.name = "BrowserAutomationError";
+    }
+}
+export function chromiumLaunchCandidates(env = process.env, platform = process.platform) {
+    const executablePath = env.LOOP_BROWSER_CHROMIUM_PATH?.trim();
+    if (executablePath)
+        return [{ executablePath }];
+    if (platform === "win32")
+        return [{ channel: "chrome" }, { channel: "msedge" }];
+    if (platform === "darwin")
+        return [{ channel: "chrome" }];
+    return [];
+}
+function tabIdForValue(map, page) {
+    for (const [id, value] of map)
+        if (value === page)
+            return id;
+    return undefined;
+}
+function truncateText(value, max) {
+    return value.length <= max ? value : `${value.slice(0, max)}\n... [snapshot truncated]`;
+}
+function clamp(value, min, max) {
+    return Number.isFinite(value) ? Math.min(max, Math.max(min, value)) : min;
+}
+function sanitizeFilename(value) {
+    return value.replace(/[\\/:*?"<>|]/g, "_").slice(0, 180) || "download";
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}

package/dist/worker/console/chat/browser-policy.js

@@ -1,15 +1,11 @@
 /**
  * Pure URL policy for the Operator Chat browser panel: normalize user input
- * into an http(s) URL and refuse destinations that would be dangerous to
- * embed in the panel iframe. Dependency-free so it is unit-testable.
+ * into an http(s) URL. Dependency-free so it is unit-testable.
  *
- * The iframe sandbox (opaque origin — no allow-same-origin, no top
- * navigation) is the primary security boundary; this module is the
- * address-bar gate on top of it: only http/https may be navigated, and
- * loopback addresses are refused so a browsed page cannot probe local
- * services by user action. The Console's OWN origin is explicitly allowed —
- * the sandbox still renders it in an opaque origin with no same-origin
- * privileges, exactly like any other site.
+ * Only the scheme is gated: javascript:, data:, file: and friends must never
+ * reach the iframe. Local addresses (localhost / 127.0.0.1 / ::1 / LAN
+ * hosts) are deliberately allowed — the panel is a development console and
+ * its users routinely browse local dev servers.
  */
 /**
  * Extract the `frame-ancestors` source list of a Content-Security-Policy
@@ -47,16 +43,6 @@ export function embeddabilityOf(probe) {
     }
     return "embeddable";
 }
-/** A loopback hostname (localhost, IPv6 ::1, 127.0.0.0/8, 0.0.0.0). */
-export function isLoopbackHostname(hostname) {
-    const host = hostname.replace(/^\[|\]$/g, "").toLowerCase();
-    if (host === "localhost" || host === "::1" || host === "0.0.0.0")
-        return true;
-    const parts = host.split(".");
-    return (parts.length === 4 &&
-        parts[0] === "127" &&
-        parts.every((part) => /^\d{1,3}$/.test(part) && Number(part) <= 255));
-}
 /**
  * Schemes that must never reach the iframe even without `//` (javascript:,
  * data:, file:, ...). Host:port lookalikes (example.com:8080) are NOT here —
@@ -88,12 +74,8 @@ const FORBIDDEN_SCHEMES = new Set([
 /**
  * Normalize one address-bar input against the navigation policy.
  * @param input raw user text.
- * @param selfOrigin the Console's own origin (window.location.origin). The
- * Console itself may be browsed in the panel (the sandbox keeps it opaque),
- * so it is let through BEFORE the loopback check — its host is normally
- * loopback.
  */
-export function normalizeBrowserUrl(input, selfOrigin) {
+export function normalizeBrowserUrl(input) {
     const trimmed = input.trim();
     if (trimmed === "")
         return { kind: "invalid" };
@@ -127,17 +109,5 @@ export function normalizeBrowserUrl(input, selfOrigin) {
     if (url.protocol !== "http:" && url.protocol !== "https:") {
         return { kind: "blocked", reason: "scheme" };
     }
-    // The Console's own origin is allowed (browsed in an opaque origin like
-    // any other site); checked before the loopback gate because its host is
-    // normally loopback.
-    try {
-        if (url.origin === new URL(selfOrigin).origin)
-            return { kind: "ok", url: url.href };
-    }
-    catch {
-        // Unparsable selfOrigin (never in practice): fall through to the gate.
-    }
-    if (isLoopbackHostname(url.hostname))
-        return { kind: "blocked", reason: "loopback" };
     return { kind: "ok", url: url.href };
 }