@apifuse/connector-sdk 2.0.0-beta.1

package/src/runtime/browser.ts

@@ -0,0 +1,724 @@
+import { createRequire } from "node:module";
+import type { LaunchOptions, Page } from "playwright";
+
+import { ConnectorError } from "../errors";
+import type {
+	BrowserClient as BrowserClientContract,
+	BrowserEngine,
+	BrowserOptions,
+} from "../types";
+
+const require = createRequire(import.meta.url);
+const DEFAULT_WAIT_TIMEOUT_MS = 30_000;
+const SELECTOR_POLL_INTERVAL_MS = 100;
+
+type PlaywrightModule = typeof import("playwright");
+type PlaywrightStealthModule = {
+	stealth(page: unknown): Promise<void>;
+};
+
+type PoolAcquireResponse = {
+	pageId: string;
+	wsEndpoint: string;
+};
+
+type JsonRpcId = number;
+
+type JsonRpcError = {
+	code?: number;
+	message?: string;
+};
+
+type JsonRpcMessage = {
+	error?: JsonRpcError;
+	id?: JsonRpcId;
+	method?: string;
+	params?: Record<string, unknown>;
+	result?: Record<string, unknown>;
+};
+
+type BrowserPageContract = {
+	close(): Promise<void>;
+	content(): Promise<string>;
+	evaluate<T>(fn: string | (() => T)): Promise<T>;
+	fill?(selector: string, text: string): Promise<void>;
+	goto(url: string): Promise<void>;
+	pageId?: string;
+	screenshot(options?: { fullPage?: boolean }): Promise<Buffer>;
+	click(selector: string): Promise<void>;
+	type(selector: string, text: string): Promise<void>;
+	waitForSelector(
+		selector: string,
+		options?: { timeout?: number },
+	): Promise<void>;
+};
+
+export type BrowserClientOptions = BrowserOptions & {
+	cdpUrl?: string;
+	executablePath?: string;
+	extraArgs?: string[];
+};
+
+type SupportedBrowserClient = BrowserClientContract & {
+	close(): Promise<void>;
+	newPage(): Promise<BrowserPageContract>;
+};
+
+function getDefaultCdpPoolUrl(env = process.env): string | undefined {
+	return env.CDP_POOL_URL ?? env.APIFUSE_CDP_POOL_URL;
+}
+
+async function importOptionalModule<T extends object>(
+	moduleName: string,
+): Promise<T> {
+	return (await import(moduleName)) as T;
+}
+
+function unwrapModuleDefault<T extends object>(module: T): T {
+	if ("default" in module) {
+		const defaultExport = module.default as unknown;
+		if (
+			(typeof defaultExport === "object" && defaultExport !== null) ||
+			typeof defaultExport === "function"
+		) {
+			return defaultExport as T;
+		}
+	}
+
+	return module;
+}
+
+function isModuleNotFoundError(error: unknown): boolean {
+	return (
+		error instanceof Error &&
+		("code" in error
+			? (error as Error & { code?: string }).code === "MODULE_NOT_FOUND"
+			: error.message.includes("Cannot find module"))
+	);
+}
+
+function delay(ms: number): Promise<void> {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function formatExpression<T>(fn: string | (() => T)): string {
+	if (typeof fn === "string") {
+		return fn;
+	}
+
+	return `(${fn.toString()})()`;
+}
+
+function toLaunchOptions(options: BrowserClientOptions): LaunchOptions {
+	return {
+		args: options.extraArgs,
+		executablePath: options.executablePath,
+		headless: options.headless ?? true,
+		proxy: options.proxy ? { server: options.proxy } : undefined,
+	};
+}
+
+async function loadPlaywright(): Promise<PlaywrightModule> {
+	try {
+		require("playwright");
+	} catch (error) {
+		if (isModuleNotFoundError(error)) {
+			throw new ConnectorError("Playwright is not installed", {
+				cause: error instanceof Error ? error : undefined,
+				fix: "Run: bun add playwright",
+			});
+		}
+
+		throw error;
+	}
+
+	try {
+		return unwrapModuleDefault(
+			await importOptionalModule<PlaywrightModule>("playwright"),
+		);
+	} catch (error) {
+		if (isModuleNotFoundError(error)) {
+			throw new ConnectorError("Playwright is not installed", {
+				cause: error instanceof Error ? error : undefined,
+				fix: "Run: bun add playwright",
+			});
+		}
+
+		throw error;
+	}
+}
+
+async function loadPlaywrightStealth(): Promise<PlaywrightStealthModule> {
+	try {
+		return unwrapModuleDefault(
+			await importOptionalModule<PlaywrightStealthModule>("playwright-stealth"),
+		);
+	} catch (error) {
+		if (isModuleNotFoundError(error)) {
+			throw new ConnectorError("playwright-stealth is not installed", {
+				cause: error instanceof Error ? error : undefined,
+				fix: "Run: bun add playwright-stealth",
+			});
+		}
+
+		throw error;
+	}
+}
+
+async function loadNodriver(): Promise<void> {
+	try {
+		await importOptionalModule("nodriver");
+	} catch (error) {
+		if (isModuleNotFoundError(error)) {
+			throw new ConnectorError("nodriver is not installed", {
+				cause: error instanceof Error ? error : undefined,
+				fix: "Run: pip install nodriver (Python) or bun add nodriver",
+			});
+		}
+
+		throw error;
+	}
+}
+
+async function loadSeleniumBase(): Promise<void> {
+	try {
+		await importOptionalModule("seleniumbase");
+	} catch (error) {
+		if (isModuleNotFoundError(error)) {
+			throw new ConnectorError("seleniumbase is not installed", {
+				cause: error instanceof Error ? error : undefined,
+				fix: "Run: pip install seleniumbase",
+			});
+		}
+
+		throw error;
+	}
+}
+
+class PlaywrightBrowserPage implements BrowserPageContract {
+	readonly pageId?: string;
+
+	constructor(private readonly page: Page) {}
+
+	async goto(url: string): Promise<void> {
+		await this.page.goto(url);
+	}
+
+	async evaluate<T>(fn: string | (() => T)): Promise<T> {
+		if (typeof fn === "string") {
+			return await this.page.evaluate(fn);
+		}
+
+		return await this.page.evaluate(fn);
+	}
+
+	async waitForSelector(
+		selector: string,
+		options?: { timeout?: number },
+	): Promise<void> {
+		await this.page.waitForSelector(selector, options);
+	}
+
+	async click(selector: string): Promise<void> {
+		await this.page.click(selector);
+	}
+
+	async type(selector: string, text: string): Promise<void> {
+		await this.page.locator(selector).fill("");
+		await this.page.type(selector, text);
+	}
+
+	async fill(selector: string, text: string): Promise<void> {
+		await this.page.fill(selector, text);
+	}
+
+	async content(): Promise<string> {
+		return await this.page.content();
+	}
+
+	async screenshot(options?: { fullPage?: boolean }): Promise<Buffer> {
+		return await this.page.screenshot(options);
+	}
+
+	async close(): Promise<void> {
+		await this.page.close();
+	}
+}
+
+class PlaywrightBrowserClient implements SupportedBrowserClient {
+	private browser: import("playwright").Browser | null = null;
+	readonly engine = "playwright-stealth" satisfies BrowserEngine;
+
+	constructor(private readonly options: BrowserClientOptions = {}) {}
+
+	private async ensureBrowser(): Promise<import("playwright").Browser> {
+		if (this.browser?.isConnected()) {
+			return this.browser;
+		}
+
+		const playwright = await loadPlaywright();
+		this.browser = await playwright.chromium.launch(
+			toLaunchOptions(this.options),
+		);
+		return this.browser;
+	}
+
+	async newPage(): Promise<BrowserPageContract> {
+		const browser = await this.ensureBrowser();
+		const page = await browser.newPage();
+
+		if (this.options.stealth ?? true) {
+			const { stealth } = await loadPlaywrightStealth();
+			await stealth(page);
+		}
+
+		return new PlaywrightBrowserPage(page);
+	}
+
+	async close(): Promise<void> {
+		const browser = this.browser;
+		this.browser = null;
+
+		if (!browser) {
+			return;
+		}
+
+		await browser.close();
+	}
+}
+
+class JsonRpcWebSocketClient {
+	private nextId = 1;
+	private readonly listeners = new Map<
+		string,
+		Set<(params: unknown) => void>
+	>();
+	private readonly pending = new Map<
+		JsonRpcId,
+		{
+			reject: (reason?: unknown) => void;
+			resolve: (value: Record<string, unknown>) => void;
+		}
+	>();
+	private socket?: WebSocket;
+	private socketPromise?: Promise<WebSocket>;
+
+	constructor(private readonly endpoint: string) {}
+
+	on(method: string, listener: (params: unknown) => void): () => void {
+		const listeners = this.listeners.get(method) ?? new Set();
+		listeners.add(listener);
+		this.listeners.set(method, listeners);
+
+		return () => {
+			listeners.delete(listener);
+			if (listeners.size === 0) {
+				this.listeners.delete(method);
+			}
+		};
+	}
+
+	async send(
+		method: string,
+		params: Record<string, unknown> = {},
+	): Promise<Record<string, unknown>> {
+		const socket = await this.getSocket();
+		const id = this.nextId++;
+
+		return await new Promise<Record<string, unknown>>((resolve, reject) => {
+			this.pending.set(id, { resolve, reject });
+
+			socket.send(
+				JSON.stringify({
+					id,
+					jsonrpc: "2.0",
+					method,
+					params,
+				}),
+			);
+		});
+	}
+
+	async close(): Promise<void> {
+		for (const pending of this.pending.values()) {
+			pending.reject(new Error(`WebSocket closed: ${this.endpoint}`));
+		}
+
+		this.pending.clear();
+		this.listeners.clear();
+		this.socket?.close();
+		this.socket = undefined;
+		this.socketPromise = undefined;
+	}
+
+	private async getSocket(): Promise<WebSocket> {
+		if (this.socket?.readyState === WebSocket.OPEN) {
+			return this.socket;
+		}
+
+		if (this.socketPromise) {
+			return this.socketPromise;
+		}
+
+		this.socketPromise = new Promise<WebSocket>((resolve, reject) => {
+			const socket = new WebSocket(this.endpoint);
+
+			socket.addEventListener("open", () => {
+				this.socket = socket;
+				resolve(socket);
+			});
+
+			socket.addEventListener("message", (event) => {
+				const rawData =
+					typeof event.data === "string"
+						? event.data
+						: Buffer.from(event.data as ArrayBufferLike).toString("utf8");
+				const payload = JSON.parse(rawData) as JsonRpcMessage;
+
+				if (typeof payload.id === "number") {
+					const pending = this.pending.get(payload.id);
+					if (!pending) {
+						return;
+					}
+
+					this.pending.delete(payload.id);
+
+					if (payload.error) {
+						pending.reject(
+							new Error(payload.error.message ?? "JSON-RPC command failed"),
+						);
+						return;
+					}
+
+					pending.resolve(payload.result ?? {});
+					return;
+				}
+
+				if (!payload.method) {
+					return;
+				}
+
+				for (const listener of this.listeners.get(payload.method) ?? []) {
+					listener(payload.params);
+				}
+			});
+
+			socket.addEventListener("close", () => {
+				for (const pending of this.pending.values()) {
+					pending.reject(new Error(`WebSocket closed: ${this.endpoint}`));
+				}
+
+				this.pending.clear();
+				this.socket = undefined;
+				this.socketPromise = undefined;
+			});
+
+			socket.addEventListener("error", () => {
+				reject(
+					new Error(
+						`Unable to connect to WebSocket endpoint: ${this.endpoint}`,
+					),
+				);
+			});
+		});
+
+		return this.socketPromise;
+	}
+}
+
+class CdpPoolBrowserPage implements BrowserPageContract {
+	private closed = false;
+	private initialized = false;
+
+	constructor(
+		readonly pageId: string,
+		private readonly pageClient: JsonRpcWebSocketClient,
+		private readonly release: (pageId: string) => Promise<void>,
+	) {}
+
+	async goto(url: string): Promise<void> {
+		await this.initialize();
+		const startedAt = Date.now();
+		let loadEventSeen = false;
+		const unsubscribe = this.pageClient.on("Page.loadEventFired", () => {
+			loadEventSeen = true;
+		});
+
+		try {
+			await this.pageClient.send("Page.navigate", { url });
+			await this.waitForDocumentReady(
+				startedAt + DEFAULT_WAIT_TIMEOUT_MS,
+				() => loadEventSeen,
+			);
+		} finally {
+			unsubscribe();
+		}
+	}
+
+	async evaluate<T>(fn: string | (() => T)): Promise<T> {
+		await this.initialize();
+		const result = await this.pageClient.send("Runtime.evaluate", {
+			awaitPromise: true,
+			expression: formatExpression(fn),
+			returnByValue: true,
+		});
+
+		if (result.exceptionDetails) {
+			throw new Error(
+				String(
+					(result.exceptionDetails as { text?: string }).text ??
+						"Browser evaluation failed",
+				),
+			);
+		}
+
+		return (result.result as { value?: T } | undefined)?.value as T;
+	}
+
+	async waitForSelector(
+		selector: string,
+		options?: { timeout?: number },
+	): Promise<void> {
+		const timeout = options?.timeout ?? DEFAULT_WAIT_TIMEOUT_MS;
+		const deadline = Date.now() + timeout;
+
+		while (Date.now() < deadline) {
+			const exists = await this.evaluate<boolean>(
+				`Boolean(document.querySelector(${JSON.stringify(selector)}))`,
+			);
+
+			if (exists) {
+				return;
+			}
+
+			await delay(SELECTOR_POLL_INTERVAL_MS);
+		}
+
+		throw new Error(`Timed out waiting for selector: ${selector}`);
+	}
+
+	async click(selector: string): Promise<void> {
+		await this.waitForSelector(selector);
+		await this.evaluate(
+			`(() => {
+				const element = document.querySelector(${JSON.stringify(selector)});
+				if (!(element instanceof HTMLElement)) {
+					throw new Error(${JSON.stringify(`Selector not found: ${selector}`)});
+				}
+				element.click();
+			})()`,
+		);
+	}
+
+	async type(selector: string, text: string): Promise<void> {
+		await this.fill(selector, text);
+	}
+
+	async fill(selector: string, text: string): Promise<void> {
+		await this.waitForSelector(selector);
+		await this.evaluate(
+			`(() => {
+				const element = document.querySelector(${JSON.stringify(selector)});
+				if (!(element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement)) {
+					throw new Error(${JSON.stringify(`Unsupported input target: ${selector}`)});
+				}
+				element.focus();
+				element.value = "";
+			})()`,
+		);
+		await this.pageClient.send("Input.insertText", { text });
+		await this.evaluate(
+			`(() => {
+				const element = document.querySelector(${JSON.stringify(selector)});
+				if (!(element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement)) {
+					throw new Error(${JSON.stringify(`Unsupported input target: ${selector}`)});
+				}
+				element.dispatchEvent(new Event("input", { bubbles: true }));
+				element.dispatchEvent(new Event("change", { bubbles: true }));
+			})()`,
+		);
+	}
+
+	async content(): Promise<string> {
+		return await this.evaluate<string>("document.documentElement.outerHTML");
+	}
+
+	async screenshot(options?: { fullPage?: boolean }): Promise<Buffer> {
+		await this.initialize();
+		const result = await this.pageClient.send("Page.captureScreenshot", {
+			captureBeyondViewport: options?.fullPage ?? false,
+			format: "png",
+			fromSurface: true,
+		});
+
+		return Buffer.from(String(result.data ?? ""), "base64");
+	}
+
+	async close(): Promise<void> {
+		if (this.closed) {
+			return;
+		}
+
+		this.closed = true;
+
+		try {
+			await this.release(this.pageId);
+		} finally {
+			await this.pageClient.close();
+		}
+	}
+
+	private async initialize(): Promise<void> {
+		if (this.initialized) {
+			return;
+		}
+
+		await this.pageClient.send("Page.enable");
+		await this.pageClient.send("Runtime.enable");
+		this.initialized = true;
+	}
+
+	private async waitForDocumentReady(
+		deadline: number,
+		isLoadEventSeen: () => boolean,
+	): Promise<void> {
+		while (Date.now() < deadline) {
+			const readyState = await this.evaluate<string>("document.readyState");
+			if (readyState === "complete" || readyState === "interactive") {
+				if (isLoadEventSeen() || readyState === "complete") {
+					return;
+				}
+			}
+
+			await delay(SELECTOR_POLL_INTERVAL_MS);
+		}
+
+		throw new Error("Timed out waiting for page load");
+	}
+}
+
+class CdpPoolBrowserClient implements SupportedBrowserClient {
+	private readonly poolClient: JsonRpcWebSocketClient;
+	readonly engine = "playwright-stealth" satisfies BrowserEngine;
+
+	constructor(options: BrowserClientOptions) {
+		if (!options.cdpUrl) {
+			throw new ConnectorError("CDP Pool URL is required", {
+				code: "BROWSER_RUNTIME_UNSUPPORTED",
+			});
+		}
+
+		this.poolClient = new JsonRpcWebSocketClient(options.cdpUrl);
+	}
+
+	async newPage(): Promise<BrowserPageContract> {
+		const acquireResult = (await this.poolClient.send(
+			"acquire",
+		)) as PoolAcquireResponse;
+		const pageClient = new JsonRpcWebSocketClient(acquireResult.wsEndpoint);
+		const page = new CdpPoolBrowserPage(
+			acquireResult.pageId,
+			pageClient,
+			async (pageId) => {
+				await this.poolClient.send("release", { pageId });
+			},
+		);
+
+		await page.evaluate(
+			`window.navigator.webdriver === true ? Object.defineProperty(window.navigator, "webdriver", { configurable: true, get: () => undefined }) : undefined`,
+		);
+
+		return page;
+	}
+
+	async close(): Promise<void> {
+		await this.poolClient.close();
+	}
+}
+
+class UnsupportedBrowserEngineClient implements SupportedBrowserClient {
+	constructor(
+		readonly engine: Extract<BrowserEngine, "nodriver" | "selenium-uc">,
+	) {}
+
+	async newPage(): Promise<never> {
+		if (this.engine === "nodriver") {
+			await loadNodriver();
+			throw new ConnectorError("nodriver engine requires Python runtime", {
+				fix: "Use connector language: python and ctx.browser in Python",
+			});
+		}
+
+		await loadSeleniumBase();
+		throw new ConnectorError("selenium-uc engine requires Python runtime", {
+			fix: "Use connector language: python",
+		});
+	}
+
+	async close(): Promise<void> {}
+}
+
+function createPlaywrightStealthClient(
+	options: BrowserClientOptions = {},
+): SupportedBrowserClient {
+	return new PlaywrightBrowserClient(options);
+}
+
+function createCdpPoolBrowserClient(
+	options: BrowserClientOptions,
+): SupportedBrowserClient {
+	return new CdpPoolBrowserClient(options);
+}
+
+function createNodriverClient(): SupportedBrowserClient {
+	return new UnsupportedBrowserEngineClient("nodriver");
+}
+
+function createSeleniumUCClient(): SupportedBrowserClient {
+	return new UnsupportedBrowserEngineClient("selenium-uc");
+}
+
+export class BrowserClient implements BrowserClientContract {
+	private readonly client: SupportedBrowserClient;
+	private readonly _engine: BrowserEngine;
+
+	constructor(options: BrowserClientOptions = {}) {
+		const resolvedOptions = {
+			...options,
+			cdpUrl: options.cdpUrl ?? getDefaultCdpPoolUrl(),
+		};
+		const engine = resolvedOptions.engine ?? "playwright-stealth";
+		this._engine = engine;
+
+		switch (engine) {
+			case "nodriver":
+				this.client = createNodriverClient();
+				break;
+			case "selenium-uc":
+				this.client = createSeleniumUCClient();
+				break;
+			default:
+				this.client = resolvedOptions.cdpUrl
+					? createCdpPoolBrowserClient(resolvedOptions)
+					: createPlaywrightStealthClient(resolvedOptions);
+		}
+	}
+
+	get engine(): BrowserEngine {
+		return this._engine;
+	}
+
+	async newPage(): Promise<unknown> {
+		return await this.client.newPage();
+	}
+
+	async close(): Promise<void> {
+		await this.client.close();
+	}
+}
+
+export function createBrowserClient(
+	options: BrowserClientOptions = {},
+): BrowserClient {
+	return new BrowserClient(options);
+}