npm - @apifuse/connector-sdk - Versions diffs - 2.0.0-beta.1 - Mend

@apifuse/connector-sdk 2.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (67) hide show

package/README.md +44 -0
package/bin/apifuse-check.ts +408 -0
package/bin/apifuse-dev.ts +222 -0
package/bin/apifuse-init.ts +390 -0
package/bin/apifuse-perf.ts +1101 -0
package/bin/apifuse-record.ts +446 -0
package/bin/apifuse-test.ts +688 -0
package/bin/apifuse.ts +51 -0
package/package.json +64 -0
package/src/__tests__/auth.test.ts +396 -0
package/src/__tests__/browser-auth.test.ts +180 -0
package/src/__tests__/browser.test.ts +632 -0
package/src/__tests__/connectors-yaml.test.ts +135 -0
package/src/__tests__/define.test.ts +225 -0
package/src/__tests__/errors.test.ts +69 -0
package/src/__tests__/executor.test.ts +214 -0
package/src/__tests__/http.test.ts +238 -0
package/src/__tests__/insights.test.ts +210 -0
package/src/__tests__/instrumentation.test.ts +290 -0
package/src/__tests__/otlp.test.ts +141 -0
package/src/__tests__/perf.test.ts +60 -0
package/src/__tests__/proxy.test.ts +359 -0
package/src/__tests__/recipes.test.ts +36 -0
package/src/__tests__/serve.test.ts +233 -0
package/src/__tests__/session.test.ts +231 -0
package/src/__tests__/state.test.ts +100 -0
package/src/__tests__/stealth.test.ts +57 -0
package/src/__tests__/testing.test.ts +97 -0
package/src/__tests__/tls.test.ts +345 -0
package/src/__tests__/types.test.ts +142 -0
package/src/__tests__/utils.test.ts +62 -0
package/src/__tests__/waterfall.test.ts +270 -0
package/src/config/connectors-yaml.ts +373 -0
package/src/config/loader.ts +122 -0
package/src/define.ts +137 -0
package/src/dev.ts +38 -0
package/src/errors.ts +68 -0
package/src/index.test.ts +1 -0
package/src/index.ts +100 -0
package/src/protocol.ts +183 -0
package/src/recipes/gov-api.ts +97 -0
package/src/recipes/rest-api.ts +152 -0
package/src/runtime/auth.ts +245 -0
package/src/runtime/browser.ts +724 -0
package/src/runtime/connector.ts +20 -0
package/src/runtime/executor.ts +51 -0
package/src/runtime/http.ts +248 -0
package/src/runtime/insights.ts +456 -0
package/src/runtime/instrumentation.ts +424 -0
package/src/runtime/otlp.ts +171 -0
package/src/runtime/perf.ts +73 -0
package/src/runtime/session.ts +573 -0
package/src/runtime/state.ts +124 -0
package/src/runtime/tls.ts +410 -0
package/src/runtime/trace.ts +261 -0
package/src/runtime/waterfall.ts +245 -0
package/src/serve.ts +665 -0
package/src/stealth/profiles.ts +391 -0
package/src/testing/helpers.ts +144 -0
package/src/testing/index.ts +2 -0
package/src/testing/run.ts +88 -0
package/src/types/playwright-stealth.d.ts +9 -0
package/src/types.ts +243 -0
package/src/utils/date.ts +163 -0
package/src/utils/parse.ts +66 -0
package/src/utils/text.ts +20 -0
package/src/utils/transform.ts +62 -0

package/src/__tests__/browser.test.ts ADDED Viewed

@@ -0,0 +1,632 @@
+import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
+type LaunchCall = {
+	args?: string[];
+	executablePath?: string;
+	headless?: boolean;
+	proxy?: { server: string };
+};
+type MockPlaywrightPage = {
+	click: (selector: string) => Promise<void>;
+	close: () => Promise<void>;
+	content: () => Promise<string>;
+	evaluate: <T>(fn: string | (() => T)) => Promise<T>;
+	fill: (selector: string, text: string) => Promise<void>;
+	goto: (url: string) => Promise<void>;
+	locator: (selector: string) => { fill(value: string): Promise<void> };
+	screenshot: (options?: { fullPage?: boolean }) => Promise<Buffer>;
+	type: (selector: string, text: string) => Promise<void>;
+	waitForSelector: (
+		selector: string,
+		options?: { timeout?: number },
+	) => Promise<void>;
+	state: {
+		clicks: string[];
+		closed: boolean;
+		content: string;
+		fills: Array<{ selector: string; text: string }>;
+		gotoUrls: string[];
+		screenshots: Array<{ fullPage?: boolean }>;
+		types: Array<{ selector: string; text: string }>;
+		waits: Array<{ selector: string; timeout?: number }>;
+	};
+};
+type MockBrowserState = {
+	browser: {
+		close: () => Promise<void>;
+		isConnected: () => boolean;
+		newPage: () => Promise<MockPlaywrightPage>;
+	};
+	closeCalls: number;
+	connected: boolean;
+	newPageCalls: number;
+	pages: MockPlaywrightPage[];
+	launchOptions?: LaunchCall;
+};
+const browserState = {
+	browsers: [] as MockBrowserState[],
+	launchCalls: [] as LaunchCall[],
+	requireError: null as Error | null,
+};
+const stealthState = {
+	callCount: 0,
+	pages: [] as MockPlaywrightPage[],
+};
+const optionalModuleState = {
+	nodriverError: null as Error | null,
+	nodriverImports: 0,
+	seleniumBaseError: null as Error | null,
+	seleniumBaseImports: 0,
+};
+const cdpState = {
+	acquireCalls: 0,
+	clicks: [] as string[],
+	closedEndpoints: [] as string[],
+	focusedSelectors: [] as string[],
+	insertedTexts: [] as string[],
+	navigateUrls: [] as string[],
+	poolReleaseCalls: [] as string[],
+	runtimeEnabled: 0,
+	pageEnabled: 0,
+	screenshotCalls: [] as boolean[],
+	webdriverPatches: 0,
+};
+const originalWebSocket = globalThis.WebSocket;
+const originalCdpPoolUrl = process.env.CDP_POOL_URL;
+function createMockPlaywrightPage(): MockPlaywrightPage {
+	const state = {
+		clicks: [] as string[],
+		closed: false,
+		content: "<html><body>local</body></html>",
+		fills: [] as Array<{ selector: string; text: string }>,
+		gotoUrls: [] as string[],
+		screenshots: [] as Array<{ fullPage?: boolean }>,
+		types: [] as Array<{ selector: string; text: string }>,
+		waits: [] as Array<{ selector: string; timeout?: number }>,
+	};
+	return {
+		state,
+		async click(selector) {
+			state.clicks.push(selector);
+		},
+		async close() {
+			state.closed = true;
+		},
+		async content() {
+			return state.content;
+		},
+		async evaluate<T>(fn: string | (() => T)) {
+			if (typeof fn === "function") {
+				return fn();
+			}
+			if (fn === "document.title") {
+				return "local-title" as T;
+			}
+			throw new Error(`Unexpected local evaluate expression: ${fn}`);
+		},
+		async fill(selector, text) {
+			state.fills.push({ selector, text });
+		},
+		async goto(url) {
+			state.gotoUrls.push(url);
+		},
+		locator(selector) {
+			return {
+				async fill(value) {
+					state.fills.push({ selector, text: value });
+				},
+			};
+		},
+		async screenshot(options) {
+			state.screenshots.push(options ?? {});
+			return Buffer.from("local-shot");
+		},
+		async type(selector, text) {
+			state.types.push({ selector, text });
+		},
+		async waitForSelector(selector, options) {
+			state.waits.push({ selector, timeout: options?.timeout });
+		},
+	};
+}
+function registerBrowserMocks() {
+	mock.module("playwright", () => {
+		if (browserState.requireError) {
+			throw browserState.requireError;
+		}
+		return {
+			chromium: {
+				launch: async (options: LaunchCall = {}) => {
+					browserState.launchCalls.push(options);
+					const state: MockBrowserState = {
+						connected: true,
+						closeCalls: 0,
+						newPageCalls: 0,
+						pages: [],
+						launchOptions: options,
+						browser: {
+							close: async () => {
+								state.closeCalls += 1;
+								state.connected = false;
+							},
+							isConnected: () => state.connected,
+							newPage: async () => {
+								state.newPageCalls += 1;
+								const page = createMockPlaywrightPage();
+								state.pages.push(page);
+								return page;
+							},
+						},
+					};
+					browserState.browsers.push(state);
+					return state.browser;
+				},
+			},
+		};
+	});
+	mock.module("playwright-stealth", () => ({
+		stealth: async (page: MockPlaywrightPage) => {
+			stealthState.callCount += 1;
+			stealthState.pages.push(page);
+		},
+	}));
+	mock.module("nodriver", () => {
+		optionalModuleState.nodriverImports += 1;
+		if (optionalModuleState.nodriverError) {
+			throw optionalModuleState.nodriverError;
+		}
+		return {};
+	});
+	mock.module("seleniumbase", () => {
+		optionalModuleState.seleniumBaseImports += 1;
+		if (optionalModuleState.seleniumBaseError) {
+			throw optionalModuleState.seleniumBaseError;
+		}
+		return {};
+	});
+}
+function parseSelector(expression: string): string | null {
+	const match = expression.match(/document\.querySelector\((".*?")\)/);
+	if (!match?.[1]) {
+		return null;
+	}
+	return JSON.parse(match[1]) as string;
+}
+class MockWebSocket {
+	static CONNECTING = 0;
+	static OPEN = 1;
+	static CLOSING = 2;
+	static CLOSED = 3;
+	readyState = MockWebSocket.CONNECTING;
+	private listeners = new Map<
+		string,
+		Set<(event?: { data?: string }) => void>
+	>();
+	constructor(private readonly endpoint: string) {
+		queueMicrotask(() => {
+			this.readyState = MockWebSocket.OPEN;
+			this.emit("open");
+		});
+	}
+	addEventListener(
+		event: string,
+		listener: (event?: { data?: string }) => void,
+	) {
+		const listeners = this.listeners.get(event) ?? new Set();
+		listeners.add(listener);
+		this.listeners.set(event, listeners);
+	}
+	close() {
+		this.readyState = MockWebSocket.CLOSED;
+		cdpState.closedEndpoints.push(this.endpoint);
+		this.emit("close");
+	}
+	send(raw: string) {
+		const message = JSON.parse(raw) as {
+			id: number;
+			method: string;
+			params?: Record<string, unknown>;
+		};
+		if (this.endpoint === "ws://pool.test") {
+			this.handlePoolMessage(message);
+			return;
+		}
+		this.handlePageMessage(message);
+	}
+	private emit(event: string, payload?: { data?: string }) {
+		for (const listener of this.listeners.get(event) ?? []) {
+			listener(payload);
+		}
+	}
+	private reply(id: number, result: Record<string, unknown>) {
+		this.emit("message", {
+			data: JSON.stringify({ id, jsonrpc: "2.0", result }),
+		});
+	}
+	private emitPageEvent(method: string, params: Record<string, unknown> = {}) {
+		this.emit("message", {
+			data: JSON.stringify({ jsonrpc: "2.0", method, params }),
+		});
+	}
+	private handlePoolMessage(message: {
+		id: number;
+		method: string;
+		params?: Record<string, unknown>;
+	}) {
+		switch (message.method) {
+			case "acquire":
+				cdpState.acquireCalls += 1;
+				this.reply(message.id, {
+					pageId: "pool-page-1",
+					wsEndpoint: "ws://page.test/devtools/page/pool-page-1",
+				});
+				break;
+			case "release":
+				cdpState.poolReleaseCalls.push(String(message.params?.pageId ?? ""));
+				this.reply(message.id, { released: true });
+				break;
+			default:
+				this.reply(message.id, {});
+		}
+	}
+	private handlePageMessage(message: {
+		id: number;
+		method: string;
+		params?: Record<string, unknown>;
+	}) {
+		switch (message.method) {
+			case "Page.enable":
+				cdpState.pageEnabled += 1;
+				this.reply(message.id, {});
+				break;
+			case "Runtime.enable":
+				cdpState.runtimeEnabled += 1;
+				this.reply(message.id, {});
+				break;
+			case "Page.navigate":
+				cdpState.navigateUrls.push(String(message.params?.url ?? ""));
+				this.reply(message.id, {});
+				queueMicrotask(() => this.emitPageEvent("Page.loadEventFired"));
+				break;
+			case "Runtime.evaluate": {
+				const expression = String(message.params?.expression ?? "");
+				const selector = parseSelector(expression);
+				if (expression.includes('window.navigator, "webdriver"')) {
+					cdpState.webdriverPatches += 1;
+					this.reply(message.id, { result: { value: undefined } });
+					break;
+				}
+				if (expression === "document.readyState") {
+					this.reply(message.id, { result: { value: "complete" } });
+					break;
+				}
+				if (expression === "document.documentElement.outerHTML") {
+					this.reply(message.id, {
+						result: { value: '<html><body><input id="name" /></body></html>' },
+					});
+					break;
+				}
+				if (expression === "document.title") {
+					this.reply(message.id, { result: { value: "remote-title" } });
+					break;
+				}
+				if (expression.includes("Boolean(document.querySelector")) {
+					this.reply(message.id, { result: { value: Boolean(selector) } });
+					break;
+				}
+				if (expression.includes("element.click()") && selector) {
+					cdpState.clicks.push(selector);
+					this.reply(message.id, { result: { value: undefined } });
+					break;
+				}
+				if (expression.includes("element.focus()") && selector) {
+					cdpState.focusedSelectors.push(selector);
+					this.reply(message.id, { result: { value: undefined } });
+					break;
+				}
+				this.reply(message.id, { result: { value: undefined } });
+				break;
+			}
+			case "Input.insertText":
+				cdpState.insertedTexts.push(String(message.params?.text ?? ""));
+				this.reply(message.id, {});
+				break;
+			case "Page.captureScreenshot":
+				cdpState.screenshotCalls.push(
+					Boolean(message.params?.captureBeyondViewport),
+				);
+				this.reply(message.id, {
+					data: Buffer.from("remote-shot").toString("base64"),
+				});
+				break;
+			default:
+				this.reply(message.id, {});
+		}
+	}
+}
+describe("createBrowserClient", () => {
+	beforeEach(() => {
+		browserState.browsers.length = 0;
+		browserState.launchCalls.length = 0;
+		browserState.requireError = null;
+		stealthState.callCount = 0;
+		stealthState.pages.length = 0;
+		optionalModuleState.nodriverError = null;
+		optionalModuleState.nodriverImports = 0;
+		optionalModuleState.seleniumBaseError = null;
+		optionalModuleState.seleniumBaseImports = 0;
+		cdpState.acquireCalls = 0;
+		cdpState.clicks.length = 0;
+		cdpState.closedEndpoints.length = 0;
+		cdpState.focusedSelectors.length = 0;
+		cdpState.insertedTexts.length = 0;
+		cdpState.navigateUrls.length = 0;
+		cdpState.poolReleaseCalls.length = 0;
+		cdpState.runtimeEnabled = 0;
+		cdpState.pageEnabled = 0;
+		cdpState.screenshotCalls.length = 0;
+		cdpState.webdriverPatches = 0;
+		process.env.CDP_POOL_URL = undefined;
+		globalThis.WebSocket = originalWebSocket;
+		registerBrowserMocks();
+	});
+	afterEach(() => {
+		if (originalCdpPoolUrl === undefined) {
+			delete process.env.CDP_POOL_URL;
+		} else {
+			process.env.CDP_POOL_URL = originalCdpPoolUrl;
+		}
+		globalThis.WebSocket = originalWebSocket;
+	});
+	it("throws ConnectorError with install hint when Playwright is unavailable", async () => {
+		browserState.requireError = Object.assign(
+			new Error("Cannot find module 'playwright'"),
+			{
+				code: "MODULE_NOT_FOUND",
+			},
+		);
+		registerBrowserMocks();
+		const { ConnectorError } = await import("../errors");
+		const { createBrowserClient } = await import("../runtime/browser");
+		const client = createBrowserClient();
+		const pagePromise = client.newPage();
+		await expect(pagePromise).rejects.toBeInstanceOf(ConnectorError);
+		await expect(pagePromise).rejects.toMatchObject({
+			fix: "Run: bun add playwright",
+			message: "Playwright is not installed",
+		});
+	});
+	it("falls back to local Playwright and implements browser page methods", async () => {
+		const { createBrowserClient } = await import("../runtime/browser");
+		const client = createBrowserClient();
+		const page = (await client.newPage()) as {
+			click(selector: string): Promise<void>;
+			content(): Promise<string>;
+			evaluate<T>(fn: string | (() => T)): Promise<T>;
+			fill(selector: string, text: string): Promise<void>;
+			goto(url: string): Promise<void>;
+			screenshot(options?: { fullPage?: boolean }): Promise<Buffer>;
+			type(selector: string, text: string): Promise<void>;
+			waitForSelector(
+				selector: string,
+				options?: { timeout?: number },
+			): Promise<void>;
+			close(): Promise<void>;
+		};
+		await page.goto("https://example.com/login");
+		await page.waitForSelector("#email", { timeout: 1234 });
+		await page.click("button[type=submit]");
+		await page.type("#email", "demo@example.com");
+		await page.fill("#otp", "123456");
+		const html = await page.content();
+		const title = await page.evaluate<string>("document.title");
+		const screenshot = await page.screenshot({ fullPage: true });
+		await page.close();
+		expect(browserState.launchCalls).toEqual([
+			{
+				args: undefined,
+				executablePath: undefined,
+				headless: true,
+				proxy: undefined,
+			},
+		]);
+		expect(stealthState.callCount).toBe(1);
+		expect(stealthState.pages).toHaveLength(1);
+		expect(browserState.browsers[0]?.newPageCalls).toBe(1);
+		expect(browserState.browsers[0]?.pages[0]?.state.gotoUrls).toEqual([
+			"https://example.com/login",
+		]);
+		expect(browserState.browsers[0]?.pages[0]?.state.waits).toEqual([
+			{ selector: "#email", timeout: 1234 },
+		]);
+		expect(browserState.browsers[0]?.pages[0]?.state.clicks).toEqual([
+			"button[type=submit]",
+		]);
+		expect(browserState.browsers[0]?.pages[0]?.state.types).toEqual([
+			{ selector: "#email", text: "demo@example.com" },
+		]);
+		expect(browserState.browsers[0]?.pages[0]?.state.fills).toEqual([
+			{ selector: "#email", text: "" },
+			{ selector: "#otp", text: "123456" },
+		]);
+		expect(html).toBe("<html><body>local</body></html>");
+		expect(title).toBe("local-title");
+		expect(screenshot.toString()).toBe("local-shot");
+		expect(browserState.browsers[0]?.pages[0]?.state.closed).toBeTrue();
+	});
+	it("passes launch options through and can disable stealth", async () => {
+		const { createBrowserClient } = await import("../runtime/browser");
+		const client = createBrowserClient({
+			extraArgs: ["--disable-dev-shm-usage"],
+			executablePath:
+				"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+			headless: false,
+			proxy: "http://127.0.0.1:8080",
+			stealth: false,
+		});
+		await client.newPage();
+		expect(browserState.launchCalls).toEqual([
+			{
+				args: ["--disable-dev-shm-usage"],
+				executablePath:
+					"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+				headless: false,
+				proxy: { server: "http://127.0.0.1:8080" },
+			},
+		]);
+		expect(stealthState.callCount).toBe(0);
+	});
+	it("uses CDP Pool when CDP_POOL_URL is configured", async () => {
+		globalThis.WebSocket = MockWebSocket as unknown as typeof WebSocket;
+		process.env.CDP_POOL_URL = "ws://pool.test";
+		const { createBrowserClient } = await import("../runtime/browser");
+		const client = createBrowserClient();
+		const page = (await client.newPage()) as {
+			pageId: string;
+			click(selector: string): Promise<void>;
+			content(): Promise<string>;
+			evaluate<T>(fn: string | (() => T)): Promise<T>;
+			goto(url: string): Promise<void>;
+			screenshot(options?: { fullPage?: boolean }): Promise<Buffer>;
+			type(selector: string, text: string): Promise<void>;
+			waitForSelector(selector: string): Promise<void>;
+			close(): Promise<void>;
+		};
+		await page.goto("https://bank.example.com/login");
+		await page.waitForSelector("#name");
+		await page.click("#submit");
+		await page.type("#name", "demo");
+		const title = await page.evaluate<string>("document.title");
+		const html = await page.content();
+		const screenshot = await page.screenshot({ fullPage: true });
+		await page.close();
+		await client.close();
+		expect(page.pageId).toBe("pool-page-1");
+		expect(browserState.launchCalls).toHaveLength(0);
+		expect(cdpState.acquireCalls).toBe(1);
+		expect(cdpState.pageEnabled).toBe(1);
+		expect(cdpState.runtimeEnabled).toBe(1);
+		expect(cdpState.webdriverPatches).toBe(1);
+		expect(cdpState.navigateUrls).toEqual(["https://bank.example.com/login"]);
+		expect(cdpState.clicks).toEqual(["#submit"]);
+		expect(cdpState.focusedSelectors).toEqual(["#name"]);
+		expect(cdpState.insertedTexts).toEqual(["demo"]);
+		expect(cdpState.screenshotCalls).toEqual([true]);
+		expect(cdpState.poolReleaseCalls).toEqual(["pool-page-1"]);
+		expect(title).toBe("remote-title");
+		expect(html).toContain('<input id="name" />');
+		expect(screenshot.toString()).toBe("remote-shot");
+		expect(cdpState.closedEndpoints).toContain(
+			"ws://page.test/devtools/page/pool-page-1",
+		);
+		expect(cdpState.closedEndpoints).toContain("ws://pool.test");
+	});
+	it("exposes the resolved engine on the browser client", async () => {
+		const { createBrowserClient } = await import("../runtime/browser");
+		const defaultClient = createBrowserClient();
+		const nodriverClient = createBrowserClient({ engine: "nodriver" });
+		expect(defaultClient.engine).toBe("playwright-stealth");
+		expect(nodriverClient.engine).toBe("nodriver");
+	});
+	it("throws a Python runtime error for the nodriver engine", async () => {
+		const { ConnectorError } = await import("../errors");
+		const { createBrowserClient } = await import("../runtime/browser");
+		const client = createBrowserClient({ engine: "nodriver" });
+		const pagePromise = client.newPage();
+		await expect(pagePromise).rejects.toBeInstanceOf(ConnectorError);
+		await expect(pagePromise).rejects.toMatchObject({
+			fix: "Use connector language: python and ctx.browser in Python",
+			message: "nodriver engine requires Python runtime",
+		});
+		expect(optionalModuleState.nodriverImports).toBe(1);
+	});
+	it("throws a Python runtime error for the selenium-uc engine", async () => {
+		const { ConnectorError } = await import("../errors");
+		const { createBrowserClient } = await import("../runtime/browser");
+		const client = createBrowserClient({ engine: "selenium-uc" });
+		const pagePromise = client.newPage();
+		await expect(pagePromise).rejects.toBeInstanceOf(ConnectorError);
+		await expect(pagePromise).rejects.toMatchObject({
+			fix: "Use connector language: python",
+			message: "selenium-uc engine requires Python runtime",
+		});
+		expect(optionalModuleState.seleniumBaseImports).toBe(1);
+	});
+	it("closes the underlying local browser instance", async () => {
+		const { createBrowserClient } = await import("../runtime/browser");
+		const client = createBrowserClient();
+		await client.newPage();
+		await client.close();
+		expect(browserState.browsers[0]?.closeCalls).toBe(1);
+		expect(browserState.browsers[0]?.connected).toBeFalse();
+	});
+});