@apifuse/provider-sdk 2.1.0-beta.3 → 2.1.0-beta.5

package/src/public-schema-field-lint.ts

@@ -0,0 +1,237 @@
+import type { ZodType } from "zod";
+
+import type { LintDiagnostic } from "./lint";
+
+type SchemaLike = ZodType & {
+	def?: Record<string, unknown>;
+	_def?: Record<string, unknown>;
+	shape?: Record<string, SchemaLike> | (() => Record<string, SchemaLike>);
+	element?: SchemaLike;
+	items?: SchemaLike[];
+	options?: SchemaLike[] | Set<SchemaLike> | Map<string, SchemaLike>;
+	innerType?: SchemaLike;
+	sourceType?: () => SchemaLike;
+	unwrap?: () => SchemaLike;
+	in?: SchemaLike;
+	out?: SchemaLike;
+	left?: SchemaLike;
+	right?: SchemaLike;
+};
+
+const ESTABLISHED_APIFUSE_PROTOCOL_FIELDS = new Set(["externalRef"]);
+
+const UPSTREAM_FIELD_REPLACEMENTS = new Map<string, string>([
+	["display", "limit"],
+	["start", "offset"],
+	["sort", "sort_by"],
+	["lprice", "lowest_price"],
+	["hprice", "highest_price"],
+	["mallName", "mall_name"],
+	["productId", "product_id"],
+	["productType", "product_type_code"],
+	["lastBuildDate", "upstream_generated_at"],
+	["meta", "summary"],
+]);
+
+function isSchema(value: unknown): value is SchemaLike {
+	return (
+		!!value &&
+		typeof value === "object" &&
+		"safeParse" in value &&
+		typeof value.safeParse === "function"
+	);
+}
+
+function getSchemaDef(schema: SchemaLike): Record<string, unknown> {
+	const def = schema.def ?? schema._def;
+	return def && typeof def === "object" ? def : {};
+}
+
+function isSchemaRecord(value: unknown): value is Record<string, SchemaLike> {
+	if (!value || typeof value !== "object") {
+		return false;
+	}
+	return Object.values(value).every(isSchema);
+}
+
+function getObjectShape(schema: SchemaLike): Record<string, SchemaLike> {
+	const rawShape =
+		typeof schema.shape === "function" ? schema.shape() : schema.shape;
+	if (isSchemaRecord(rawShape)) {
+		return rawShape;
+	}
+
+	const defShape = getSchemaDef(schema).shape;
+	if (typeof defShape === "function") {
+		const resolved = defShape();
+		return isSchemaRecord(resolved) ? resolved : {};
+	}
+	return isSchemaRecord(defShape) ? defShape : {};
+}
+
+function appendSchemaChildren(
+	children: SchemaLike[],
+	value: unknown,
+): SchemaLike[] {
+	if (isSchema(value)) {
+		children.push(value);
+		return children;
+	}
+	if (Array.isArray(value)) {
+		children.push(...value.filter(isSchema));
+		return children;
+	}
+	if (value instanceof Set) {
+		children.push(...Array.from(value).filter(isSchema));
+		return children;
+	}
+	if (value instanceof Map) {
+		children.push(...Array.from(value.values()).filter(isSchema));
+		return children;
+	}
+	return children;
+}
+
+function safeSourceType(schema: SchemaLike): SchemaLike | undefined {
+	try {
+		return schema.sourceType?.();
+	} catch {
+		return undefined;
+	}
+}
+
+function safeUnwrap(schema: SchemaLike): SchemaLike | undefined {
+	try {
+		return schema.unwrap?.();
+	} catch {
+		return undefined;
+	}
+}
+
+function getTransparentChildSchemas(schema: SchemaLike): SchemaLike[] {
+	const def = getSchemaDef(schema);
+	const children: SchemaLike[] = [];
+	for (const value of [
+		schema.element,
+		schema.items,
+		schema.options,
+		schema.innerType,
+		safeSourceType(schema),
+		safeUnwrap(schema),
+		schema.in,
+		schema.out,
+		schema.left,
+		schema.right,
+		def.schema,
+		def.innerType,
+		def.type,
+		def.valueType,
+		def.item,
+		def.items,
+		def.rest,
+		def.catchall,
+		def.option,
+		def.options,
+		def.pipe,
+		def.payload,
+		def.sourceType,
+		def.left,
+		def.right,
+	]) {
+		appendSchemaChildren(children, value);
+	}
+	return children;
+}
+
+function recommendedReplacement(fieldName: string): string | undefined {
+	if (/^category\d+$/.test(fieldName)) {
+		return "category_path";
+	}
+	if (UPSTREAM_FIELD_REPLACEMENTS.has(fieldName)) {
+		return UPSTREAM_FIELD_REPLACEMENTS.get(fieldName);
+	}
+	if (/[a-z][A-Z]/.test(fieldName)) {
+		return fieldName.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase();
+	}
+	return undefined;
+}
+
+function collectPublicSchemaFieldDiagnostics(
+	providerId: string,
+	operationId: string,
+	schema: unknown,
+	basePath: string,
+	seen = new Set<SchemaLike>(),
+): LintDiagnostic[] {
+	if (!isSchema(schema) || seen.has(schema)) {
+		return [];
+	}
+
+	seen.add(schema);
+	const diagnostics: LintDiagnostic[] = [];
+	for (const [fieldName, child] of Object.entries(getObjectShape(schema))) {
+		const fieldPath = `${basePath}.${fieldName}`;
+		const replacement = ESTABLISHED_APIFUSE_PROTOCOL_FIELDS.has(fieldName)
+			? undefined
+			: recommendedReplacement(fieldName);
+		if (replacement) {
+			diagnostics.push({
+				rule: "public-schema-upstream-field",
+				level: "error",
+				field: fieldPath,
+				message: `Provider "${providerId}" operation "${operationId}" public schema field "${fieldPath}" uses upstream-shaped field "${fieldName}"; use APIFuse field "${replacement}" instead.`,
+			});
+		}
+		diagnostics.push(
+			...collectPublicSchemaFieldDiagnostics(
+				providerId,
+				operationId,
+				child,
+				fieldPath,
+				seen,
+			),
+		);
+	}
+
+	for (const child of getTransparentChildSchemas(schema)) {
+		const childPath = schema.element === child ? `${basePath}[]` : basePath;
+		diagnostics.push(
+			...collectPublicSchemaFieldDiagnostics(
+				providerId,
+				operationId,
+				child,
+				childPath,
+				seen,
+			),
+		);
+	}
+
+	return diagnostics;
+}
+
+export function lintPublicSchemaFieldNames(
+	providerId: string | undefined,
+	operationId: string,
+	input: unknown,
+	output: unknown,
+	enforce: boolean,
+): LintDiagnostic[] {
+	if (!providerId || !enforce) {
+		return [];
+	}
+
+	return [
+		...collectPublicSchemaFieldDiagnostics(
+			providerId,
+			operationId,
+			input,
+			"input",
+		),
+		...collectPublicSchemaFieldDiagnostics(
+			providerId,
+			operationId,
+			output,
+			"output",
+		),
+	];
+}

package/src/runtime/auth-flow.ts

@@ -4,7 +4,10 @@ import type {
 	EnvContext,
 	FlowContext,
 	HttpClient,
+	StealthClient,
+	SttContext,
 } from "../types";
+import { createUnsupportedSttClient } from "./stt";
 
 function normalizeAllowedKeys(allowedKeys: string[]): Set<string> {
 	return new Set(allowedKeys.filter((key) => key.trim().length > 0));
@@ -47,6 +50,7 @@ export function createScratchpad(
 
 export function createFlowContext(options: {
 	http: HttpClient;
+	stealth: StealthClient;
 	env: EnvContext;
 	tenantId: string;
 	providerId: string;
@@ -54,6 +58,7 @@ export function createFlowContext(options: {
 	externalRef?: string;
 	allowedKeys: string[];
 	initialContext?: Record<string, unknown>;
+	stt?: SttContext;
 }): FlowContext {
 	return {
 		connectionId: options.connectionId,
@@ -61,7 +66,9 @@ export function createFlowContext(options: {
 		tenantId: options.tenantId,
 		providerId: options.providerId,
 		http: options.http,
+		stealth: options.stealth,
 		env: options.env,
 		context: createScratchpad(options.allowedKeys, options.initialContext),
+		stt: options.stt ?? createUnsupportedSttClient(),
 	};
 }

package/src/runtime/browser.ts

@@ -13,10 +13,14 @@ const DEFAULT_WAIT_TIMEOUT_MS = 30_000;
 const SELECTOR_POLL_INTERVAL_MS = 100;
 
 type PlaywrightModule = typeof import("playwright");
-type PlaywrightStealthModule = {
-	stealth(page: unknown): Promise<void>;
+type PlaywrightExtraModule = {
+	chromium: PlaywrightModule["chromium"] & { use(plugin: unknown): unknown };
 };
 
+type StealthPluginFactory = (options?: {
+	enabledEvasions?: Set<string>;
+}) => unknown;
+
 type PoolAcquireResponse = {
 	pageId: string;
 	wsEndpoint: string;
@@ -65,7 +69,7 @@ type SupportedBrowserClient = BrowserClientContract & {
 };
 
 function getDefaultCdpPoolUrl(env = process.env): string | undefined {
-	return env.CDP_POOL_URL ?? env.APIFUSE_CDP_POOL_URL;
+	return env.APIFUSE__CDP_POOL__URL;
 }
 
 async function importOptionalModule<T extends object>(
@@ -89,11 +93,16 @@ function unwrapModuleDefault<T extends object>(module: T): T {
 }
 
 function isModuleNotFoundError(error: unknown): boolean {
+	if (!(error instanceof Error)) {
+		return false;
+	}
+
+	const code = "code" in error ? error.code : undefined;
 	return (
-		error instanceof Error &&
-		("code" in error
-			? (error as Error & { code?: string }).code === "MODULE_NOT_FOUND"
-			: error.message.includes("Cannot find module"))
+		code === "MODULE_NOT_FOUND" ||
+		code === "ERR_MODULE_NOT_FOUND" ||
+		error.message.includes("Cannot find module") ||
+		error.message.includes("Cannot find package")
 	);
 }
 
@@ -120,7 +129,7 @@ function toLaunchOptions(options: BrowserClientOptions): LaunchOptions {
 
 async function loadPlaywright(): Promise<PlaywrightModule> {
 	try {
-		require("playwright");
+		await importOptionalModule<PlaywrightModule>("playwright");
 	} catch (error) {
 		if (isModuleNotFoundError(error)) {
 			throw new ProviderError("Playwright is not installed", {
@@ -148,16 +157,31 @@ async function loadPlaywright(): Promise<PlaywrightModule> {
 	}
 }
 
-async function loadPlaywrightStealth(): Promise<PlaywrightStealthModule> {
+const playwrightExtraStealthLaunchers = new WeakSet<object>();
+
+async function loadPlaywrightExtra(): Promise<PlaywrightExtraModule> {
+	try {
+		require("playwright");
+	} catch (error) {
+		if (isModuleNotFoundError(error)) {
+			throw new ProviderError("Playwright is not installed", {
+				cause: error instanceof Error ? error : undefined,
+				fix: "Run: bun add playwright",
+			});
+		}
+
+		throw error;
+	}
+
 	try {
 		return unwrapModuleDefault(
-			await importOptionalModule<PlaywrightStealthModule>("playwright-stealth"),
+			await importOptionalModule<PlaywrightExtraModule>("playwright-extra"),
 		);
 	} catch (error) {
 		if (isModuleNotFoundError(error)) {
-			throw new ProviderError("playwright-stealth is not installed", {
+			throw new ProviderError("playwright-extra is not installed", {
 				cause: error instanceof Error ? error : undefined,
-				fix: "Run: bun add playwright-stealth",
+				fix: "Run: bun add playwright-extra puppeteer-extra-plugin-stealth",
 			});
 		}
 
@@ -165,6 +189,45 @@ async function loadPlaywrightStealth(): Promise<PlaywrightStealthModule> {
 	}
 }
 
+async function loadStealthPluginFactory(): Promise<StealthPluginFactory> {
+	try {
+		return unwrapModuleDefault(
+			await importOptionalModule<StealthPluginFactory>(
+				"puppeteer-extra-plugin-stealth",
+			),
+		);
+	} catch (error) {
+		if (isModuleNotFoundError(error)) {
+			throw new ProviderError(
+				"puppeteer-extra-plugin-stealth is not installed",
+				{
+					cause: error instanceof Error ? error : undefined,
+					fix: "Run: bun add playwright-extra puppeteer-extra-plugin-stealth",
+				},
+			);
+		}
+
+		throw error;
+	}
+}
+
+async function loadChromiumLauncher(
+	options: BrowserClientOptions,
+): Promise<PlaywrightModule["chromium"]> {
+	if (!(options.stealth ?? true)) {
+		return (await loadPlaywright()).chromium;
+	}
+
+	const playwrightExtra = await loadPlaywrightExtra();
+	if (!playwrightExtraStealthLaunchers.has(playwrightExtra.chromium)) {
+		const createStealthPlugin = await loadStealthPluginFactory();
+		playwrightExtra.chromium.use(createStealthPlugin());
+		playwrightExtraStealthLaunchers.add(playwrightExtra.chromium);
+	}
+
+	return playwrightExtra.chromium;
+}
+
 async function loadNodriver(): Promise<void> {
 	try {
 		await importOptionalModule("nodriver");
@@ -256,10 +319,8 @@ class PlaywrightBrowserClient implements SupportedBrowserClient {
 			return this.browser;
 		}
 
-		const playwright = await loadPlaywright();
-		this.browser = await playwright.chromium.launch(
-			toLaunchOptions(this.options),
-		);
+		const chromium = await loadChromiumLauncher(this.options);
+		this.browser = await chromium.launch(toLaunchOptions(this.options));
 		return this.browser;
 	}
 
@@ -267,11 +328,6 @@ class PlaywrightBrowserClient implements SupportedBrowserClient {
 		const browser = await this.ensureBrowser();
 		const page = await browser.newPage();
 
-		if (this.options.stealth ?? true) {
-			const { stealth } = await loadPlaywrightStealth();
-			await stealth(page);
-		}
-
 		return new PlaywrightBrowserPage(page);
 	}