@apifuse/connector-sdk 2.0.0-beta.1

package/src/runtime/tls.ts

@@ -0,0 +1,410 @@
+import { ModuleClient, SessionClient } from "tlsclientwrapper";
+
+import type { ProxyResolutionOptions } from "../config/loader";
+import { resolveProxyConfig } from "../config/loader";
+import { TransportError } from "../errors";
+import { getStealthProfile } from "../stealth/profiles";
+import type {
+	CookieJar,
+	TlsClient,
+	TlsFetchOptions,
+	TlsResponse,
+	TlsSession,
+} from "../types";
+
+const DEFAULT_PROFILE = "chrome-146";
+
+const MISSING_PROXY_WARNING =
+	"[connector-sdk] Connector requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
+
+export type TlsClientOptions = ProxyResolutionOptions & {
+	warn?: (message: string) => void;
+};
+
+const TLS_PROFILE_MAP: Record<string, string> = {
+	"chrome-131": "chrome_131",
+	"chrome-133": "chrome_133",
+	"chrome-144": "chrome_144",
+	"chrome-146": "chrome_146",
+	"firefox-132": "firefox_132",
+	"firefox-133": "firefox_133",
+	"firefox-135": "firefox_135",
+	"firefox-147": "firefox_147",
+	"safari-15": "safari_15_6_1",
+	"safari-16": "safari_16_0",
+	"ios-safari-17": "safari_ios_17_0",
+	"ios-safari-18": "safari_ios_18_0",
+	"ios-safari-26": "safari_ios_26_0",
+};
+
+type TlsClientWrapperResponse = {
+	status: number;
+	body: string;
+	headers?: Record<string, string>;
+	rawHeaders?: Record<string, string> | [string, string][];
+	usedProtocol?: string;
+	httpVersion?: string;
+	cookies?:
+		| Record<string, { name?: string; value?: string }>
+		| Array<{ name?: string; value?: string }>;
+};
+
+class CookieJarImpl implements CookieJar {
+	private readonly cookies: Record<string, string>;
+
+	constructor(cookieStrings: string[]) {
+		this.cookies = {};
+
+		for (const cookieString of cookieStrings) {
+			const [nameValue] = cookieString.split(";");
+			if (!nameValue) {
+				continue;
+			}
+
+			const separatorIndex = nameValue.indexOf("=");
+			if (separatorIndex === -1) {
+				continue;
+			}
+
+			const name = nameValue.slice(0, separatorIndex).trim();
+			const value = nameValue.slice(separatorIndex + 1).trim();
+			this.cookies[name] = value;
+		}
+	}
+
+	get(name: string): string | undefined {
+		return this.cookies[name];
+	}
+
+	getAll(): Record<string, string> {
+		return { ...this.cookies };
+	}
+
+	toString(): string {
+		return Object.entries(this.cookies)
+			.map(([name, value]) => `${name}=${value}`)
+			.join("; ");
+	}
+
+	find(predicate: (cookie: string) => boolean): string | undefined {
+		for (const [name, value] of Object.entries(this.cookies)) {
+			const cookie = `${name}=${value}`;
+			if (predicate(cookie)) {
+				return cookie;
+			}
+		}
+
+		return undefined;
+	}
+}
+
+function resolveIdentifier(profileName: string): string {
+	try {
+		const profile = getStealthProfile(profileName);
+		if (profile.tlsClientIdentifier) {
+			return profile.tlsClientIdentifier;
+		}
+	} catch {
+		// Unknown profile name falls through to static mapping and heuristic.
+	}
+
+	return TLS_PROFILE_MAP[profileName] ?? profileName.replaceAll("-", "_");
+}
+
+function resolveUrl(baseUrl: string, url: string): string {
+	return new URL(url, baseUrl).toString();
+}
+
+function toCookieStrings(
+	response: TlsClientWrapperResponse,
+): string[] | undefined {
+	const cookies = response.cookies;
+	if (!cookies) {
+		return undefined;
+	}
+
+	const values = Array.isArray(cookies) ? cookies : Object.values(cookies);
+	const normalized = values
+		.map((cookie) => {
+			if (!cookie?.name) {
+				return null;
+			}
+
+			return `${cookie.name}=${cookie.value ?? ""}`;
+		})
+		.filter((cookie): cookie is string => cookie !== null);
+
+	return normalized.length > 0 ? normalized : undefined;
+}
+
+function normalizeRawHeaders(
+	rawHeaders: TlsClientWrapperResponse["rawHeaders"],
+	headers: Record<string, string>,
+): [string, string][] {
+	if (Array.isArray(rawHeaders)) {
+		return rawHeaders;
+	}
+
+	if (rawHeaders && typeof rawHeaders === "object") {
+		return Object.entries(rawHeaders);
+	}
+
+	return Object.entries(headers);
+}
+
+export function normalizeResponse(
+	response: TlsClientWrapperResponse,
+): TlsResponse {
+	const headers = response.headers ?? {};
+	const cookies = new CookieJarImpl(toCookieStrings(response) ?? []);
+
+	return {
+		status: response.status,
+		ok: response.status >= 200 && response.status < 300,
+		headers,
+		rawHeaders: normalizeRawHeaders(response.rawHeaders, headers),
+		body: response.body,
+		httpVersion: response.httpVersion ?? response.usedProtocol,
+		cookies,
+		json<T>(): Promise<T> {
+			return Promise.resolve(JSON.parse(response.body) as T);
+		},
+	};
+}
+
+function getErrorMessage(error: unknown): string {
+	if (error instanceof Error) {
+		return error.toString();
+	}
+
+	return String(error);
+}
+
+function normalizeBody(body: TlsFetchOptions["body"]): string | null {
+	if (body === undefined) {
+		return null;
+	}
+
+	if (typeof body === "string") {
+		return body;
+	}
+
+	if (Buffer.isBuffer(body)) {
+		return body.toString();
+	}
+
+	return String(body);
+}
+
+async function sendRequest(
+	session: SessionClient,
+	method: string,
+	url: string,
+	options: TlsFetchOptions,
+): Promise<TlsClientWrapperResponse> {
+	const requestOptions = {
+		headers: options.headers,
+		proxy: options.proxy,
+		...(options.headerOrder ? { headerOrder: options.headerOrder } : {}),
+	};
+
+	switch (method) {
+		case "POST":
+			return (await session.post(
+				url,
+				normalizeBody(options.body),
+				requestOptions,
+			)) as TlsClientWrapperResponse;
+		case "PUT":
+			return (await session.put(
+				url,
+				normalizeBody(options.body),
+				requestOptions,
+			)) as TlsClientWrapperResponse;
+		case "PATCH":
+			return (await session.patch(
+				url,
+				normalizeBody(options.body),
+				requestOptions,
+			)) as TlsClientWrapperResponse;
+		case "DELETE":
+			return (await session.delete(
+				url,
+				requestOptions,
+			)) as TlsClientWrapperResponse;
+		case "HEAD":
+			return (await session.head(
+				url,
+				requestOptions,
+			)) as TlsClientWrapperResponse;
+		case "OPTIONS":
+			return (await session.options(
+				url,
+				requestOptions,
+			)) as TlsClientWrapperResponse;
+		default:
+			return (await session.get(
+				url,
+				requestOptions,
+			)) as TlsClientWrapperResponse;
+	}
+}
+
+function createSessionFetcher(
+	baseUrl: string,
+	defaultProfile: string,
+	moduleClient: ModuleClient,
+	clientOptions: TlsClientOptions,
+): TlsSession {
+	let sessionClient: SessionClient | null = null;
+	let activeProxy: string | undefined;
+	let activeTlsIdentifier: string | undefined;
+	let hasWarnedMissingProxy = false;
+	const warn = clientOptions.warn ?? console.warn;
+
+	function resolveRequestProxy(options?: TlsFetchOptions): string | undefined {
+		const resolvedProxy = resolveProxyConfig({
+			proxy: options?.proxy ?? clientOptions.proxy,
+			upstream: clientOptions.upstream,
+			apifuseConfig: clientOptions.apifuseConfig,
+		});
+
+		if (resolvedProxy.shouldWarn && !hasWarnedMissingProxy) {
+			hasWarnedMissingProxy = true;
+			warn(MISSING_PROXY_WARNING);
+		}
+
+		return resolvedProxy.url;
+	}
+
+	function closeCurrentSession(): void {
+		const client = sessionClient as
+			| (SessionClient & {
+					destroySession?: () => Promise<unknown>;
+					close?: () => void;
+			  })
+			| null;
+
+		void client?.destroySession?.();
+		client?.close?.();
+		sessionClient = null;
+		activeProxy = undefined;
+		activeTlsIdentifier = undefined;
+	}
+
+	function getSessionClient(
+		profile?: string,
+		proxy?: string,
+		ja3?: string,
+	): SessionClient {
+		const tlsIdentifier = ja3 ?? resolveIdentifier(profile ?? defaultProfile);
+
+		if (
+			!sessionClient ||
+			activeProxy !== proxy ||
+			activeTlsIdentifier !== tlsIdentifier
+		) {
+			if (sessionClient) {
+				closeCurrentSession();
+			}
+
+			sessionClient = new SessionClient(moduleClient, {
+				tlsClientIdentifier: tlsIdentifier,
+				...(proxy ? { proxyUrl: proxy } : {}),
+				timeoutSeconds: 30,
+			} as ConstructorParameters<typeof SessionClient>[1]);
+			activeProxy = proxy;
+			activeTlsIdentifier = tlsIdentifier;
+		}
+
+		return sessionClient;
+	}
+
+	return {
+		async fetch(url, options = {}) {
+			const proxy = resolveRequestProxy(options);
+			const session = getSessionClient(
+				options.profile,
+				proxy,
+				options.tls?.ja3,
+			);
+			const requestUrl = resolveUrl(baseUrl, url);
+
+			try {
+				const method = (options.method ?? "GET").toUpperCase();
+				const response = await sendRequest(session, method, requestUrl, {
+					...options,
+					proxy,
+				});
+
+				if (response.status >= 400) {
+					throw new TransportError(
+						`HTTP ${response.status}: ${response.body || "Request failed"}`,
+						{
+							status: response.status,
+						},
+					);
+				}
+
+				return normalizeResponse(response);
+			} catch (error) {
+				if (error instanceof TransportError) {
+					throw error;
+				}
+
+				throw new TransportError(`Network error: ${getErrorMessage(error)}`, {
+					status: 0,
+					cause: error instanceof Error ? error : undefined,
+				});
+			}
+		},
+		close() {
+			closeCurrentSession();
+		},
+	};
+}
+
+export function createTlsClient(
+	baseUrl: string,
+	defaultProfileOrOptions: string | TlsClientOptions = DEFAULT_PROFILE,
+	clientOptions: TlsClientOptions = {},
+): TlsClient {
+	const defaultProfile =
+		typeof defaultProfileOrOptions === "string"
+			? defaultProfileOrOptions
+			: DEFAULT_PROFILE;
+	const resolvedClientOptions =
+		typeof defaultProfileOrOptions === "string"
+			? clientOptions
+			: defaultProfileOrOptions;
+	const moduleClient = new ModuleClient();
+	let sharedSession: TlsSession | null = null;
+
+	function getSharedSession(): TlsSession {
+		if (!sharedSession) {
+			sharedSession = createSessionFetcher(
+				baseUrl,
+				defaultProfile,
+				moduleClient,
+				resolvedClientOptions,
+			);
+		}
+
+		return sharedSession;
+	}
+
+	return {
+		fetch(url: string, options?: TlsFetchOptions) {
+			return getSharedSession().fetch(url, options);
+		},
+		createSession(opts?: { profile?: string }) {
+			const sessionProfile = opts?.profile ?? defaultProfile;
+			return createSessionFetcher(
+				baseUrl,
+				sessionProfile,
+				moduleClient,
+				resolvedClientOptions,
+			);
+		},
+	};
+}

package/src/runtime/trace.ts

@@ -0,0 +1,261 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+
+import type {
+	TraceContext as BaseTraceContext,
+	TraceAttributeValue,
+	TraceConfig,
+	TraceSpan,
+} from "../types";
+import { exportSpansOTLP, type OTLPExportOptions } from "./otlp";
+
+export type SpanAttributeValue = TraceAttributeValue;
+
+export type Span = TraceSpan;
+
+export interface TraceContext extends BaseTraceContext {
+	getSpans(): Span[];
+}
+
+export interface CreateTraceContextOptions {
+	maxSpans?: number;
+	onSpan?: (span: Span) => void;
+	exportOptions?: OTLPExportOptions;
+	resourceAttributes?: Record<string, string>;
+}
+
+type SpanHookOptions<T> = {
+	attributes?: Record<string, unknown>;
+	onSuccess?: (value: T) => Record<string, unknown> | undefined;
+	onError?: (error: unknown) => Record<string, unknown> | undefined;
+};
+
+type PendingSpan = {
+	id: string;
+	name: string;
+	startedAt: number;
+	parentId?: string;
+	sequence: number;
+	attributes: Record<string, SpanAttributeValue>;
+};
+
+type CompletedSpanEntry = {
+	sequence: number;
+	span: Span;
+};
+
+export interface TraceRecorder {
+	runSpan<T>(
+		name: string,
+		fn: () => Promise<T> | T,
+		options?: SpanHookOptions<T>,
+	): Promise<T>;
+}
+
+export const TRACE_RECORDER = Symbol.for(
+	"@apifuse/connector-sdk/runtime/trace-recorder",
+);
+
+type InternalTraceContext = TraceContext & {
+	[TRACE_RECORDER]: TraceRecorder;
+	_exportOptions?: OTLPExportOptions;
+	_resourceAttributes?: Record<string, string>;
+};
+
+function buildOTLPExportOptions(
+	config?: TraceConfig,
+): OTLPExportOptions | undefined {
+	if (!config || config.exporter !== "otlp") {
+		return undefined;
+	}
+
+	const endpoint = config.otlp?.endpoint ?? config.endpoint;
+	if (!endpoint) {
+		return undefined;
+	}
+
+	return {
+		endpoint,
+		headers: config.otlp?.headers,
+		timeout: config.otlp?.timeout,
+	};
+}
+
+export function resolveTraceContextOptions(
+	config?: TraceConfig,
+): CreateTraceContextOptions {
+	return {
+		maxSpans: config?.maxSpans,
+		onSpan: config?.onSpan,
+		exportOptions: buildOTLPExportOptions(config),
+	};
+}
+
+function normalizeAttributeValue(
+	value: unknown,
+): SpanAttributeValue | undefined {
+	if (
+		typeof value === "string" ||
+		typeof value === "number" ||
+		typeof value === "boolean"
+	) {
+		return value;
+	}
+
+	if (value === null || value === undefined) {
+		return undefined;
+	}
+
+	return String(value);
+}
+
+function normalizeAttributes(
+	attributes?: Record<string, unknown>,
+): Record<string, SpanAttributeValue> {
+	if (!attributes) {
+		return {};
+	}
+
+	const normalizedEntries = Object.entries(attributes)
+		.map(([key, value]) => [key, normalizeAttributeValue(value)] as const)
+		.filter(
+			(entry): entry is readonly [string, SpanAttributeValue] =>
+				entry[1] !== undefined,
+		);
+
+	return Object.fromEntries(normalizedEntries);
+}
+
+function insertCompletedSpan(
+	completed: CompletedSpanEntry[],
+	entry: CompletedSpanEntry,
+	maxSpans: number,
+): void {
+	const insertAt = completed.findIndex(
+		(existingEntry) => existingEntry.sequence > entry.sequence,
+	);
+
+	if (insertAt === -1) {
+		completed.push(entry);
+	} else {
+		completed.splice(insertAt, 0, entry);
+	}
+
+	if (completed.length > maxSpans) {
+		completed.splice(0, completed.length - maxSpans);
+	}
+}
+
+export function getTraceRecorder(
+	trace: BaseTraceContext,
+): TraceRecorder | null {
+	return (trace as Partial<InternalTraceContext>)[TRACE_RECORDER] ?? null;
+}
+
+export function createTraceContext(
+	options: CreateTraceContextOptions = {},
+): TraceContext {
+	const maxSpans = options.maxSpans ?? 1000;
+	const completed: CompletedSpanEntry[] = [];
+	const activeSpanStorage = new AsyncLocalStorage<PendingSpan | undefined>();
+	let sequence = 0;
+	let traceContext!: InternalTraceContext;
+
+	const scheduleExport = () => {
+		if (!traceContext._exportOptions) {
+			return;
+		}
+
+		const spans = completed.map((entry) => ({ ...entry.span }));
+		setImmediate(() => {
+			void exportSpansOTLP(
+				spans,
+				traceContext._exportOptions as OTLPExportOptions,
+				traceContext._resourceAttributes,
+			);
+		});
+	};
+
+	const recorder: TraceRecorder = {
+		async runSpan(name, fn, spanOptions = {}) {
+			const pendingSpan: PendingSpan = {
+				id: crypto.randomUUID(),
+				name,
+				startedAt: Date.now(),
+				parentId: activeSpanStorage.getStore()?.id,
+				sequence: sequence++,
+				attributes: normalizeAttributes(spanOptions.attributes),
+			};
+
+			const finalize = (
+				status: Span["status"],
+				extraAttributes?: Record<string, unknown>,
+				error?: string,
+			) => {
+				const endedAt = Date.now();
+				const duration = endedAt - pendingSpan.startedAt;
+				const attributes = {
+					...pendingSpan.attributes,
+					...normalizeAttributes(extraAttributes),
+				};
+
+				if (attributes.duration_ms === undefined) {
+					attributes.duration_ms = duration;
+				}
+
+				const span: Span = {
+					id: pendingSpan.id,
+					name: pendingSpan.name,
+					startedAt: pendingSpan.startedAt,
+					endedAt,
+					duration_ms: duration,
+					status,
+					attributes,
+					...(error ? { error } : {}),
+					...(pendingSpan.parentId ? { parentId: pendingSpan.parentId } : {}),
+				};
+
+				insertCompletedSpan(
+					completed,
+					{ sequence: pendingSpan.sequence, span },
+					maxSpans,
+				);
+				options.onSpan?.(span);
+
+				if (!pendingSpan.parentId) {
+					scheduleExport();
+				}
+			};
+
+			return activeSpanStorage.run(pendingSpan, async () => {
+				try {
+					const value = await fn();
+					const successAttributes = spanOptions.onSuccess?.(value);
+					finalize("ok", successAttributes ?? undefined);
+					return value;
+				} catch (error) {
+					const errorMessage =
+						error instanceof Error ? error.message : String(error);
+					const errorAttributes = spanOptions.onError?.(error);
+					finalize("error", errorAttributes ?? undefined, errorMessage);
+					throw error;
+				}
+			});
+		},
+	};
+
+	traceContext = {
+		span(name, fn) {
+			return recorder.runSpan(name, fn);
+		},
+		getSpans() {
+			return completed.map((entry) => ({ ...entry.span }));
+		},
+		...(options.exportOptions ? { _exportOptions: options.exportOptions } : {}),
+		...(options.resourceAttributes
+			? { _resourceAttributes: { ...options.resourceAttributes } }
+			: {}),
+		[TRACE_RECORDER]: recorder,
+	};
+
+	return traceContext;
+}