@apifuse/connector-sdk 2.0.0-beta.1

package/src/config/loader.ts

@@ -0,0 +1,122 @@
+import { existsSync } from "node:fs";
+import path from "node:path";
+
+import type { TraceConfig } from "../types";
+
+export type ProxyOptions = {
+	url: string;
+};
+
+export type ProxyConfig = Partial<ProxyOptions> & {
+	provider?: string;
+	apiKey?: string;
+};
+
+export type BrowserConfig = {
+	executablePath?: string;
+	headless?: boolean;
+};
+
+export type SessionConfig = {
+	storage?: "sqlite" | "supabase";
+	path?: string;
+};
+
+export type ApiFuseConfig = {
+	proxy?: ProxyConfig;
+	browser?: BrowserConfig;
+	session?: SessionConfig;
+	trace?: TraceConfig;
+	credentials?: Record<string, Record<string, string>>;
+};
+
+export type ProxyResolutionOptions = {
+	proxy?: string;
+	upstream?: { proxy?: boolean };
+	apifuseConfig?: Pick<ApiFuseConfig, "proxy">;
+};
+
+export type ResolvedProxyConfig = {
+	shouldWarn: boolean;
+	url?: string;
+};
+
+function normalizeProxyUrl(url?: string): string | undefined {
+	const normalized = url?.trim();
+	return normalized ? normalized : undefined;
+}
+
+function syncProxyEnv(config: ApiFuseConfig): void {
+	const configProxyUrl = normalizeProxyUrl(config.proxy?.url);
+	if (!process.env.APIFUSE_PROXY_URL && configProxyUrl) {
+		process.env.APIFUSE_PROXY_URL = configProxyUrl;
+	}
+}
+
+export function resolveProxyConfig(
+	options: ProxyResolutionOptions = {},
+): ResolvedProxyConfig {
+	const explicitProxyUrl = normalizeProxyUrl(options.proxy);
+	if (explicitProxyUrl) {
+		return { shouldWarn: false, url: explicitProxyUrl };
+	}
+
+	if (!options.upstream?.proxy) {
+		return { shouldWarn: false };
+	}
+
+	const envProxyUrl = normalizeProxyUrl(process.env.APIFUSE_PROXY_URL);
+	if (envProxyUrl) {
+		return { shouldWarn: false, url: envProxyUrl };
+	}
+
+	const configuredProxyUrl = normalizeProxyUrl(
+		options.apifuseConfig?.proxy?.url,
+	);
+	if (configuredProxyUrl) {
+		return { shouldWarn: false, url: configuredProxyUrl };
+	}
+
+	return { shouldWarn: true };
+}
+
+export function defineConfig(config: ApiFuseConfig): ApiFuseConfig {
+	return config;
+}
+
+async function importConfig(filePath: string): Promise<ApiFuseConfig | null> {
+	try {
+		const moduleUrl = new URL(`file://${encodeURI(filePath)}`);
+		const mod = (await import(moduleUrl.href)) as { default?: ApiFuseConfig };
+		if (mod.default && typeof mod.default === "object") {
+			return mod.default;
+		}
+		console.warn(`[connector-sdk] Ignoring invalid config export: ${filePath}`);
+		return {};
+	} catch (error) {
+		console.warn(`[connector-sdk] Failed to load config ${filePath}:`, error);
+		return null;
+	}
+}
+
+export async function loadApiFuseConfig(
+	dir: string = process.cwd(),
+): Promise<ApiFuseConfig> {
+	const tsPath = path.resolve(dir, "apifuse.config.ts");
+	if (existsSync(tsPath)) {
+		const config = await importConfig(tsPath);
+		const resolvedConfig = config ?? {};
+		syncProxyEnv(resolvedConfig);
+		return resolvedConfig;
+	}
+
+	const jsPath = path.resolve(dir, "apifuse.config.js");
+	if (existsSync(jsPath)) {
+		const config = await importConfig(jsPath);
+		const resolvedConfig = config ?? {};
+		syncProxyEnv(resolvedConfig);
+		return resolvedConfig;
+	}
+
+	return {};
+}

package/src/define.ts

@@ -0,0 +1,137 @@
+import type { ZodType } from "zod";
+import { ConnectorError, ValidationError } from "./errors";
+import type {
+	AuthConfig,
+	BrowserEngine,
+	ConnectorDefinition,
+	OperationDefinition,
+	StealthPlatform,
+} from "./types";
+
+const CONNECTOR_ID_REGEX = /^[a-z][a-z0-9]*(-[a-z][a-z0-9]*)*$/;
+
+type ConnectorOperation = OperationDefinition<ZodType, ZodType>;
+
+export interface ConnectorConfig<
+	TOperations extends Record<string, ConnectorOperation>,
+> {
+	id: string;
+	version: string;
+	runtime: "standard" | "browser";
+	stealth?: {
+		profile: string;
+		platform: StealthPlatform;
+	};
+	proxy?: boolean;
+	browser?: {
+		engine: BrowserEngine;
+	};
+	auth?: AuthConfig;
+	meta: {
+		displayName: string;
+		description?: string;
+		category: string;
+		tags?: string[];
+		icon?: string;
+	};
+	operations: TOperations;
+}
+
+function validateOperationFixtures(
+	connectorId: string,
+	operations: Record<string, ConnectorOperation>,
+): void {
+	for (const [operationName, operation] of Object.entries(operations)) {
+		if (typeof operation.handler !== "function") {
+			throw new ValidationError(
+				`Operation handler must be defined for connector "${connectorId}" operation "${operationName}"`,
+				{
+					fix: `Add operations.${operationName}.handler as an async function with signature (ctx, input) => Promise<output>`,
+				},
+			);
+		}
+
+		if (operation.fixtures?.request !== undefined) {
+			const result = operation.input.safeParse(operation.fixtures.request);
+
+			if (!result.success) {
+				throw new ValidationError(
+					`Fixture request does not match input schema for connector "${connectorId}" operation "${operationName}"`,
+					{
+						fix: `Update operations.${operationName}.fixtures.request to match operations.${operationName}.input`,
+						zodError: result.error,
+					},
+				);
+			}
+		}
+
+		if (operation.fixtures?.response !== undefined) {
+			const result = operation.output.safeParse(operation.fixtures.response);
+
+			if (!result.success) {
+				throw new ValidationError(
+					`Fixture response does not match output schema for connector "${connectorId}" operation "${operationName}"`,
+					{
+						fix: `Update operations.${operationName}.fixtures.response to match operations.${operationName}.output`,
+						zodError: result.error,
+					},
+				);
+			}
+		}
+	}
+}
+
+export function defineConnector<
+	TOperations extends Record<string, ConnectorOperation>,
+>(
+	config: ConnectorConfig<TOperations>,
+): ConnectorDefinition & { operations: TOperations } {
+	if (!CONNECTOR_ID_REGEX.test(config.id)) {
+		throw new ConnectorError(`Invalid connector id: "${config.id}"`, {
+			fix: 'Use lowercase alphanumeric with dashes, e.g., "coingecko-prices"',
+		});
+	}
+
+	if (Object.keys(config.operations).length === 0) {
+		throw new ConnectorError(
+			`Connector "${config.id}" must define at least one operation`,
+			{ fix: "Add at least one operation to the operations object" },
+		);
+	}
+
+	validateOperationFixtures(config.id, config.operations);
+
+	if (config.runtime === "browser" && !config.browser) {
+		throw new ConnectorError(
+			`Connector "${config.id}" must define browser.engine when runtime is "browser"`,
+			{
+				fix: 'Add browser: { engine: "nodriver" } or another supported engine',
+			},
+		);
+	}
+
+	if (config.browser && config.runtime !== "browser") {
+		throw new ConnectorError(
+			`Connector "${config.id}" cannot define browser config unless runtime is "browser"`,
+			{ fix: 'Set runtime: "browser" or remove the browser config' },
+		);
+	}
+
+	if (config.proxy && !config.stealth) {
+		console.warn(
+			`[connector-sdk] Connector "${config.id}" enables proxy without a stealth profile.`,
+		);
+	}
+
+	return {
+		id: config.id,
+		version: config.version,
+		runtime: config.runtime,
+		stealth: config.stealth,
+		proxy: config.proxy,
+		browser: config.browser,
+		auth: config.auth,
+		meta: config.meta,
+		operations: config.operations,
+	};
+}

package/src/dev.ts

@@ -0,0 +1,38 @@
+import { createConnectorServer } from "./serve";
+import type { ConnectorDefinition } from "./types";
+
+export interface DevServerOptions {
+	port?: number;
+	sessionDbPath?: string;
+}
+
+export function createDevServer(
+	connector: ConnectorDefinition,
+	options?: DevServerOptions,
+): { start: () => void } {
+	const port = options?.port ?? 3900;
+	const server = createConnectorServer(connector, {
+		port,
+		sessionDbPath: options?.sessionDbPath,
+	});
+
+	return {
+		start: () => {
+			server.start();
+			console.log(
+				`[apifuse dev] ${connector.id}@${connector.version} running at http://localhost:${port}`,
+			);
+			console.log(
+				`[apifuse dev] Operations: ${Object.keys(connector.operations).join(", ")}`,
+			);
+			console.log(`[apifuse dev] Health: http://localhost:${port}/health`);
+		},
+	};
+}
+
+export function startDevServer(
+	connector: ConnectorDefinition,
+	options?: DevServerOptions,
+): void {
+	createDevServer(connector, options).start();
+}

package/src/errors.ts

@@ -0,0 +1,68 @@
+export type ConnectorErrorOptions = {
+	fix?: string;
+	code?: string;
+	cause?: Error;
+};
+
+export class ConnectorError extends Error {
+	constructor(
+		message: string,
+		public readonly options?: ConnectorErrorOptions,
+	) {
+		super(message);
+		this.name = "ConnectorError";
+		if (options?.cause) {
+			this.cause = options.cause;
+		}
+	}
+
+	get fix(): string | undefined {
+		return this.options?.fix;
+	}
+
+	get code(): string | undefined {
+		return this.options?.code;
+	}
+}
+
+export class SDKError extends ConnectorError {
+	constructor(message: string, options?: ConnectorErrorOptions) {
+		super(message, options);
+		this.name = "SDKError";
+	}
+}
+
+export class AuthError extends ConnectorError {
+	constructor(message: string, options?: ConnectorErrorOptions) {
+		super(message, options);
+		this.name = "AuthError";
+	}
+}
+
+export type ValidationErrorOptions = ConnectorErrorOptions & {
+	zodError?: unknown;
+};
+
+export class ValidationError extends ConnectorError {
+	readonly zodError?: unknown;
+
+	constructor(message: string, options?: ValidationErrorOptions) {
+		super(message, options);
+		this.name = "ValidationError";
+		this.zodError = options?.zodError;
+	}
+}
+
+export type TransportErrorOptions = ConnectorErrorOptions & {
+	status?: number;
+};
+
+export class TransportError extends ConnectorError {
+	readonly status?: number;
+
+	constructor(message: string, options?: TransportErrorOptions) {
+		super(message, options);
+		this.name = "TransportError";
+		this.status = options?.status;
+	}
+}

package/src/index.test.ts

@@ -0,0 +1 @@
+// placeholder to keep bun test from erroring on zero test files

package/src/index.ts

@@ -0,0 +1,100 @@
+// @apifuse/connector-sdk
+
+export { z } from "zod";
+export type {
+	ConnectorConfig,
+	ConnectorsYaml,
+	PoolSizeRange,
+} from "./config/connectors-yaml";
+export {
+	CdpSchema,
+	ConnectorConfigSchema,
+	ConnectorsYamlSchema,
+	ContainerSchema,
+	loadConnectorsYaml,
+	parseConnectorsYaml,
+	resolveConnectorConfig,
+	SecuritySchema,
+} from "./config/connectors-yaml";
+export type {
+	ApiFuseConfig,
+	BrowserConfig,
+	ProxyConfig,
+	SessionConfig,
+} from "./config/loader";
+export { defineConfig, loadApiFuseConfig } from "./config/loader";
+export { defineConnector } from "./define";
+export type { DevServerOptions } from "./dev";
+export { createDevServer, startDevServer } from "./dev";
+export * from "./errors";
+export * from "./protocol";
+export * from "./recipes/gov-api";
+export * from "./recipes/rest-api";
+export type { AuthManager } from "./runtime/auth";
+export { createAuthManager } from "./runtime/auth";
+export type { BrowserClientOptions } from "./runtime/browser";
+export { BrowserClient, createBrowserClient } from "./runtime/browser";
+export { getConnectorBaseUrl } from "./runtime/connector";
+export { executeOperation } from "./runtime/executor";
+export { createHttpClient } from "./runtime/http";
+export type { Insight, InsightSeverity } from "./runtime/insights";
+export { generateInsights } from "./runtime/insights";
+export {
+	type InstrumentationOptions,
+	type InstrumentedConnectorContext,
+	wrapWithInstrumentation,
+} from "./runtime/instrumentation";
+export type {
+	SessionFetch,
+	SupabaseSessionStoreConfig,
+} from "./runtime/session";
+export {
+	createSessionStore,
+	createSqliteSessionStore,
+	SupabaseSessionStore,
+} from "./runtime/session";
+export { createStateContext } from "./runtime/state";
+export { createTlsClient } from "./runtime/tls";
+export {
+	type CreateTraceContextOptions,
+	createTraceContext,
+	type Span,
+	type TraceContext,
+} from "./runtime/trace";
+export { createConnectorServer } from "./serve";
+export { getStealthProfile, listStealthProfiles } from "./stealth/profiles";
+export {
+	runStandardTests,
+	snapshotTransform,
+	toMatchShape,
+} from "./testing";
+export type {
+	ApiFuseResponse,
+	AuthConfig,
+	AuthContext,
+	AuthField,
+	AuthMode,
+	BrowserEngine,
+	BrowserOptions,
+	ConnectorContext,
+	ConnectorDefinition,
+	ConnectorMeta,
+	HttpClient,
+	HttpResponse,
+	OperationDefinition,
+	RequestOptions,
+	SessionStore,
+	StateContext,
+	StealthPlatform,
+	StealthProfile,
+	TlsClient,
+	TlsFetchOptions,
+	TlsResponse,
+	TlsSession,
+	TraceConfig,
+	TraceSpan,
+} from "./types";
+export * from "./utils/date";
+export * from "./utils/parse";
+export * from "./utils/text";
+export * from "./utils/transform";

package/src/protocol.ts

@@ -0,0 +1,183 @@
+/**
+ * Container Protocol — Communication contract between Route Server and Connector Containers.
+ *
+ * All connector containers expose HTTP endpoints. Route Server sends requests
+ * and receives responses in these formats.
+ */
+
+// ═══════════════════════════════════════════════════════════════
+// Execute Operation
+// POST /execute/:operationId
+// ═══════════════════════════════════════════════════════════════
+
+export interface ExecuteRequest {
+	/** Validated input (already passed Zod validation by Route Server) */
+	input: unknown;
+	/** Decrypted session key-value pairs (loaded by Route Server from DB) */
+	session?: Record<string, string>;
+	/** Trace context for distributed tracing */
+	trace?: { traceId: string; spanId: string };
+}
+
+export interface ExecuteResponse {
+	/** Operation result (will be Zod-validated by Route Server) */
+	data: unknown;
+	/** Session mutations to persist back to DB (key→value for set, key→null for delete) */
+	sessionPatch?: Record<string, string | null>;
+	/** Trace spans collected during execution */
+	trace?: { spans: unknown[] };
+}
+
+// ═══════════════════════════════════════════════════════════════
+// Connect (Auth Exchange)
+// POST /connect
+// ═══════════════════════════════════════════════════════════════
+
+export interface ConnectRequest {
+	/** User-provided credentials */
+	credentials: Record<string, string>;
+	/** Existing session (for re-auth scenarios) */
+	session?: Record<string, string>;
+}
+
+/** Successful connection */
+export interface ConnectSuccessResponse {
+	status: "success";
+	/** Display name for the connected upstream account */
+	accountRef?: string;
+	/** Session data to persist (full session state after exchange) */
+	sessionPatch: Record<string, string>;
+}
+
+/** Connection requires additional field (e.g., OTP) */
+export interface ConnectPendingFieldResponse {
+	status: "pending_field";
+	/** Name of the field to request from user */
+	field: string;
+	/** Label to display to user */
+	fieldLabel?: string;
+	/** Opaque token to resume the connection flow */
+	resumeToken: string;
+	/** Partial session state to persist (intermediate state) */
+	sessionPatch?: Record<string, string>;
+}
+
+/** Connection failed */
+export interface ConnectFailedResponse {
+	status: "failed";
+	/** Error code for programmatic handling */
+	error: string;
+	/** Human-readable error message */
+	message?: string;
+}
+
+export type ConnectResponse =
+	| ConnectSuccessResponse
+	| ConnectPendingFieldResponse
+	| ConnectFailedResponse;
+
+// ═══════════════════════════════════════════════════════════════
+// Resume (Continue deferred auth)
+// POST /connect/resume
+// ═══════════════════════════════════════════════════════════════
+
+export interface ResumeRequest {
+	/** Resume token from ConnectPendingFieldResponse */
+	resumeToken: string;
+	/** User-provided value for the requested field */
+	fieldValue: string;
+	/** Current session state */
+	session?: Record<string, string>;
+}
+
+export type ResumeResponse = ConnectResponse; // same as ConnectResponse
+
+// ═══════════════════════════════════════════════════════════════
+// Refresh (Auth token refresh)
+// POST /refresh
+// ═══════════════════════════════════════════════════════════════
+
+export interface RefreshRequest {
+	/** Current session to refresh */
+	session: Record<string, string>;
+}
+
+export interface RefreshSuccessResponse {
+	status: "success";
+	/** Updated session data */
+	sessionPatch: Record<string, string>;
+}
+
+export interface RefreshFailedResponse {
+	status: "failed";
+	error: string;
+	message?: string;
+}
+
+export type RefreshResponse = RefreshSuccessResponse | RefreshFailedResponse;
+
+// ═══════════════════════════════════════════════════════════════
+// Disconnect
+// POST /disconnect
+// ═══════════════════════════════════════════════════════════════
+
+export interface DisconnectRequest {
+	/** Session to clean up */
+	session: Record<string, string>;
+}
+
+export interface DisconnectResponse {
+	status: "success" | "failed";
+	error?: string;
+}
+
+// ═══════════════════════════════════════════════════════════════
+// Health
+// GET /health
+// ═══════════════════════════════════════════════════════════════
+
+export interface HealthResponse {
+	status: "ok" | "degraded" | "error";
+	connector: string;
+	version: string;
+	uptime?: number;
+}
+
+// ═══════════════════════════════════════════════════════════════
+// Schema
+// GET /schema/:operationId
+// ═══════════════════════════════════════════════════════════════
+
+export interface SchemaResponse {
+	operationId: string;
+	description?: string;
+	input: Record<string, unknown>; // JSON Schema
+	output: Record<string, unknown>; // JSON Schema
+	hints?: Record<string, string>;
+}
+
+// ═══════════════════════════════════════════════════════════════
+// Error envelope (used for any error response)
+// ═══════════════════════════════════════════════════════════════
+
+export interface ContainerErrorResponse {
+	error: {
+		code: string;
+		message: string;
+		details?: unknown;
+	};
+}
+
+// ═══════════════════════════════════════════════════════════════
+// Standard endpoint paths
+// ═══════════════════════════════════════════════════════════════
+
+export const CONTAINER_ENDPOINTS = {
+	execute: "/execute/:operationId",
+	connect: "/connect",
+	resume: "/connect/resume",
+	refresh: "/refresh",
+	disconnect: "/disconnect",
+	health: "/health",
+	schema: "/schema/:operationId",
+} as const;