@apifuse/provider-sdk 2.0.0-beta.1

package/src/config/providers-yaml.ts

@@ -0,0 +1,370 @@
+import { readFile } from "node:fs/promises";
+
+import { z } from "zod";
+
+import { SDKError, ValidationError } from "../errors";
+
+type YamlObject = Record<string, unknown>;
+
+function stripInlineComment(line: string): string {
+	let inSingleQuote = false;
+	let inDoubleQuote = false;
+
+	for (let index = 0; index < line.length; index += 1) {
+		const char = line[index];
+
+		if (char === "'" && !inDoubleQuote) {
+			inSingleQuote = !inSingleQuote;
+			continue;
+		}
+
+		if (char === '"' && !inSingleQuote) {
+			inDoubleQuote = !inDoubleQuote;
+			continue;
+		}
+
+		if (char === "#" && !inSingleQuote && !inDoubleQuote) {
+			return line.slice(0, index).trimEnd();
+		}
+	}
+
+	return line;
+}
+
+function parseQuotedString(value: string): string {
+	if (
+		(value.startsWith('"') && value.endsWith('"')) ||
+		(value.startsWith("'") && value.endsWith("'"))
+	) {
+		return value.slice(1, -1);
+	}
+
+	return value;
+}
+
+function splitFlowArrayItems(value: string): string[] {
+	const items: string[] = [];
+	let current = "";
+	let inSingleQuote = false;
+	let inDoubleQuote = false;
+
+	for (let index = 0; index < value.length; index += 1) {
+		const char = value[index];
+
+		if (char === "'" && !inDoubleQuote) {
+			inSingleQuote = !inSingleQuote;
+			current += char;
+			continue;
+		}
+
+		if (char === '"' && !inSingleQuote) {
+			inDoubleQuote = !inDoubleQuote;
+			current += char;
+			continue;
+		}
+
+		if (char === "," && !inSingleQuote && !inDoubleQuote) {
+			items.push(current.trim());
+			current = "";
+			continue;
+		}
+
+		current += char;
+	}
+
+	if (current.trim()) {
+		items.push(current.trim());
+	}
+
+	return items;
+}
+
+function parseYamlScalar(value: string): unknown {
+	if (value === "{}") {
+		return {};
+	}
+
+	if (value === "[]") {
+		return [];
+	}
+
+	if (value.startsWith("[") && value.endsWith("]")) {
+		const inner = value.slice(1, -1).trim();
+		if (!inner) {
+			return [];
+		}
+
+		return splitFlowArrayItems(inner).map((item) => parseYamlScalar(item));
+	}
+
+	if (value === "true") {
+		return true;
+	}
+
+	if (value === "false") {
+		return false;
+	}
+
+	if (/^-?\d+$/.test(value)) {
+		return Number.parseInt(value, 10);
+	}
+
+	return parseQuotedString(value);
+}
+
+function parseYamlObject(yamlContent: string): YamlObject {
+	const root: YamlObject = {};
+	const stack: Array<{ container: YamlObject; indent: number }> = [
+		{ container: root, indent: -1 },
+	];
+
+	for (const rawLine of yamlContent.split(/\r?\n/u)) {
+		const withoutComment = stripInlineComment(rawLine);
+		if (!withoutComment.trim()) {
+			continue;
+		}
+
+		const indent = withoutComment.match(/^\s*/u)?.[0].length ?? 0;
+		const line = withoutComment.trim();
+		const separatorIndex = line.indexOf(":");
+
+		if (separatorIndex === -1) {
+			throw new Error(`Invalid YAML line: ${rawLine}`);
+		}
+
+		const key = line.slice(0, separatorIndex).trim();
+		const rawValue = line.slice(separatorIndex + 1).trim();
+
+		while (stack.length > 1 && indent <= stack[stack.length - 1]?.indent) {
+			stack.pop();
+		}
+
+		const parent = stack[stack.length - 1]?.container;
+		if (!parent) {
+			throw new Error(`Could not resolve YAML parent for line: ${rawLine}`);
+		}
+
+		if (!rawValue) {
+			const child: YamlObject = {};
+			parent[key] = child;
+			stack.push({ container: child, indent });
+			continue;
+		}
+
+		parent[key] = parseYamlScalar(rawValue);
+	}
+
+	return root;
+}
+
+const PoolSizeRangeInputSchema = z
+	.string()
+	.regex(/^\d+\.\.\d+$/, "pool-size must use min..max format")
+	.transform((value, ctx) => {
+		const [minRaw, maxRaw] = value.split("..");
+		const min = Number.parseInt(minRaw, 10);
+		const max = Number.parseInt(maxRaw, 10);
+
+		if (min > max) {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				message: "pool-size min must be less than or equal to max",
+			});
+			return z.NEVER;
+		}
+
+		return { max, min };
+	});
+
+const PoolSizeRangeSchema = z.object({
+	max: z.number().int(),
+	min: z.number().int(),
+});
+
+const SecurityOverrideSchema = z
+	.object({
+		"read-only-rootfs": z.boolean().optional(),
+		"no-new-privileges": z.boolean().optional(),
+		egress: z.array(z.string()).optional(),
+		env: z.array(z.string()).optional(),
+	})
+	.passthrough();
+
+const ContainerOverrideSchema = z
+	.object({
+		image: z.string().optional(),
+		memory: z.string().optional(),
+		cpu: z.string().optional(),
+		timeout: z.string().optional(),
+		"restart-policy": z.enum(["on-failure", "always", "never"]).optional(),
+		"max-restarts": z.number().int().optional(),
+	})
+	.passthrough();
+
+const CdpOverrideSchema = z
+	.object({
+		"pool-size": PoolSizeRangeInputSchema,
+		"page-timeout": z.string().optional(),
+		"pages-per-instance": z.number().int().optional(),
+		"idle-timeout": z.string().optional(),
+		"chrome-flags": z.array(z.string()).optional(),
+	})
+	.passthrough();
+
+const DefaultsSchema = z
+	.object({
+		container: ContainerOverrideSchema.optional(),
+		security: SecurityOverrideSchema.optional(),
+	})
+	.passthrough();
+
+const ProviderConfigOverrideSchema = z
+	.object({
+		container: ContainerOverrideSchema.optional(),
+		runtime: z.enum(["standard", "browser"]).optional(),
+		replicas: z.number().int().optional(),
+		security: SecurityOverrideSchema.optional(),
+		cdp: CdpOverrideSchema.optional(),
+	})
+	.passthrough();
+
+export const SecuritySchema = z
+	.object({
+		"read-only-rootfs": z.boolean().default(true),
+		"no-new-privileges": z.boolean().default(true),
+		egress: z.array(z.string()).optional(),
+		env: z.array(z.string()).optional(),
+	})
+	.passthrough();
+
+export const ContainerSchema = z
+	.object({
+		image: z.string().default("auto"),
+		memory: z.string().default("256Mi"),
+		cpu: z.string().default("250m"),
+		timeout: z.string().default("30s"),
+		"restart-policy": z
+			.enum(["on-failure", "always", "never"])
+			.default("on-failure"),
+		"max-restarts": z.number().int().default(3),
+	})
+	.passthrough();
+
+export const CdpSchema = z
+	.object({
+		"pool-size": PoolSizeRangeSchema,
+		"page-timeout": z.string().default("120s"),
+		"pages-per-instance": z.number().int().default(4),
+		"idle-timeout": z.string().default("5m"),
+		"chrome-flags": z.array(z.string()).optional(),
+	})
+	.passthrough();
+
+export const ProviderConfigSchema = z
+	.object({
+		container: ContainerSchema,
+		runtime: z.enum(["standard", "browser"]).default("standard"),
+		replicas: z.number().int().default(1),
+		security: SecuritySchema,
+		cdp: CdpSchema.optional(),
+	})
+	.passthrough();
+
+export const ProvidersYamlSchema = z
+	.object({
+		defaults: DefaultsSchema.optional(),
+		providers: z.record(z.string(), ProviderConfigOverrideSchema),
+	})
+	.passthrough();
+
+export type PoolSizeRange = z.infer<typeof PoolSizeRangeSchema>;
+export type ProvidersYaml = z.infer<typeof ProvidersYamlSchema>;
+export type ProviderConfig = z.infer<typeof ProviderConfigSchema>;
+
+function toValidationError(error: unknown): ValidationError {
+	if (error instanceof z.ZodError) {
+		return new ValidationError("Invalid providers.yaml configuration", {
+			code: "INVALID_CONNECTORS_YAML",
+			fix: "Match the providers.yaml schema from openspec/provider-sdk/08-infrastructure.md §8.2.",
+			zodError: error,
+		});
+	}
+
+	return new ValidationError("Invalid providers.yaml configuration", {
+		code: "INVALID_CONNECTORS_YAML",
+		cause: error instanceof Error ? error : undefined,
+		fix: "Check providers.yaml syntax and field values.",
+	});
+}
+
+export function parseProvidersYaml(yamlContent: string): ProvidersYaml {
+	try {
+		const parsed = parseYamlObject(yamlContent);
+		return ProvidersYamlSchema.parse(parsed);
+	} catch (error) {
+		if (error instanceof z.ZodError) {
+			throw toValidationError(error);
+		}
+
+		throw new SDKError("Failed to parse providers.yaml", {
+			code: "CONNECTORS_YAML_PARSE_FAILED",
+			cause: error instanceof Error ? error : undefined,
+			fix: "Ensure providers.yaml contains valid YAML.",
+		});
+	}
+}
+
+export async function loadProvidersYaml(
+	filePath: string,
+): Promise<ProvidersYaml> {
+	try {
+		const yamlContent = await readFile(filePath, "utf8");
+		return parseProvidersYaml(yamlContent);
+	} catch (error) {
+		if (error instanceof SDKError || error instanceof ValidationError) {
+			throw error;
+		}
+
+		throw new SDKError(`Failed to load providers.yaml from ${filePath}`, {
+			code: "CONNECTORS_YAML_LOAD_FAILED",
+			cause: error instanceof Error ? error : undefined,
+			fix: "Verify the file exists and is readable.",
+		});
+	}
+}
+
+export function resolveProviderConfig(
+	yaml: ProvidersYaml,
+	providerId: string,
+): ProviderConfig {
+	const providerOverride = yaml.providers[providerId];
+
+	if (!providerOverride) {
+		throw new SDKError(`Provider "${providerId}" not found in providers.yaml`, {
+			code: "CONNECTOR_NOT_FOUND",
+			fix: `Add ${providerId} under providers: in providers.yaml.`,
+		});
+	}
+
+	const container = ContainerSchema.parse({
+		...yaml.defaults?.container,
+		...providerOverride.container,
+	});
+
+	const security = SecuritySchema.parse({
+		...yaml.defaults?.security,
+		...providerOverride.security,
+	});
+
+	const resolved = {
+		container,
+		security,
+		runtime: providerOverride.runtime ?? "standard",
+		replicas: providerOverride.replicas ?? 1,
+		cdp: providerOverride.cdp
+			? CdpSchema.parse(providerOverride.cdp)
+			: undefined,
+	};
+
+	return ProviderConfigSchema.parse(resolved);
+}

package/src/define.ts

@@ -0,0 +1,137 @@
+import type { ZodType } from "zod";
+import { ProviderError, ValidationError } from "./errors";
+import type {
+	AuthConfig,
+	BrowserEngine,
+	OperationDefinition,
+	ProviderDefinition,
+	StealthPlatform,
+} from "./types";
+
+const CONNECTOR_ID_REGEX = /^[a-z][a-z0-9]*(-[a-z][a-z0-9]*)*$/;
+
+type ProviderOperation = OperationDefinition<ZodType, ZodType>;
+
+export interface ProviderConfig<
+	TOperations extends Record<string, ProviderOperation>,
+> {
+	id: string;
+	version: string;
+	runtime: "standard" | "browser";
+	stealth?: {
+		profile: string;
+		platform: StealthPlatform;
+	};
+	proxy?: boolean;
+	browser?: {
+		engine: BrowserEngine;
+	};
+	auth?: AuthConfig;
+	meta: {
+		displayName: string;
+		description?: string;
+		category: string;
+		tags?: string[];
+		icon?: string;
+	};
+	operations: TOperations;
+}
+
+function validateOperationFixtures(
+	providerId: string,
+	operations: Record<string, ProviderOperation>,
+): void {
+	for (const [operationName, operation] of Object.entries(operations)) {
+		if (typeof operation.handler !== "function") {
+			throw new ValidationError(
+				`Operation handler must be defined for provider "${providerId}" operation "${operationName}"`,
+				{
+					fix: `Add operations.${operationName}.handler as an async function with signature (ctx, input) => Promise<output>`,
+				},
+			);
+		}
+
+		if (operation.fixtures?.request !== undefined) {
+			const result = operation.input.safeParse(operation.fixtures.request);
+
+			if (!result.success) {
+				throw new ValidationError(
+					`Fixture request does not match input schema for provider "${providerId}" operation "${operationName}"`,
+					{
+						fix: `Update operations.${operationName}.fixtures.request to match operations.${operationName}.input`,
+						zodError: result.error,
+					},
+				);
+			}
+		}
+
+		if (operation.fixtures?.response !== undefined) {
+			const result = operation.output.safeParse(operation.fixtures.response);
+
+			if (!result.success) {
+				throw new ValidationError(
+					`Fixture response does not match output schema for provider "${providerId}" operation "${operationName}"`,
+					{
+						fix: `Update operations.${operationName}.fixtures.response to match operations.${operationName}.output`,
+						zodError: result.error,
+					},
+				);
+			}
+		}
+	}
+}
+
+export function defineProvider<
+	TOperations extends Record<string, ProviderOperation>,
+>(
+	config: ProviderConfig<TOperations>,
+): ProviderDefinition & { operations: TOperations } {
+	if (!CONNECTOR_ID_REGEX.test(config.id)) {
+		throw new ProviderError(`Invalid provider id: "${config.id}"`, {
+			fix: 'Use lowercase alphanumeric with dashes, e.g., "coingecko-prices"',
+		});
+	}
+
+	if (Object.keys(config.operations).length === 0) {
+		throw new ProviderError(
+			`Provider "${config.id}" must define at least one operation`,
+			{ fix: "Add at least one operation to the operations object" },
+		);
+	}
+
+	validateOperationFixtures(config.id, config.operations);
+
+	if (config.runtime === "browser" && !config.browser) {
+		throw new ProviderError(
+			`Provider "${config.id}" must define browser.engine when runtime is "browser"`,
+			{
+				fix: 'Add browser: { engine: "nodriver" } or another supported engine',
+			},
+		);
+	}
+
+	if (config.browser && config.runtime !== "browser") {
+		throw new ProviderError(
+			`Provider "${config.id}" cannot define browser config unless runtime is "browser"`,
+			{ fix: 'Set runtime: "browser" or remove the browser config' },
+		);
+	}
+
+	if (config.proxy && !config.stealth) {
+		console.warn(
+			`[provider-sdk] Provider "${config.id}" enables proxy without a stealth profile.`,
+		);
+	}
+
+	return {
+		id: config.id,
+		version: config.version,
+		runtime: config.runtime,
+		stealth: config.stealth,
+		proxy: config.proxy,
+		browser: config.browser,
+		auth: config.auth,
+		meta: config.meta,
+		operations: config.operations,
+	};
+}

package/src/dev.ts

@@ -0,0 +1,38 @@
+import { createProviderServer } from "./serve";
+import type { ProviderDefinition } from "./types";
+
+export interface DevServerOptions {
+	port?: number;
+	sessionDbPath?: string;
+}
+
+export function createDevServer(
+	provider: ProviderDefinition,
+	options?: DevServerOptions,
+): { start: () => void } {
+	const port = options?.port ?? 3900;
+	const server = createProviderServer(provider, {
+		port,
+		sessionDbPath: options?.sessionDbPath,
+	});
+
+	return {
+		start: () => {
+			server.start();
+			console.log(
+				`[apifuse dev] ${provider.id}@${provider.version} running at http://localhost:${port}`,
+			);
+			console.log(
+				`[apifuse dev] Operations: ${Object.keys(provider.operations).join(", ")}`,
+			);
+			console.log(`[apifuse dev] Health: http://localhost:${port}/health`);
+		},
+	};
+}
+
+export function startDevServer(
+	provider: ProviderDefinition,
+	options?: DevServerOptions,
+): void {
+	createDevServer(provider, options).start();
+}

package/src/errors.ts

@@ -0,0 +1,68 @@
+export type ProviderErrorOptions = {
+	fix?: string;
+	code?: string;
+	cause?: Error;
+};
+
+export class ProviderError extends Error {
+	constructor(
+		message: string,
+		public readonly options?: ProviderErrorOptions,
+	) {
+		super(message);
+		this.name = "ProviderError";
+		if (options?.cause) {
+			this.cause = options.cause;
+		}
+	}
+
+	get fix(): string | undefined {
+		return this.options?.fix;
+	}
+
+	get code(): string | undefined {
+		return this.options?.code;
+	}
+}
+
+export class SDKError extends ProviderError {
+	constructor(message: string, options?: ProviderErrorOptions) {
+		super(message, options);
+		this.name = "SDKError";
+	}
+}
+
+export class AuthError extends ProviderError {
+	constructor(message: string, options?: ProviderErrorOptions) {
+		super(message, options);
+		this.name = "AuthError";
+	}
+}
+
+export type ValidationErrorOptions = ProviderErrorOptions & {
+	zodError?: unknown;
+};
+
+export class ValidationError extends ProviderError {
+	readonly zodError?: unknown;
+
+	constructor(message: string, options?: ValidationErrorOptions) {
+		super(message, options);
+		this.name = "ValidationError";
+		this.zodError = options?.zodError;
+	}
+}
+
+export type TransportErrorOptions = ProviderErrorOptions & {
+	status?: number;
+};
+
+export class TransportError extends ProviderError {
+	readonly status?: number;
+
+	constructor(message: string, options?: TransportErrorOptions) {
+		super(message, options);
+		this.name = "TransportError";
+		this.status = options?.status;
+	}
+}

package/src/index.test.ts

@@ -0,0 +1 @@
+// placeholder to keep bun test from erroring on zero test files

package/src/index.ts

@@ -0,0 +1,100 @@
+// @apifuse/provider-sdk
+
+export { z } from "zod";
+export type {
+	ApiFuseConfig,
+	BrowserConfig,
+	ProxyConfig,
+	SessionConfig,
+} from "./config/loader";
+export { defineConfig, loadApiFuseConfig } from "./config/loader";
+export type {
+	PoolSizeRange,
+	ProviderConfig,
+	ProvidersYaml,
+} from "./config/providers-yaml";
+export {
+	CdpSchema,
+	ContainerSchema,
+	loadProvidersYaml,
+	ProviderConfigSchema,
+	ProvidersYamlSchema,
+	parseProvidersYaml,
+	resolveProviderConfig,
+	SecuritySchema,
+} from "./config/providers-yaml";
+export { defineProvider } from "./define";
+export type { DevServerOptions } from "./dev";
+export { createDevServer, startDevServer } from "./dev";
+export * from "./errors";
+export * from "./protocol";
+export * from "./recipes/gov-api";
+export * from "./recipes/rest-api";
+export type { AuthManager } from "./runtime/auth";
+export { createAuthManager } from "./runtime/auth";
+export type { BrowserClientOptions } from "./runtime/browser";
+export { BrowserClient, createBrowserClient } from "./runtime/browser";
+export { executeOperation } from "./runtime/executor";
+export { createHttpClient } from "./runtime/http";
+export type { Insight, InsightSeverity } from "./runtime/insights";
+export { generateInsights } from "./runtime/insights";
+export {
+	type InstrumentationOptions,
+	type InstrumentedProviderContext,
+	wrapWithInstrumentation,
+} from "./runtime/instrumentation";
+export { getProviderBaseUrl } from "./runtime/provider";
+export type {
+	SessionFetch,
+	SupabaseSessionStoreConfig,
+} from "./runtime/session";
+export {
+	createSessionStore,
+	createSqliteSessionStore,
+	SupabaseSessionStore,
+} from "./runtime/session";
+export { createStateContext } from "./runtime/state";
+export { createTlsClient } from "./runtime/tls";
+export {
+	type CreateTraceContextOptions,
+	createTraceContext,
+	type Span,
+	type TraceContext,
+} from "./runtime/trace";
+export { createProviderServer } from "./serve";
+export { getStealthProfile, listStealthProfiles } from "./stealth/profiles";
+export {
+	runStandardTests,
+	snapshotTransform,
+	toMatchShape,
+} from "./testing";
+export type {
+	ApiFuseResponse,
+	AuthConfig,
+	AuthContext,
+	AuthField,
+	AuthMode,
+	BrowserEngine,
+	BrowserOptions,
+	HttpClient,
+	HttpResponse,
+	OperationDefinition,
+	ProviderContext,
+	ProviderDefinition,
+	ProviderMeta,
+	RequestOptions,
+	SessionStore,
+	StateContext,
+	StealthPlatform,
+	StealthProfile,
+	TlsClient,
+	TlsFetchOptions,
+	TlsResponse,
+	TlsSession,
+	TraceConfig,
+	TraceSpan,
+} from "./types";
+export * from "./utils/date";
+export * from "./utils/parse";
+export * from "./utils/text";
+export * from "./utils/transform";