@apifuse/provider-sdk 2.2.0-beta.11 → 2.2.0-beta.13

package/dist/types.d.ts

@@ -1,4 +1,5 @@
 import type ms from "ms";
+import type { SerializedCookieJar } from "tough-cookie";
 import type { infer as ZodInfer, ZodType } from "zod";
 /** Minimal Standard Schema v1 shape accepted by provider operations. */
 export interface StandardSchemaV1<Input = unknown, Output = Input> {
@@ -698,6 +699,12 @@ export interface ProviderProxyPolicy {
         affinity?: ProviderProxySessionAffinity;
         lifetimeMinutes?: number;
         poolSize?: number;
+        /**
+         * Seconds before hard sticky expiry at which native connections receive
+         * the `expiring` event so the provider can drain and reconnect cleanly.
+         * Declared by the provider; the SDK does not assume a default cut point.
+         */
+        drainLeadSeconds?: number;
     };
 }
 export type ProviderProxyConfig = boolean | ProviderProxyPolicy;
@@ -839,6 +846,11 @@ export interface HttpRetrySummary {
 export interface RequestOptions {
     headers?: Record<string, string>;
     params?: RequestParams;
+    /**
+     * Query parameters whose values contain credentials or other secret material.
+     * They are sent like `params`, but redacted from SDK errors, traces, and recorded fixtures.
+     */
+    sensitiveParams?: Record<string, string>;
     proxy?: string;
     timeout?: number;
     /**
@@ -853,6 +865,12 @@ export interface StealthFetchOptions extends RequestOptions {
     method?: HttpMethod;
     body?: string | Buffer;
     redirect?: "follow" | "manual" | "error";
+    /**
+     * Maximum decoded response-body bytes to buffer. When set, the stealth
+     * transport aborts the response and throws `response_too_large` if the
+     * declared or streamed body exceeds this limit.
+     */
+    maxBodyBytes?: number;
     /**
      * Offsets policy-managed proxy pool selection for caller-managed retries.
      * Use when a request receives an upstream challenge page rather than a
@@ -872,17 +890,42 @@ export interface StealthFetchOptions extends RequestOptions {
     };
 }
 export interface CookieJar {
-    get(name: string): string | undefined;
-    getAll(): Record<string, string>;
-    toString(): string;
-    find?(predicate: (cookie: string) => boolean): string | undefined;
+    /** URL-less reads use the jar's response URL or session base URL. */
+    get(name: string, url?: string): string | undefined;
+    getAll(url?: string): Record<string, string>;
+    toString(url?: string): string;
+    find?(predicate: (cookie: string) => boolean, url?: string): string | undefined;
+}
+/**
+ * Version 1 of the JSON-safe, attribute-preserving stealth cookie store.
+ * The nested jar is tough-cookie's serialized form and retains cookie origin,
+ * Path, Secure, expiry, host-only, and other RFC attributes.
+ */
+export interface StealthCookieStoreV1 {
+    readonly version: 1;
+    readonly jar: SerializedCookieJar;
 }
+/** Cookie persistence formats understood by this SDK version. */
+export type StealthCookieStore = StealthCookieStoreV1;
 export interface StealthSessionCookies extends CookieJar {
-    has(name: string): boolean;
-    setFromCookieStrings(cookieStrings: readonly string[]): void;
-    toHeader(): string;
+    has(name: string, url?: string): boolean;
+    /** URL-less writes are scoped to the session base URL. */
+    setFromCookieStrings(cookieStrings: readonly string[], url?: string): void;
+    toHeader(url?: string): string;
+    /**
+     * Returns every cookie as a flat name/value map, collapsing duplicate names.
+     * @deprecated Use serialize() for lossless, attribute-preserving persistence.
+     */
     snapshot(): Record<string, string>;
+    /**
+     * Restores flat values as host-only, Path=/ cookies on the session base URL.
+     * @deprecated Use deserialize() with state produced by serialize().
+     */
     restore(cookies: Record<string, string>): void;
+    /** Returns a versioned, JSON-safe, attribute-preserving representation of every cookie. */
+    serialize(): StealthCookieStoreV1;
+    /** Replaces the jar with a previously serialized, attribute-preserving cookie store. */
+    deserialize(state: StealthCookieStore): void;
     clear(): void;
 }
 export interface DeclarativeStealthResponse {
@@ -925,7 +968,13 @@ export interface StealthRedirectRunResult {
     final: StealthResponse;
     hops: StealthRedirectHop[];
     reason: "completed" | "stopped" | "max_hops" | "missing_location" | "loop";
+    /**
+     * Complete flat view across all redirect hosts. Attributes and duplicate names are lost.
+     * @deprecated Use cookieStore for lossless persistence.
+     */
     cookies: Record<string, string>;
+    /** Versioned, attribute-preserving cookie state accumulated across the redirect chain. */
+    cookieStore: StealthCookieStoreV1;
 }
 export interface StealthSession {
     fetch(url: string, options?: StealthFetchOptions): Promise<StealthResponse>;
@@ -988,6 +1037,139 @@ export interface HttpClient {
     stream(url: string, options?: RequestWithMethodOptions): Promise<HttpStreamResponse>;
     sse(url: string, options?: RequestWithMethodOptions): Promise<AsyncIterable<SseMessage>>;
 }
+/** Request-scoped file reference accepted by provider operation inputs. */
+export interface ProviderFileRef {
+    readonly type: "request_file";
+    readonly id: string;
+    readonly filename: string;
+    readonly mime_type?: string;
+    readonly size: number;
+    readonly sha256?: string;
+}
+/** File body resolved from a request-scoped {@link ProviderFileRef}. */
+export type ProviderResolvedFile = Omit<ProviderFileRef, "mime_type"> & {
+    readonly mimeType?: string;
+    arrayBuffer(): Promise<ArrayBuffer>;
+    bytes(): Promise<Uint8Array>;
+    stream(): ReadableStream<Uint8Array>;
+};
+/** Resolver supplied by runtimes that accept request-scoped file inputs. */
+export interface ProviderFilesContext {
+    has(input: string | ProviderFileRef): boolean;
+    resolve(input: string | ProviderFileRef): Promise<ProviderResolvedFile>;
+}
+export type NativeTcpTlsMode = "required" | "allowed" | "disabled";
+export interface NativeTcpPortRange {
+    readonly start: number;
+    readonly end: number;
+}
+/** Static native TCP egress declared by a provider. */
+export interface NativeTcpEgressRule {
+    readonly host: string;
+    readonly ports: readonly number[];
+    readonly tls: NativeTcpTlsMode;
+}
+/**
+ * Bounded native TCP egress discovered through a declared bootstrap endpoint.
+ * Host suffixes are exact DNS suffixes, not wildcard patterns.
+ */
+export interface NativeTcpDynamicEgressRule {
+    readonly sourceHost?: string;
+    readonly sourceHostSuffixes?: readonly string[];
+    readonly sourcePorts?: readonly number[];
+    readonly sourcePortRanges?: readonly NativeTcpPortRange[];
+    readonly targetHostSuffixes: readonly string[];
+    readonly targetPorts?: readonly number[];
+    readonly targetPortRanges?: readonly NativeTcpPortRange[];
+    readonly tls: NativeTcpTlsMode;
+    readonly ttlMs?: number;
+    readonly maxGrants?: number;
+}
+/** Common TCP/TLS connection input supported by the native runtime. */
+export interface NativeNetworkConnectInput {
+    readonly host: string;
+    readonly port: number;
+    readonly serverName?: string;
+    readonly rejectUnauthorized?: boolean;
+    /**
+     * Maximum time without a successful socket read before the connection is
+     * closed. Opt-in; when absent, reads can remain pending indefinitely.
+     */
+    readonly idleTimeoutMs?: number;
+    /** Maximum time allowed to establish the TCP/SOCKS/TLS connection. */
+    readonly timeoutMs?: number;
+    readonly signal?: AbortSignal;
+    /** Overrides the credential-derived sticky affinity key. */
+    readonly affinityKey?: string;
+}
+export type NativeNetworkConnectOptions = Omit<NativeNetworkConnectInput, "serverName" | "rejectUnauthorized">;
+export type NativeTlsConnectOptions = NativeNetworkConnectInput;
+export interface NativeNetworkDynamicGrantOptions {
+    readonly sourceHost: string;
+    readonly sourcePort: number;
+    readonly host: string;
+    readonly port: number;
+    readonly tls: NativeTcpTlsMode;
+    readonly ttlMs?: number;
+}
+export interface NativeNetworkEgressGrant {
+    revoke(): void;
+}
+/** Consumer-facing alias used by native TCP providers. */
+export type NativeTcpEgressGrant = NativeNetworkEgressGrant;
+/** Resolved egress identity for a native connection routed through a proxy. */
+export interface NativeProxyEgressInfo {
+    readonly vendor: ProviderProxyProvider;
+    readonly sticky: boolean;
+    /** Vendor sticky session id (sid). Absent for rotating sessions. */
+    readonly sessionId?: string;
+    /** Hard expiry of the sticky binding, ISO 8601. */
+    readonly expiresAt?: string;
+}
+export type NativeProxyExpiringReason = "sticky_expiry";
+export interface NativeProxyExpiringEvent {
+    readonly expiresAt: string;
+    readonly leadSeconds: number;
+    readonly reason: NativeProxyExpiringReason;
+}
+/**
+ * Cooperative drain handler. The SDK awaits this before closing a socket whose
+ * sticky proxy binding is about to expire, then force-closes at hard expiry.
+ */
+export type NativeProxyDrainHandler = (event: NativeProxyExpiringEvent) => void | Promise<void>;
+/** Typed reason recorded when the SDK closes a native connection intentionally. */
+export interface NativeNetworkCloseReason {
+    readonly code: string;
+    readonly message: string;
+}
+/** Byte-oriented connection returned by the native TCP/TLS runtime. */
+export interface NativeNetworkConnection {
+    /** Present when the connection was routed through a proxy. */
+    readonly proxy?: NativeProxyEgressInfo;
+    /** Present after an SDK-planned close, such as sticky proxy expiry. */
+    readonly closeReason?: NativeNetworkCloseReason;
+    /** Register a cooperative drain handler for sticky-expiry reconnects. */
+    onExpiring?(handler: NativeProxyDrainHandler): void;
+    read(): Promise<Uint8Array | null>;
+    write(data: Uint8Array): Promise<void>;
+    close(): Promise<void>;
+}
+export interface NativeNetworkClient {
+    connectTcp(input: NativeNetworkConnectOptions): Promise<NativeNetworkConnection>;
+    connectTls(input: NativeTlsConnectOptions): Promise<NativeNetworkConnection>;
+    grantTcpEgress(input: NativeNetworkDynamicGrantOptions): NativeNetworkEgressGrant;
+}
+export interface NativeContext {
+    readonly network: NativeNetworkClient;
+}
+/** Consumer-facing alias for the native capability on provider contexts. */
+export type NativeProviderContext = NativeContext;
+export interface NativeProviderConfig {
+    readonly network?: {
+        readonly tcp?: readonly NativeTcpEgressRule[];
+        readonly dynamicTcp?: readonly NativeTcpDynamicEgressRule[];
+    };
+}
 export interface ProviderCacheKeyOptions {
     /**
      * Additional field names to omit from stable key material. The SDK always
@@ -1319,6 +1501,8 @@ export interface FlowContext {
     tenantId: string;
     providerId: string;
     http: HttpClient;
+    /** Present when the selected runtime supplies native network capabilities. */
+    readonly native?: NativeProviderContext;
     stealth: StealthClient;
     env: EnvContext;
     credential?: CredentialContext;
@@ -1404,6 +1588,10 @@ export interface ProviderContext {
     credential: CredentialContext;
     request?: ProviderRequestContext;
     http: HttpClient;
+    /** Present for requests carrying runtime-resolvable file references. */
+    readonly files?: ProviderFilesContext;
+    /** Present when the selected runtime supplies native network capabilities. */
+    readonly native?: NativeProviderContext;
     cache: ProviderCache;
     state: ProviderRuntimeState;
     stealth: StealthClient;
@@ -1480,6 +1668,11 @@ export interface OperationDefinition<TInput extends SchemaLike = SchemaLike, TOu
     fixtures?: {
         request: InferSchemaOutput<TInput>;
         response: InferSchemaOutput<TOutput>;
+        /**
+         * KST calendar date when `response` evidence was captured. Date fields in
+         * the response align with this date, not a resolved relative request date.
+         */
+        recordedAt?: string;
     };
     upstream?: {
         baseUrl?: string;
@@ -1538,6 +1731,7 @@ export interface ProviderDefinition {
      */
     deployment?: ProviderDeploymentOverrides;
     allowedHosts?: string[];
+    native?: NativeProviderConfig;
     stealth?: {
         profile: string;
         platform: StealthPlatform;

package/package.json

@@ -1,5 +1,5 @@
 {
-	"version": "2.2.0-beta.11",
+	"version": "2.2.0-beta.13",
 	"name": "@apifuse/provider-sdk",
 	"private": false,
 	"type": "module",
@@ -114,6 +114,8 @@
 		"puppeteer-extra-plugin-stealth": "^2.11.2",
 		"re2-wasm": "^1.0",
 		"safe-regex": "^2.1",
+		"socks": "^2.8.9",
+		"tough-cookie": "^6.0.2",
 		"zod": "^4.4.3"
 	},
 	"repository": {

package/src/config/loader.ts

@@ -156,10 +156,19 @@ export type ProxyTelemetrySink = {
 	recordProxyVendorFailover?(event: ProxyVendorFailoverTelemetryEvent): void;
 };
 
+export type ProxyResolutionSource =
+	| "explicit"
+	| "env"
+	| "config"
+	| "smartproxy-allocator"
+	| "nodemaven-gateway";
+
 export type ResolvedProxyConfig = {
 	shouldWarn: boolean;
 	url?: string;
-	source?: "explicit" | "env" | "config" | "smartproxy-allocator" | "nodemaven-gateway";
+	/** SDK-native vendor that supplied the URL, when applicable. */
+	vendor?: ProxyVendorName;
+	source?: ProxyResolutionSource;
 	protocol?: ProxyProtocol;
 	diagnostics?: Record<string, string | number | boolean>;
 };
@@ -656,6 +665,18 @@ export async function resolveProxyConfigAsync(
 	return { shouldWarn: true };
 }
 
+/**
+ * Resolve the proxy URL for a provider-owned consumer such as a CAPTCHA solver.
+ * Vendor allocation and failover remain owned by the SDK.
+ */
+export async function resolveProxy(
+	options: ProxyResolutionOptions = {},
+): Promise<ResolvedProxyConfig> {
+	const resolved = await resolveProxyConfigAsync(options);
+	const vendor = vendorFromResolvedSource(resolved.source);
+	return vendor ? { ...resolved, vendor } : resolved;
+}
+
 /**
  * Each vendor's default egress protocol, chosen from live KR benchmarks. HTTP
  * CONNECT wins for nodemaven (socks5 adds ~500ms through the gateway) and ties

package/src/define.ts

@@ -2,6 +2,7 @@ import ms from "ms";
 
 import { ProviderError, ValidationError } from "./errors.js";
 import { safeParseSchemaSync } from "./schema.js";
+import { resolveHealthCheckInputDateTokens } from "./server/self-test-input-tokens.js";
 import type {
 	AuthConfig,
 	BrowserEngine,
@@ -20,6 +21,7 @@ import type {
 	OperationSseTransport,
 	OperationTransport,
 	OperationWebSocketTransport,
+	NativeProviderConfig,
 	ProviderAccessConfig,
 	ProviderDefinition,
 	ProviderDeploymentOverrides,
@@ -205,6 +207,7 @@ export interface ProviderConfig<TOperations extends Record<string, ProviderOpera
 	 */
 	deployment?: ProviderDeploymentOverrides;
 	allowedHosts?: string[];
+	native?: NativeProviderConfig;
 	stealth?: {
 		profile: string;
 		platform: StealthPlatform;
@@ -437,7 +440,7 @@ function validateProviderProxy(config: {
 		}
 		rejectUnknownFields(
 			proxy.session,
-			new Set(["affinity", "lifetimeMinutes", "poolSize"]),
+			new Set(["affinity", "lifetimeMinutes", "poolSize", "drainLeadSeconds"]),
 			"proxy.session",
 		);
 		if (proxy.session.affinity !== undefined) {
@@ -460,6 +463,34 @@ function validateProviderProxy(config: {
 				`Provider "${config.id}" has invalid proxy.session.poolSize: must be a positive integer.`,
 			);
 		}
+		const drainLeadSeconds = proxy.session.drainLeadSeconds;
+		if (
+			drainLeadSeconds !== undefined &&
+			(!Number.isFinite(drainLeadSeconds) || drainLeadSeconds <= 0)
+		) {
+			throw new ValidationError(
+				`Provider "${config.id}" has invalid proxy.session.drainLeadSeconds: must be a positive number of seconds.`,
+				{
+					fix: `Use proxy.session.drainLeadSeconds: 120 to receive the sticky-expiry drain event 120s before hard expiry.`,
+				},
+			);
+		}
+		// A drain lead longer than the sticky lifetime would fire the expiring
+		// event before the session is even established, so the provider would
+		// never get a usable window. Reject the contradiction at build time.
+		if (
+			drainLeadSeconds !== undefined &&
+			lifetime !== undefined &&
+			Number.isFinite(lifetime) &&
+			drainLeadSeconds >= lifetime * 60
+		) {
+			throw new ValidationError(
+				`Provider "${config.id}" has proxy.session.drainLeadSeconds (${drainLeadSeconds}s) greater than or equal to proxy.session.lifetimeMinutes (${lifetime}m).`,
+				{
+					fix: `Lower drainLeadSeconds below the sticky lifetime so the drain event leaves a usable session window.`,
+				},
+			);
+		}
 	}
 	// Every credentialed vendor in a required-mode chain must declare its
 	// provider secret(s) so a missing credential fails at build/validation time,
@@ -2040,6 +2071,26 @@ function validateOperationFixtures(
 					fix: `Add operations.${operationName}.handler as an async function with signature (ctx, input) => Promise<output>`,
 				},
 			);
+		if (operation.fixtures?.recordedAt !== undefined) {
+			const recordedAt = operation.fixtures.recordedAt;
+			const parsed =
+				typeof recordedAt === "string"
+					? new Date(`${recordedAt}T00:00:00.000Z`)
+					: new Date(Number.NaN);
+			const isCalendarDate =
+				typeof recordedAt === "string" &&
+				/^\d{4}-\d{2}-\d{2}$/.test(recordedAt) &&
+				!Number.isNaN(parsed.getTime()) &&
+				parsed.toISOString().slice(0, 10) === recordedAt;
+			const kstToday = new Date(Date.now() + 9 * 60 * 60 * 1000).toISOString().slice(0, 10);
+			if (!isCalendarDate || recordedAt > kstToday)
+				throw new ValidationError(
+					`Fixture recordedAt must be a valid, non-future KST calendar date for provider "${providerId}" operation "${operationName}"`,
+					{
+						fix: `Set operations.${operationName}.fixtures.recordedAt to the KST capture date in YYYY-MM-DD format; it must not be in the future.`,
+					},
+				);
+		}
 		if (operation.fixtures?.request !== undefined) {
 			const result = safeParseSchemaSync(
 				operation.input,
@@ -2073,6 +2124,31 @@ function validateOperationFixtures(
 	}
 }
 
+function resolveOperationFixtureRequests<TOperations extends Record<string, ProviderOperation>>(
+	operations: TOperations,
+): TOperations {
+	let changed = false;
+	const resolvedOperations = Object.fromEntries(
+		Object.entries(operations).map(([operationName, operation]) => {
+			if (operation.fixtures?.request === undefined) return [operationName, operation];
+			const request = resolveHealthCheckInputDateTokens(operation.fixtures.request);
+			if (request === operation.fixtures.request) return [operationName, operation];
+			changed = true;
+			return [
+				operationName,
+				{
+					...operation,
+					fixtures: {
+						...operation.fixtures,
+						request,
+					},
+				},
+			];
+		}),
+	) as TOperations;
+	return changed ? resolvedOperations : operations;
+}
+
 /**
  * Shallow shape guard only: the `deployment` object is passed through
  * verbatim and deliberately not deep-validated by the SDK — the APIFuse
@@ -2093,6 +2169,7 @@ export function defineProvider<
 	config: TConfig & AuthStartNoInputGuard<TConfig>,
 ): ProviderDefinition & { operations: OperationMapConfig<TOperations> } {
 	validateProviderShape(config);
+	const operations = resolveOperationFixtureRequests(config.operations);
 	if (!CONNECTOR_ID_REGEX.test(config.id))
 		throw new ProviderError(`Invalid provider id: "${config.id}"`, {
 			fix: 'Use lowercase alphanumeric with dashes, e.g., "korea-air-quality"',
@@ -2125,7 +2202,7 @@ export function defineProvider<
 		config.healthProbe ?? config.healthMonitor,
 		config.healthProbe !== undefined ? "healthProbe" : "healthMonitor",
 	);
-	validateOperationFixtures(config.id, config.operations);
+	validateOperationFixtures(config.id, operations);
 	validateProviderDeployment(config.id, config.deployment);
 	validateProviderProxy(config);
 	validateProviderStt(config);
@@ -2149,6 +2226,7 @@ export function defineProvider<
 		// are owned by the APIFuse registry builder, not the SDK.
 		deployment: config.deployment,
 		allowedHosts: config.allowedHosts,
+		native: config.native,
 		stealth: config.stealth,
 		proxy: config.proxy,
 		stt: config.stt,
@@ -2160,7 +2238,7 @@ export function defineProvider<
 		credential: config.credential,
 		context: config.context,
 		meta: config.meta,
-		operations: config.operations,
+		operations,
 		// Transitional healthMonitor → healthProbe alias: mirror whichever field
 		// was declared onto both so old and new consumers keep working.
 		healthMonitor: config.healthMonitor ?? config.healthProbe,

package/src/errors.ts

@@ -79,6 +79,21 @@ export class SDKError extends ProviderError {
 	}
 }
 
+/** Raised when persisted stealth cookies use a store version this SDK cannot read. */
+export class StealthCookieStoreVersionError extends SDKError {
+	constructor(public readonly version: unknown) {
+		const displayedVersion =
+			typeof version === "string" || typeof version === "number"
+				? String(version)
+				: "missing or invalid";
+		super(`Unsupported stealth cookie store version: ${displayedVersion}`, {
+			code: "unsupported_stealth_cookie_store_version",
+			details: { receivedVersion: version, supportedVersions: [1] },
+		});
+		this.name = "StealthCookieStoreVersionError";
+	}
+}
+
 export class AuthError extends ProviderError {
 	constructor(message: string, options?: ProviderErrorOptions) {
 		super(message, options);