@apifuse/provider-sdk 2.2.0-beta.2 → 2.2.0-beta.21

package/src/server/serve.ts

@@ -3,62 +3,83 @@ import { join } from "node:path";
 
 import { Hono } from "hono";
 import { z } from "zod";
-import { AuthAbortError, createAuthFlowHelpers } from "../auth";
+import { AuthAbortError, createAuthFlowHelpers } from "../auth.js";
+import {
+	SDK_OWNED_PROVIDER_ERROR_CODES,
+	SDK_RUNTIME_OWNED_ERROR_CODES,
+} from "../error-resolution.js";
 import {
 	AuthError,
+	isProviderError,
+	isSessionExpiredError,
+	isTransportError,
+	isValidationError,
 	ProviderError,
-	SessionExpiredError,
-	TransportError,
-} from "../errors";
+} from "../errors.js";
 import {
 	loadProviderLocaleCatalogs,
 	localizeAuthTurn,
 	type ProviderLocaleCatalogMap,
-} from "../i18n/catalog";
-import type { ProviderLocale } from "../i18n/keys";
+} from "../i18n/catalog.js";
+import type { ProviderLocale } from "../i18n/keys.js";
 import {
 	categoryForStatus,
+	type ProviderErrorSource,
+	sourceForCategory,
 	isRetryableCategory,
 	PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
 	type ProviderErrorCategory,
-} from "../observability";
-import { createScratchpad } from "../runtime/auth-flow";
-import { createBrowserClient } from "../runtime/browser";
-import { createProviderCache } from "../runtime/cache";
+} from "../observability.js";
+import { createScratchpad } from "../runtime/auth-flow.js";
+import { createBrowserClient } from "../runtime/browser.js";
+import { createProviderCache } from "../runtime/cache.js";
 import {
 	createProviderChoiceContext,
 	PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV,
-} from "../runtime/choice";
-import { createCredentialContext } from "../runtime/credential";
-import { createEnvContext } from "../runtime/env";
-import { executeOperation } from "../runtime/executor";
-import { createHttpClient } from "../runtime/http";
-import { wrapWithInstrumentation } from "../runtime/instrumentation";
-import { getProviderBaseUrl } from "../runtime/provider";
+} from "../runtime/choice.js";
+import { createCredentialContext } from "../runtime/credential.js";
+import { createEnvContext } from "../runtime/env.js";
+import { executeOperation } from "../runtime/executor.js";
+import { createHttpClient } from "../runtime/http.js";
+import { wrapWithInstrumentation } from "../runtime/instrumentation.js";
+import {
+	createEnvVendorCredentialResolver,
+	createNativeNetworkClient,
+} from "../runtime/native-network.js";
+import { getProviderBaseUrl } from "../runtime/provider.js";
 import {
 	PROXY_AUTH_IP_DENIED_CODE,
 	PROXY_EDGE_AUTH_REJECTED_CODE,
 	PROXY_POOL_EXHAUSTED_CODE,
-} from "../runtime/proxy-errors";
+} from "../runtime/proxy-errors.js";
+import { PROVIDER_TELEMETRY_HEADER, ProxyTelemetryCollector } from "../runtime/proxy-telemetry.js";
 import {
-	PROVIDER_TELEMETRY_HEADER,
-	ProxyTelemetryCollector,
-} from "../runtime/proxy-telemetry";
+	assertRequiredSecretsPresent,
+	listMissingRequiredSecrets,
+	MISSING_SECRET_CODE,
+} from "../runtime/secrets.js";
 import {
 	createProviderRuntimeStateFromEnv,
 	createUnsupportedProviderRuntimeState,
-} from "../runtime/state";
-import { createStealthClient } from "../runtime/stealth";
-import { createSttClientFromEnv } from "../runtime/stt";
-import { createTraceContext } from "../runtime/trace";
-import { parseSchema } from "../schema";
-import { getStealthProfile } from "../stealth/profiles";
+} from "../runtime/state.js";
+import { createStealthClient } from "../runtime/stealth.js";
+import { createSttClientFromEnv } from "../runtime/stt.js";
+import { createTraceContext } from "../runtime/trace.js";
+import { parseSchema } from "../schema.js";
+import {
+	STATEFUL_NONCE_HEADER as STATEFUL_FORWARDING_NONCE_HEADER,
+	STATEFUL_SIGNATURE_HEADER as STATEFUL_FORWARDING_SIGNATURE_HEADER,
+	STATEFUL_TIMESTAMP_HEADER as STATEFUL_FORWARDING_TIMESTAMP_HEADER,
+	verifyStatefulRequestSignature,
+} from "../stateful-signing.js";
+import { StatefulRoutingDeadlineError } from "../stateful/stateful-provider-session-routing.js";
+import { getStealthProfile } from "../stealth/profiles.js";
 import {
 	APIFUSE_STREAM_DONE_EVENT,
 	APIFUSE_STREAM_ERROR_EVENT,
 	encodeSseEvent,
 	error as streamError,
-} from "../stream";
+} from "../stream.js";
 import type {
 	AuthContext,
 	AuthTurn,
@@ -67,37 +88,118 @@ import type {
 	FlowContextStore,
 	HttpRetrySummary,
 	OperationDefinition,
+	OperationErrorCode,
 	OperationHttpStreamTransport,
 	OperationSseTransport,
+	ProviderErrorStatus,
 	ProviderContext,
 	ProviderDefinition,
+	ProviderProxyPolicy,
 	ProviderRuntimeState,
 	ProviderStreamEvent,
 	StealthClient,
 	SttContext,
-} from "../types";
+} from "../types.js";
+import { VALID_OPERATION_ERROR_STATUSES } from "../types.js";
 import {
 	createSelfTestApp,
+	createSelfTestAuthFlowInvoke,
 	createSelfTestInvoke,
 	resolveSelfTestPort,
-} from "./self-test";
-import { resolveSelfTestMasterSecrets } from "./self-test-token";
+} from "./self-test.js";
+import { resolveSelfTestMasterSecrets } from "./self-test-token.js";
 import {
 	type AuthFlowRequest,
 	AuthFlowRequestSchema,
 	type AuthFlowResponse,
 	type AuthFlowSuccessResponse,
+	OperationConnectionSchema,
 	type OperationErrorResponse,
 	type OperationRequest,
 	OperationRequestSchema,
 	type OperationResponse,
 	type OperationSuccessResponse,
-} from "./types";
+} from "./types.js";
 
 const DEFAULT_HOST = "0.0.0.0";
 const DEFAULT_PORT = 3000;
+/** Compact SDK-owned error classification emitted separately from the public response body. */
+export const ERROR_OBSERVABILITY_HEADER = "X-ApiFuse-Error-Observability";
+export type ErrorObservabilityDetails = {
+	category: ProviderErrorCategory;
+	taxonomyVersion: string;
+	retryable: boolean;
+	upstreamStatus?: number;
+};
 const AUTH_FLOW_LOCALES = ["en", "ko", "ja"] as const;
 const retryResponseMeta = new WeakMap<ProviderContext, HttpRetrySummary>();
+const STATEFUL_INTERNAL_OPERATIONS_ROUTE = "/__apifuse/stateful/operations";
+const STATEFUL_FORWARDING_SOURCE_POD_HEADER = "x-apifuse-stateful-source-pod";
+const DEFAULT_STATEFUL_FORWARDING_MAX_SKEW_MS = 5 * 60_000;
+const DEFAULT_STATEFUL_FORWARDING_REPLAY_CACHE_MAX_ENTRIES = 10_000;
+const STATEFUL_FORWARDING_REPLAY_BUCKET_MS = 10_000;
+const STATEFUL_FORWARDING_REPLAY_RETRY_AFTER_SECONDS = Math.ceil(
+	STATEFUL_FORWARDING_REPLAY_BUCKET_MS / 1_000,
+);
+
+export const ProviderServerStatefulForwardEnvelopeSchema = z
+	.object({
+		requestId: z.string().min(1),
+		providerId: z.string().min(1),
+		operationId: z.string().min(1),
+		sessionKey: z.string().min(1),
+		connectionId: z.string().min(1),
+		serviceAccountId: z.string().min(1),
+		ownerPodId: z.string().min(1),
+		generation: z.number().int().positive(),
+		sourcePodId: z.string().min(1),
+		forwardedAt: z.string().refine((value) => Number.isFinite(Date.parse(value))),
+		deadlineAt: z
+			.string()
+			.refine((value) => Number.isFinite(Date.parse(value)))
+			.optional(),
+		idempotencyKey: z.string().min(1).optional(),
+		operationRequest: OperationRequestSchema.extend({
+			connection: OperationConnectionSchema.strict().optional(),
+		}).strict(),
+	})
+	.strict();
+
+export type ProviderServerStatefulForwardEnvelope = Readonly<
+	z.infer<typeof ProviderServerStatefulForwardEnvelopeSchema>
+>;
+
+export type ProviderServerStatefulOwnerFence = Readonly<
+	Pick<
+		ProviderServerStatefulForwardEnvelope,
+		| "providerId"
+		| "sessionKey"
+		| "ownerPodId"
+		| "generation"
+		| "sourcePodId"
+		| "forwardedAt"
+		| "requestId"
+		| "idempotencyKey"
+	>
+>;
+
+export type ProviderServerStatefulOwnerFenceValidator = (
+	fence: ProviderServerStatefulOwnerFence,
+	signal: AbortSignal,
+) => boolean | Promise<boolean>;
+
+export type ProviderServerOperationExecutorInput = {
+	readonly provider: ProviderDefinition;
+	readonly operationId: string;
+	readonly ctx: ProviderContext;
+	readonly request: OperationRequest & { readonly deadlineAt?: string };
+	readonly signal?: AbortSignal;
+	readonly internalStatefulForward?: ProviderServerStatefulForwardEnvelope;
+};
+
+export type ProviderServerOperationExecutor = (
+	input: ProviderServerOperationExecutorInput,
+) => Promise<unknown>;
 
 type RequestCleanup = () => void | Promise<void>;
 
@@ -156,9 +258,7 @@ function createStealthStub(): StealthClient {
 	};
 }
 
-function getProviderStealthBaseUrl(
-	provider: ProviderDefinition,
-): string | undefined {
+function getProviderStealthBaseUrl(provider: ProviderDefinition): string | undefined {
 	const baseUrl = getProviderBaseUrl(provider);
 	if (baseUrl) {
 		return baseUrl;
@@ -168,15 +268,10 @@ function getProviderStealthBaseUrl(
 }
 
 function getProviderStealthProfile(provider: ProviderDefinition) {
-	return provider.stealth?.profile
-		? getStealthProfile(provider.stealth.profile)
-		: undefined;
+	return provider.stealth?.profile ? getStealthProfile(provider.stealth.profile) : undefined;
 }
 
-function isProductionProviderBrowserMode(
-	provider: ProviderDefinition,
-	env = process.env,
-): boolean {
+function isProductionProviderBrowserMode(provider: ProviderDefinition, env = process.env): boolean {
 	if (provider.runtime !== "browser") {
 		return false;
 	}
@@ -185,9 +280,7 @@ function isProductionProviderBrowserMode(
 		return true;
 	}
 
-	return (
-		env.NODE_ENV === "production" && env.APIFUSE__PROVIDER__ID === provider.id
-	);
+	return env.NODE_ENV === "production" && env.APIFUSE__PROVIDER__ID === provider.id;
 }
 
 export function resolveProviderProxyAffinityKey(
@@ -195,24 +288,26 @@ export function resolveProviderProxyAffinityKey(
 	request: OperationRequest,
 	operationId: string,
 ): string {
-	const connectionKey =
-		resolveOperationConnectionId(request) ?? request.connection?.externalRef;
+	const connectionKey = resolveOperationConnectionId(request) ?? request.connection?.externalRef;
 	const affinity =
-		typeof provider.proxy === "object"
-			? provider.proxy.session?.affinity
-			: undefined;
+		typeof provider.proxy === "object" ? provider.proxy.session?.affinity : undefined;
 	if (affinity === "operation") {
 		return `${provider.id}/${operationId}`;
 	}
 	return connectionKey ?? provider.id;
 }
 
-function resolveOperationConnectionId(
-	request: OperationRequest,
-): string | undefined {
+function resolveOperationConnectionId(request: OperationRequest): string | undefined {
 	return request.connection?.id ?? request.connectionId;
 }
 
+function resolveNativeProxyPolicy(provider: ProviderDefinition): ProviderProxyPolicy | undefined {
+	if (typeof provider.proxy === "object") return provider.proxy;
+	if (provider.proxy === true) return { mode: "optional" };
+	if (provider.proxy === false) return { mode: "disabled" };
+	return undefined;
+}
+
 function createProviderContext(
 	provider: ProviderDefinition,
 	request: OperationRequest,
@@ -226,11 +321,7 @@ function createProviderContext(
 	const stealthProfile = getProviderStealthProfile(provider);
 	const proxyClientOptions = {
 		upstream: { proxy: provider.proxy },
-		affinityKey: resolveProviderProxyAffinityKey(
-			provider,
-			request,
-			operationId,
-		),
+		affinityKey: resolveProviderProxyAffinityKey(provider, request, operationId),
 		telemetry: proxyTelemetry,
 	};
 	let wrappedContext: ProviderContext | undefined;
@@ -269,11 +360,7 @@ function createProviderContext(
 		state,
 		stealth: stealthBaseUrl
 			? stealthProfile
-				? createStealthClient(
-						stealthBaseUrl,
-						stealthProfile.name,
-						stealthClientOptions,
-					)
+				? createStealthClient(stealthBaseUrl, stealthProfile.name, stealthClientOptions)
 				: createStealthClient(stealthBaseUrl, stealthClientOptions)
 			: createStealthStub(),
 		browser:
@@ -287,6 +374,18 @@ function createProviderContext(
 						engine: provider.browser?.engine,
 					})
 				: createBrowserStub(),
+		...(provider.native
+			? {
+					native: {
+						network: createNativeNetworkClient({
+							egress: provider.native.network,
+							proxyPolicy: resolveNativeProxyPolicy(provider),
+							affinityKey: proxyClientOptions.affinityKey,
+							credentials: createEnvVendorCredentialResolver(env),
+						}),
+					},
+				}
+			: {}),
 		trace: createTraceContext(),
 		auth: createAuthStub(),
 		stt: options.stt ?? createSttClientFromEnv(provider.stt),
@@ -379,6 +478,7 @@ function createAuthFlowContext(
 
 	return {
 		context: {
+			flowId: request.flowId,
 			connectionId: request.connectionId,
 			externalRef: request.externalRef,
 			tenantId: request.tenantId ?? "",
@@ -386,14 +486,29 @@ function createAuthFlowContext(
 			http: createHttpClient(baseUrl, proxyClientOptions),
 			stealth: stealthBaseUrl
 				? stealthProfile
-					? createStealthClient(
-							stealthBaseUrl,
-							stealthProfile.name,
-							stealthClientOptions,
-						)
+					? createStealthClient(stealthBaseUrl, stealthProfile.name, stealthClientOptions)
 					: createStealthClient(stealthBaseUrl, stealthClientOptions)
 				: createStealthStub(),
-			env: createEnvContext(provider.secrets?.map((secret) => secret.name)),
+			...(provider.native
+				? {
+						native: {
+							network: createNativeNetworkClient({
+								egress: provider.native.network,
+								proxyPolicy: resolveNativeProxyPolicy(provider),
+								affinityKey: proxyClientOptions.affinityKey,
+								credentials: createEnvVendorCredentialResolver(
+									createEnvContext(provider.secrets?.map((secret) => secret.name)),
+								),
+							}),
+						},
+					}
+				: {}),
+			env: createEnvContext([
+				...(provider.secrets?.map((secret) => secret.name) ?? []),
+				...(provider.auth?.mode === "oauth2_proxied"
+					? ["APIFUSE__AUTH_PROXY__URL"]
+					: []),
+			]),
 			credential,
 			context: flowContextStore.context,
 			stt: options.stt ?? createSttClientFromEnv(provider.stt),
@@ -422,7 +537,7 @@ export type ProviderServerLogEvent =
 	| (ProviderServerLogEventBase & {
 			level: "info";
 			event: "provider_request_completed";
-	  })
+		})
 	| (ProviderServerLogEventBase & {
 			level: "warn" | "error";
 			event: "provider_request_failed";
@@ -433,8 +548,16 @@ export type ProviderServerLogEvent =
 			errorCategory?: ProviderErrorCategory;
 			taxonomyVersion?: string;
 			retryable?: boolean;
+			signal?: "unregistered_provider_error_code";
+			signalFix?: string;
 			issues?: Array<{ path: string; code: string; message: string }>;
-	  })
+		})
+	| {
+			level: "warn";
+			event: "provider_secrets_missing";
+			providerId: string;
+			missingSecrets: string[];
+		}
 	| {
 			level: "warn";
 			event: "provider_cleanup_failed";
@@ -445,18 +568,59 @@ export type ProviderServerLogEvent =
 			resource: "browser" | "stealth";
 			errorClass: string;
 			message: string;
-	  };
+		}
+	| {
+			level: "error";
+			event: "provider_shutdown_hook_failed";
+			providerId: string;
+			hookIndex: number;
+			errorClass: string;
+			message: string;
+		};
 
 export type ProviderServerLogger = (event: ProviderServerLogEvent) => void;
 
 export type ProviderServerOptions = {
 	logger?: ProviderServerLogger;
+	/** Optional provider-specific operation executor. Stateful providers use this to preserve provider-local runtime semantics. */
+	operationExecutor?: ProviderServerOperationExecutor;
+	/** Optional signed internal executor for stateful owner forwarding. */
+	internalOperationExecutor?: ProviderServerOperationExecutor;
+	statefulForwarding?: {
+		readonly secret: string;
+		readonly maxSkewMs?: number;
+		readonly replayCacheMaxEntries?: number;
+		/** Required fail-closed check against the SDK/runtime owner registry. */
+		readonly validateOwnerFence: ProviderServerStatefulOwnerFenceValidator;
+	};
 	/** Optional STT override for tests or custom hosts; local/prod normally resolves from env. */
 	stt?: SttContext;
 	/** Optional runtime state override for tests or custom hosts. Production resolves Redis from env and fails closed when unavailable. */
 	state?: ProviderRuntimeState;
 	/** Allow process-local runtime state only for local development and tests. */
 	allowMemoryStateFallback?: boolean;
+	/**
+	 * Graceful process shutdown. Hooks run in declaration order after listeners stop accepting work.
+	 *
+	 * @example
+	 * ```ts
+	 * await serve(provider, {
+	 *	 shutdown: {
+	 *		 hooks: [
+	 *			 async () => { await emitter.flush(); },
+	 *			 async () => { await sessionManager.closeAll("server-shutdown"); },
+	 *			 async () => { await lease.release(); },
+	 *			 async () => { await router.close(); },
+	 *		 ],
+	 *	 },
+	 * });
+	 * ```
+	 */
+	shutdown?: {
+		readonly hooks?: Array<() => Promise<void>>;
+		readonly signals?: boolean | NodeJS.Signals[];
+		readonly timeoutMs?: number;
+	};
 };
 
 const defaultProviderServerLogger: ProviderServerLogger = (event) => {
@@ -503,19 +667,57 @@ function zodDetails(error: z.ZodError): Array<{
 	}));
 }
 
+// Category-level projection with code-aware honesty overrides: a missing
+// deployment secret is an APIFuse-side defect even though its category
+// (credential_unavailable) usually means a caller credential problem, an
+// internal stateful-routing deadline is APIFuse-owned despite its timeout
+// category, and the built-in upstream failure families keep their upstream
+// attribution even when the author left the category at the provider_error
+// default.
+function publicErrorSource(
+	error: unknown,
+	category: ProviderErrorCategory,
+): ProviderErrorSource {
+	if (error instanceof StatefulRoutingDeadlineError) return "apifuse";
+	if (isProviderError(error)) {
+		if (error.code === MISSING_SECRET_CODE) return "apifuse";
+		if (error.code === "UPSTREAM_ERROR" || error.code === "BLOCKED") {
+			return "upstream_failure";
+		}
+	}
+	return sourceForCategory(category);
+}
+
 function toErrorResponse(
 	error: unknown,
 	requestId?: string,
+	declaredErrorCode?: OperationErrorCode,
 ): OperationErrorResponse {
-	if (error instanceof ProviderError) {
-		const details = publicProviderErrorDetails(error);
+	const observability = errorObservabilityDetails(error, declaredErrorCode);
+	const source = publicErrorSource(error, observability.category);
+	if (error instanceof StatefulRoutingDeadlineError) {
+		return {
+			error: {
+				code: "STATEFUL_FORWARDING_DEADLINE_EXPIRED",
+				message: "Stateful forwarding deadline expired.",
+				...(requestId ? { requestId } : {}),
+				retryable: observability.retryable,
+				source,
+			},
+		};
+	}
+
+	if (isProviderError(error)) {
+		const details = error.details;
 		return {
 			error: {
 				code: error.code ?? "provider_error",
 				message: publicProviderErrorMessage(error),
 				...(requestId ? { requestId } : {}),
+				retryable: observability.retryable,
+				source,
 				...(error.fix ? { fix: error.fix } : {}),
-				...(details ? { details } : {}),
+				...(details !== undefined ? { details } : {}),
 			},
 		};
 	}
@@ -526,64 +728,72 @@ function toErrorResponse(
 				code: "invalid_request",
 				message: "Invalid request body",
 				...(requestId ? { requestId } : {}),
+				retryable: observability.retryable,
+				source,
 				details: zodDetails(error),
 			},
 		};
 	}
 
+	// A masked internal error MUST NOT be advertised as retryable: without an
+	// explicit retryable:false the hub (bori provider-backed engine) defaults 5xx
+	// to retryable:true, which turns a deterministic pre-upstream crash into an
+	// infinite START->CONTINUE->restart loop (2026-07-22 catchtable reserve RCA).
+	// We still refuse to leak message/stack — only the error class name (or the
+	// primitive type for non-Error throwables) is surfaced for ops triage.
 	return {
 		error: {
 			code: "internal_error",
 			message: "Internal error",
 			...(requestId ? { requestId } : {}),
+			retryable: observability.retryable,
+			source,
+			details: {
+				retryable: false,
+				category: "internal_error",
+				errorClass: error instanceof Error ? error.name : typeof error,
+			},
 		},
 	};
 }
 
-function publicProviderErrorDetails(error: ProviderError): unknown {
-	const providerDetails = error.details;
-	const observabilityDetails = providerObservabilityDetails(error);
-
-	if (providerDetails === undefined) {
-		return observabilityDetails;
-	}
-	if (observabilityDetails === undefined) {
-		return providerDetails;
-	}
-	if (isPlainRecord(providerDetails) && isPlainRecord(observabilityDetails)) {
-		return { ...providerDetails, ...observabilityDetails };
-	}
-	return {
-		provider: providerDetails,
-		observability: observabilityDetails,
-	};
-}
-
-function isPlainRecord(value: unknown): value is Record<string, unknown> {
-	return value !== null && typeof value === "object" && !Array.isArray(value);
-}
-
-function providerObservabilityDetails(error: ProviderError):
-	| {
-			category: ProviderErrorCategory;
-			taxonomyVersion: string;
-			retryable: boolean;
-			upstreamStatus?: number;
-	  }
-	| undefined {
+// Accepts `unknown` so the branded guards narrow cleanly from the top: the
+// subtype error classes are structurally compatible with ProviderError, so
+// narrowing from a ProviderError-typed value would collapse the negative branch
+// to `never`. Narrowing from unknown avoids that while still recognizing errors
+// from a duplicate SDK module instance.
+function providerObservabilityDetails(
+	error: unknown,
+	declaredErrorCode?: OperationErrorCode,
+): ErrorObservabilityDetails | undefined {
+	const declaredRetryable = sdkOwnsErrorResolution(error)
+		? undefined
+		: declaredErrorCode?.retryable;
 	// Session-expiry surfaces the credential_expired category + the opt-in
 	// retryable signal so Gateway/Credential Service can refresh and re-drive the
 	// operation (see design.md §4.3 D3). Without this branch the auth error would
 	// serialize as a bare 401 with no retryable/category, losing the refresh
 	// signal for exactly the retryOnAuthRefresh operations it is meant to enable.
-	if (error instanceof SessionExpiredError) {
+	if (isSessionExpiredError(error)) {
 		return {
 			category: error.options?.category ?? "credential_expired",
 			taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
-			retryable: error.options?.retryable ?? false,
+			retryable: error.options?.retryable ?? declaredRetryable ?? false,
+		};
+	}
+	// Missing-secret errors carry the canonical credential_unavailable category
+	// so Gateway/observability can attribute the failure to provisioning, not
+	// the upstream. Matched by code (not constructor) so both the SDK-owned
+	// runtime gate and any not-yet-migrated provider-thrown MISSING_SECRET
+	// serialize identically, including across duplicate SDK module instances.
+	if (isProviderError(error) && error.code === MISSING_SECRET_CODE) {
+		return {
+			category: error.options?.category ?? "credential_unavailable",
+			taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
+			retryable: error.options?.retryable ?? declaredRetryable ?? false,
 		};
 	}
-	if (!(error instanceof TransportError)) {
+	if (!isTransportError(error)) {
 		return undefined;
 	}
 	const isProxyPoolCode =
@@ -615,8 +825,86 @@ function providerObservabilityDetails(error: ProviderError):
 	};
 }
 
+function errorObservabilityDetails(
+	error: unknown,
+	declaredErrorCode?: OperationErrorCode,
+): ErrorObservabilityDetails {
+	const effectiveDeclaration = sdkOwnsErrorResolution(error) ? undefined : declaredErrorCode;
+	const providerDetails = providerObservabilityDetails(error, effectiveDeclaration);
+	if (providerDetails) return providerDetails;
+
+	if (error instanceof z.ZodError || isValidationError(error)) {
+		const declaredStatus = effectiveDeclaration?.status;
+		return {
+			category:
+				isProviderError(error) && error.options?.category
+					? error.options.category
+					: isEmittableErrorStatus(declaredStatus) &&
+							categoryForStatus(declaredStatus) === "upstream_rejected"
+						? "upstream_rejected"
+						: isEmittableErrorStatus(declaredStatus) && declaredStatus >= 500
+							? "provider_error"
+							: "input_validation",
+			taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
+			retryable: isProviderError(error)
+				? (error.options?.retryable ?? effectiveDeclaration?.retryable ?? false)
+				: false,
+		};
+	}
+
+	if (error instanceof StatefulRoutingDeadlineError) {
+		return {
+			category: "timeout",
+			taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
+			retryable: false,
+		};
+	}
+
+	if (isProviderError(error)) {
+		// Deterministic upstream refusals default to the rejection category:
+		// the UPSTREAM_REJECTED family and any operation-declared rejection
+		// status (409/410/422) classify as upstream_rejected unless the
+		// author set an explicit category.
+		const declaredStatus = effectiveDeclaration?.status;
+		const rejectionDefault =
+			error.code === "UPSTREAM_REJECTED" ||
+			(isEmittableErrorStatus(declaredStatus) &&
+				categoryForStatus(declaredStatus) === "upstream_rejected")
+				? ("upstream_rejected" as const)
+				: ("provider_error" as const);
+		return {
+			category: error.options?.category ?? rejectionDefault,
+			taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
+			retryable: error.options?.retryable ?? effectiveDeclaration?.retryable ?? false,
+		};
+	}
+
+	return {
+		category: "internal_error",
+		taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
+		retryable: false,
+	};
+}
+
+function responseWithErrorObservability(
+	response: Response,
+	error: unknown,
+	declaredErrorCode?: OperationErrorCode,
+): Response {
+	const headers = new Headers(response.headers);
+	headers.set(
+		ERROR_OBSERVABILITY_HEADER,
+		JSON.stringify(errorObservabilityDetails(error, declaredErrorCode)),
+	);
+	return new Response(response.body, {
+		status: response.status,
+		statusText: response.statusText,
+		headers,
+	});
+}
+
 function publicProviderErrorMessage(error: ProviderError): string {
-	if (error instanceof TransportError) {
+	if (isTransportError(error)) {
 		if (error.code === PROXY_AUTH_IP_DENIED_CODE) {
 			return error.message;
 		}
@@ -639,22 +927,35 @@ function publicProviderErrorMessage(error: ProviderError): string {
 	return error.message;
 }
 
-function toStatusCode(
-	error: unknown,
-): 400 | 401 | 404 | 429 | 500 | 502 | 503 | 504 {
+function isEmittableErrorStatus(value: unknown): value is ProviderErrorStatus {
+	return (
+		typeof value === "number" &&
+		VALID_OPERATION_ERROR_STATUSES.some((status) => status === value)
+	);
+}
+
+function toStatusCode(error: unknown, declaredErrorCode?: OperationErrorCode): ProviderErrorStatus {
 	if (error instanceof z.ZodError) {
 		return 400;
 	}
-
-	if (error instanceof TransportError) {
-		return error.code === "transport_timeout" ? 504 : 502;
+	if (error instanceof StatefulRoutingDeadlineError) {
+		return 504;
 	}
-
-	if (error instanceof ProviderError) {
+	if (isProviderError(error)) {
+		if (
+			!sdkOwnsErrorResolution(error) &&
+			isEmittableErrorStatus(declaredErrorCode?.status)
+		) {
+			return declaredErrorCode.status;
+		}
 		switch (error.code) {
 			case "AUTH_REQUIRED":
 			case "reauth_required":
 				return 401;
+			// Unprovisioned declared secret: a deployment/config defect, never an
+			// upstream failure — explicit 400 (was only reached via fallthrough).
+			case MISSING_SECRET_CODE:
+				return 400;
 			case "NOT_FOUND":
 			case "not_found":
 			case "NO_DATA":
@@ -663,20 +964,66 @@ function toStatusCode(
 			case "UPSTREAM_RATE_LIMIT":
 			case "LIMITED_NUMBER_OF_SERVICE_REQUESTS_EXCEEDS_ERROR":
 				return 429;
+			// Deterministic upstream business refusal (honest-provider-error-
+			// contract): the upstream evaluated the request and said no under
+			// its own rules — a conflict with upstream state, never a 5xx.
+			case "UPSTREAM_REJECTED":
+				return 409;
 			case "UPSTREAM_ERROR":
 			case "BLOCKED":
 				return 502;
 			case "STT_UNAVAILABLE":
 			case "UNSUPPORTED_STT_BACKEND":
+			case "STATEFUL_FORWARDING_REPLAY_CACHE_FULL":
 				return 503;
 		}
+		if (isTransportError(error)) {
+			return error.code === "transport_timeout" ? 504 : 502;
+		}
+		if (isValidationError(error)) {
+			return error.options?.category === "output_validation" ? 500 : 400;
+		}
 
-		return 400;
+		return 500;
 	}
 
 	return 500;
 }
 
+function sdkOwnsErrorResolution(error: unknown): boolean {
+	if (isSessionExpiredError(error)) return true;
+	if (isTransportError(error)) return true;
+	if (error instanceof z.ZodError) return true;
+	if (error instanceof StatefulRoutingDeadlineError) return true;
+	return (
+		isProviderError(error) &&
+		typeof error.code === "string" &&
+		SDK_RUNTIME_OWNED_ERROR_CODES.has(error.code)
+	);
+}
+
+type OperationErrorCodeLookup = ReadonlyMap<string, ReadonlyMap<string, OperationErrorCode>>;
+
+function buildOperationErrorCodeLookup(provider: ProviderDefinition): OperationErrorCodeLookup {
+	return new Map(
+		Object.entries(provider.operations).flatMap(([operationId, operation]) => {
+			const errorCodes = operation.docs?.errorCodes;
+			return errorCodes?.length
+				? [[operationId, new Map(errorCodes.map((entry) => [entry.code, entry]))] as const]
+				: [];
+		}),
+	);
+}
+
+function declaredErrorCodeFor(
+	error: unknown,
+	operationId: string | undefined,
+	lookup: OperationErrorCodeLookup,
+): OperationErrorCode | undefined {
+	if (!operationId || !isProviderError(error) || typeof error.code !== "string") return undefined;
+	return lookup.get(operationId)?.get(error.code);
+}
+
 function extractRequestId(raw: unknown): string | undefined {
 	if (!raw || typeof raw !== "object") {
 		return undefined;
@@ -695,21 +1042,26 @@ function logProviderError(
 	error: unknown,
 	status: number,
 	cost: ProviderRequestCost,
+	declaredErrorCode?: OperationErrorCode,
 ): void {
-	const code =
-		error instanceof ProviderError
-			? (error.code ?? "provider_error")
-			: error instanceof z.ZodError
-				? "invalid_request"
+	const code = isProviderError(error)
+		? (error.code ?? "provider_error")
+		: error instanceof z.ZodError
+			? "invalid_request"
+			: error instanceof StatefulRoutingDeadlineError
+				? "STATEFUL_FORWARDING_DEADLINE_EXPIRED"
 				: "internal_error";
 	const errorClass = error instanceof Error ? error.name : typeof error;
 	const message = error instanceof Error ? error.message : String(error);
-	const details =
-		error instanceof ProviderError
-			? providerObservabilityDetails(error)
-			: undefined;
-	const emit =
-		typeof logger === "function" ? logger : defaultProviderServerLogger;
+	const details = errorObservabilityDetails(error, declaredErrorCode);
+	const isUnregisteredProviderErrorCode =
+		status === 500 &&
+		isProviderError(error) &&
+		!isValidationError(error) &&
+		typeof error.code === "string" &&
+		!SDK_OWNED_PROVIDER_ERROR_CODES.has(error.code) &&
+		declaredErrorCode === undefined;
+	const emit = typeof logger === "function" ? logger : defaultProviderServerLogger;
 	emit({
 		level: status >= 500 ? "error" : "warn",
 		event: "provider_request_failed",
@@ -722,14 +1074,15 @@ function logProviderError(
 		code,
 		errorClass,
 		message,
-		...(error instanceof TransportError && error.upstreamStatus
-			? { upstreamStatus: error.upstreamStatus }
-			: {}),
-		...(details
+		...(details.upstreamStatus ? { upstreamStatus: details.upstreamStatus } : {}),
+		errorCategory: details.category,
+		taxonomyVersion: details.taxonomyVersion,
+		retryable: details.retryable,
+		...(isUnregisteredProviderErrorCode
 			? {
-					errorCategory: details.category,
-					taxonomyVersion: details.taxonomyVersion,
-					retryable: details.retryable,
+					signal: "unregistered_provider_error_code" as const,
+					signalFix:
+						"Declare this code (with status and retryable) in the operation's docs.errorCodes so it serves its intended status instead of 500.",
 				}
 			: {}),
 		...(error instanceof z.ZodError ? { issues: zodDetails(error) } : {}),
@@ -744,8 +1097,7 @@ function logProviderCleanupError(
 	resource: "browser" | "stealth",
 	error: unknown,
 ): void {
-	const emit =
-		typeof logger === "function" ? logger : defaultProviderServerLogger;
+	const emit = typeof logger === "function" ? logger : defaultProviderServerLogger;
 	const errorClass = error instanceof Error ? error.name : typeof error;
 	const message = error instanceof Error ? error.message : String(error);
 	emit({
@@ -770,8 +1122,7 @@ function logProviderSuccess(
 	status: number,
 	cost: ProviderRequestCost,
 ): void {
-	const emit =
-		typeof logger === "function" ? logger : defaultProviderServerLogger;
+	const emit = typeof logger === "function" ? logger : defaultProviderServerLogger;
 	emit({
 		level: "info",
 		event: "provider_request_completed",
@@ -817,18 +1168,13 @@ function toJsonSuccessResponse(
 	};
 }
 
-function isAsyncIterable<T = unknown>(
-	value: unknown,
-): value is AsyncIterable<T> {
+function isAsyncIterable<T = unknown>(value: unknown): value is AsyncIterable<T> {
 	if (!value || typeof value !== "object") return false;
 	const iterator = Reflect.get(value, Symbol.asyncIterator);
 	return typeof iterator === "function";
 }
 
-function responseWithCleanup(
-	response: Response,
-	cleanup: RequestCleanup,
-): Response {
+function responseWithCleanup(response: Response, cleanup: RequestCleanup): Response {
 	if (!response.body) {
 		void cleanup();
 		return response;
@@ -877,10 +1223,7 @@ async function validateSseEvent(
 	const transport = getSseTransport(operation);
 	const schema = transport?.events?.[event.event];
 	if (!schema) {
-		if (
-			event.event === APIFUSE_STREAM_ERROR_EVENT ||
-			event.event === APIFUSE_STREAM_DONE_EVENT
-		) {
+		if (event.event === APIFUSE_STREAM_ERROR_EVENT || event.event === APIFUSE_STREAM_DONE_EVENT) {
 			return event;
 		}
 		throw new ProviderError(
@@ -893,11 +1236,7 @@ async function validateSseEvent(
 			},
 		);
 	}
-	const data = await parseSchema(
-		schema,
-		event.data,
-		`transport.events.${event.event}`,
-	);
+	const data = await parseSchema(schema, event.data, `transport.events.${event.event}`);
 	return { ...event, data };
 }
 
@@ -917,8 +1256,7 @@ function assertStreamPayloadWithinLimit(
 	throw new ProviderError(
 		`Stream ${kind} exceeded declared byte limit (${actualBytes} > ${maxBytes}).`,
 		{
-			code:
-				kind === "event" ? "STREAM_EVENT_TOO_LARGE" : "STREAM_CHUNK_TOO_LARGE",
+			code: kind === "event" ? "STREAM_EVENT_TOO_LARGE" : "STREAM_CHUNK_TOO_LARGE",
 			retryable: false,
 			category: "input_validation",
 			fix:
@@ -963,15 +1301,10 @@ function toSseResponse(
 				const validated = await validateSseEvent(operation, next.value);
 				const encodedEvent = encodeSseEvent(validated);
 				const encodedBytes = encoder.encode(encodedEvent);
-				assertStreamPayloadWithinLimit(
-					encodedBytes.byteLength,
-					transport?.maxEventBytes,
-					"event",
-				);
+				assertStreamPayloadWithinLimit(encodedBytes.byteLength, transport?.maxEventBytes, "event");
 				controller.enqueue(encodedBytes);
 			} catch (error) {
-				const message =
-					error instanceof Error ? error.message : "Stream failed";
+				const message = error instanceof Error ? error.message : "Stream failed";
 				controller.enqueue(
 					encoder.encode(
 						encodeSseEvent(
@@ -1018,11 +1351,7 @@ function enforceStreamChunkLimit(
 					return;
 				}
 				if (value) {
-					assertStreamPayloadWithinLimit(
-						byteLength(value),
-						maxChunkBytes,
-						"chunk",
-					);
+					assertStreamPayloadWithinLimit(byteLength(value), maxChunkBytes, "chunk");
 					controller.enqueue(value);
 				}
 			} catch (error) {
@@ -1042,10 +1371,7 @@ function toStreamingResponse(
 	requestId?: string,
 ): Response {
 	const transport = operation.transport?.kind ?? "json";
-	if (
-		transport === "sse" &&
-		(result instanceof Response || result instanceof ReadableStream)
-	) {
+	if (transport === "sse" && (result instanceof Response || result instanceof ReadableStream)) {
 		void cleanup();
 		throw new ProviderError(
 			"SSE operations must return an AsyncIterable of typed stream.event(...) values.",
@@ -1059,20 +1385,13 @@ function toStreamingResponse(
 	}
 	if (result instanceof Response) {
 		const httpTransport = getHttpStreamTransport(operation);
-		if (
-			httpTransport &&
-			result.body &&
-			httpTransport?.maxChunkBytes !== undefined
-		) {
+		if (httpTransport && result.body && httpTransport?.maxChunkBytes !== undefined) {
 			return responseWithCleanup(
-				new Response(
-					enforceStreamChunkLimit(result.body, httpTransport.maxChunkBytes),
-					{
-						headers: result.headers,
-						status: result.status,
-						statusText: result.statusText,
-					},
-				),
+				new Response(enforceStreamChunkLimit(result.body, httpTransport.maxChunkBytes), {
+					headers: result.headers,
+					status: result.status,
+					statusText: result.statusText,
+				}),
 				cleanup,
 			);
 		}
@@ -1092,8 +1411,7 @@ function toStreamingResponse(
 						: {
 								"Content-Type":
 									operation.transport?.kind === "http-stream"
-										? (operation.transport.contentType ??
-											"application/octet-stream")
+										? (operation.transport.contentType ?? "application/octet-stream")
 										: "application/octet-stream",
 							},
 			}),
@@ -1113,18 +1431,14 @@ function toStreamingResponse(
 	);
 }
 
-function getSseTransport(
-	operation: OperationDefinition,
-): OperationSseTransport | undefined {
+function getSseTransport(operation: OperationDefinition): OperationSseTransport | undefined {
 	return operation.transport?.kind === "sse" ? operation.transport : undefined;
 }
 
 function getHttpStreamTransport(
 	operation: OperationDefinition,
 ): OperationHttpStreamTransport | undefined {
-	return operation.transport?.kind === "http-stream"
-		? operation.transport
-		: undefined;
+	return operation.transport?.kind === "http-stream" ? operation.transport : undefined;
 }
 
 function toAuthFlowResponse(
@@ -1145,9 +1459,7 @@ function toAuthFlowResponse(
 	};
 }
 
-function authFlowLocaleFromHeaders(
-	headers?: Record<string, string>,
-): ProviderLocale {
+function authFlowLocaleFromHeaders(headers?: Record<string, string>): ProviderLocale {
 	const header = Object.entries(headers ?? {}).find(
 		([key]) => key.toLowerCase() === "accept-language",
 	)?.[1];
@@ -1165,9 +1477,7 @@ function isAuthFlowLocale(value: string | undefined): value is ProviderLocale {
 }
 
 function isAuthTurn(value: unknown): value is AuthTurn {
-	return (
-		!!value && typeof value === "object" && "kind" in value && "turnId" in value
-	);
+	return !!value && typeof value === "object" && "kind" in value && "turnId" in value;
 }
 
 function loadAuthFlowLocaleCatalogs(
@@ -1204,10 +1514,7 @@ function materializeAuthFlowTurn(
 	});
 }
 
-function withAuthRequestHeaders(
-	request: AuthFlowRequest,
-	headers: Headers,
-): AuthFlowRequest {
+function withAuthRequestHeaders(request: AuthFlowRequest, headers: Headers): AuthFlowRequest {
 	return {
 		...request,
 		headers: {
@@ -1225,17 +1532,9 @@ async function handleOperation(
 	state: ProviderRuntimeState = createUnsupportedProviderRuntimeState(),
 	proxyTelemetry?: ProxyTelemetryCollector,
 ): Promise<Response | OperationResponse> {
-	const ctx = createProviderContext(
-		provider,
-		request,
-		operationId,
-		options,
-		state,
-		proxyTelemetry,
-	);
+	const ctx = createProviderContext(provider, request, operationId, options, state, proxyTelemetry);
 	const operation = provider.operations[operationId];
-	const streaming =
-		operation?.transport?.kind && operation.transport.kind !== "json";
+	const streaming = operation?.transport?.kind && operation.transport.kind !== "json";
 	let cleanupCalled = false;
 	const cleanup = async () => {
 		if (cleanupCalled) return;
@@ -1266,12 +1565,14 @@ async function handleOperation(
 		}
 	};
 	try {
-		const result = await executeOperation(
-			provider,
-			operationId,
-			ctx,
-			request.input,
-		);
+		const result = options.operationExecutor
+			? await options.operationExecutor({
+					provider,
+					operationId,
+					ctx,
+					request,
+				})
+			: await executeOperation(provider, operationId, ctx, request.input);
 		if (streaming && operation) {
 			return toStreamingResponse(operation, result, cleanup, request.requestId);
 		}
@@ -1315,13 +1616,16 @@ async function handleAuthFlow(
 		});
 	}
 
-	const { context, getPatch } = createAuthFlowContext(
-		provider,
-		request,
-		options,
-		signal,
-	);
+	// Same SDK-owned gate as executeOperation: OAuth/credentials ceremonies
+	// depend on declared secrets (client ids/secrets), so fail structured before
+	// any flow code runs instead of at whatever point the ceremony first reads
+	// the env. `abort` stays exempt: a user must always be able to cancel a
+	// stranded flow even when provisioning is broken.
+	const { context, getPatch } = createAuthFlowContext(provider, request, options, signal);
 	try {
+		if (route !== "abort") {
+			assertRequiredSecretsPresent(provider, context.env);
+		}
 		const result =
 			route === "start"
 				? await flow.start(context)
@@ -1363,12 +1667,143 @@ async function handleAuthFlow(
 	}
 }
 
+class StatefulForwardingReplayCache {
+	readonly #nonces = new Map<string, number>();
+	readonly #expiryBuckets = new Map<number, Set<string>>();
+	#nextExpiryBucket?: number;
+	#latestExpiryBucket?: number;
+
+	constructor(private readonly maxEntries: number) {}
+
+	claim(nonce: string, expiresAtMs: number, nowMs: number): "accepted" | "replayed" | "full" {
+		this.dropExpiredBuckets(nowMs);
+		if (this.#nonces.has(nonce)) return "replayed";
+		if (this.#nonces.size >= this.maxEntries) return "full";
+		const expiryBucket =
+			Math.ceil(expiresAtMs / STATEFUL_FORWARDING_REPLAY_BUCKET_MS) *
+			STATEFUL_FORWARDING_REPLAY_BUCKET_MS;
+		this.#nonces.set(nonce, expiryBucket);
+		const bucket = this.#expiryBuckets.get(expiryBucket) ?? new Set<string>();
+		bucket.add(nonce);
+		this.#expiryBuckets.set(expiryBucket, bucket);
+		this.#nextExpiryBucket = Math.min(this.#nextExpiryBucket ?? expiryBucket, expiryBucket);
+		this.#latestExpiryBucket = Math.max(this.#latestExpiryBucket ?? expiryBucket, expiryBucket);
+		return "accepted";
+	}
+
+	private dropExpiredBuckets(nowMs: number): void {
+		if (this.#nextExpiryBucket === undefined || this.#latestExpiryBucket === undefined) return;
+		if (nowMs >= this.#latestExpiryBucket) {
+			this.#nonces.clear();
+			this.#expiryBuckets.clear();
+			this.#nextExpiryBucket = undefined;
+			this.#latestExpiryBucket = undefined;
+			return;
+		}
+		while (this.#nextExpiryBucket <= nowMs) {
+			const bucket = this.#expiryBuckets.get(this.#nextExpiryBucket);
+			if (bucket) {
+				for (const cachedNonce of bucket) this.#nonces.delete(cachedNonce);
+				this.#expiryBuckets.delete(this.#nextExpiryBucket);
+			}
+			this.#nextExpiryBucket += STATEFUL_FORWARDING_REPLAY_BUCKET_MS;
+		}
+	}
+}
+
+function verifyStatefulForwardingRequest(input: {
+	readonly options: ProviderServerOptions;
+	readonly rawBody: string;
+	readonly headers: Headers;
+	readonly method: string;
+	readonly path: string;
+	readonly replayCache: StatefulForwardingReplayCache;
+}): void {
+	const config = input.options.statefulForwarding;
+	if (!config?.secret) {
+		throw new ProviderError("Stateful forwarding is not configured.", {
+			code: "STATEFUL_FORWARDING_NOT_CONFIGURED",
+		});
+	}
+	const timestamp = input.headers.get(STATEFUL_FORWARDING_TIMESTAMP_HEADER) ?? "";
+	const signature = input.headers.get(STATEFUL_FORWARDING_SIGNATURE_HEADER) ?? "";
+	const nonce = input.headers.get(STATEFUL_FORWARDING_NONCE_HEADER) ?? "";
+	if (!timestamp || !signature || !nonce) {
+		throw new ProviderError("Stateful forwarding signature headers are missing.", {
+			code: "STATEFUL_FORWARDING_SIGNATURE_MISSING",
+		});
+	}
+	if (nonce.length > 256) {
+		throw new ProviderError("Stateful forwarding nonce is invalid.", {
+			code: "STATEFUL_FORWARDING_NONCE_INVALID",
+		});
+	}
+	const timestampMs = Date.parse(timestamp);
+	const maxSkewMs = config.maxSkewMs ?? DEFAULT_STATEFUL_FORWARDING_MAX_SKEW_MS;
+	if (!Number.isFinite(timestampMs) || Math.abs(Date.now() - timestampMs) > maxSkewMs) {
+		throw new ProviderError(
+			"Stateful forwarding signature timestamp is outside the allowed skew.",
+			{ code: "STATEFUL_FORWARDING_TIMESTAMP_INVALID" },
+		);
+	}
+	if (
+		!verifyStatefulRequestSignature({
+			secret: config.secret,
+			timestamp,
+			rawBody: input.rawBody,
+			method: input.method,
+			path: input.path,
+			nonce,
+			signature,
+		})
+	) {
+		throw new ProviderError("Stateful forwarding signature is invalid.", {
+			code: "STATEFUL_FORWARDING_SIGNATURE_INVALID",
+		});
+	}
+	const replayResult = input.replayCache.claim(nonce, timestampMs + maxSkewMs, Date.now());
+	if (replayResult === "replayed") {
+		throw new ProviderError("Stateful forwarding nonce has already been used.", {
+			code: "STATEFUL_FORWARDING_REPLAY_DETECTED",
+		});
+	}
+	if (replayResult === "full") {
+		throw new ProviderError("Stateful forwarding replay cache is at capacity.", {
+			code: "STATEFUL_FORWARDING_REPLAY_CACHE_FULL",
+		});
+	}
+}
+
+function operationRequestFromForwardingEnvelope(
+	envelope: ProviderServerStatefulForwardEnvelope,
+): OperationRequest & { readonly deadlineAt?: string } {
+	return {
+		...envelope.operationRequest,
+		...(envelope.deadlineAt !== undefined ? { deadlineAt: envelope.deadlineAt } : {}),
+	};
+}
+
+function parseStatefulForwardingEnvelope(rawBody: unknown): ProviderServerStatefulForwardEnvelope {
+	const parsed = ProviderServerStatefulForwardEnvelopeSchema.safeParse(rawBody);
+	if (parsed.success) return parsed.data;
+	throw new ProviderError("Stateful forwarding envelope is invalid.", {
+		code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
+		details: zodDetails(parsed.error),
+	});
+}
+
 export function createServerApp(
 	provider: ProviderDefinition,
 	options: ProviderServerOptions = {},
 ): Hono {
+	validateStatefulServerConfig(options);
 	const app = new Hono();
 	const logger = options.logger ?? defaultProviderServerLogger;
+	const operationErrorCodes = buildOperationErrorCodeLookup(provider);
+	const statefulForwardingReplayCache = new StatefulForwardingReplayCache(
+		options.statefulForwarding?.replayCacheMaxEntries ??
+			DEFAULT_STATEFUL_FORWARDING_REPLAY_CACHE_MAX_ENTRIES,
+	);
 	const state =
 		options.state ??
 		createProviderRuntimeStateFromEnv({
@@ -1376,17 +1811,28 @@ export function createServerApp(
 			allowMemoryFallback: options.allowMemoryStateFallback === true,
 		});
 
-	app.notFound((c) =>
-		c.json(
-			{
-				error: {
-					code: "not_found",
-					message: "Not found",
-				},
-			},
-			404,
-		),
+	// Boot-time visibility for unprovisioned declared secrets: emit a structured
+	// warn so deploy tooling/alerting sees the gap the moment the pod boots,
+	// instead of discovering it request-by-request. Deliberately log-only — a
+	// boot crash would trade a structured MISSING_SECRET signal for
+	// CrashLoopBackOff. Requests still fail closed via the executeOperation gate.
+	const missingSecretsAtBoot = listMissingRequiredSecrets(
+		provider,
+		createEnvContext(provider.secrets?.map((secret) => secret.name)),
 	);
+	if (missingSecretsAtBoot.length > 0) {
+		logger({
+			level: "warn",
+			event: "provider_secrets_missing",
+			providerId: provider.id,
+			missingSecrets: missingSecretsAtBoot,
+		});
+	}
+
+	app.notFound((c) => {
+		const error = new ProviderError("Not found", { code: "not_found", retryable: false });
+		return responseWithErrorObservability(c.json(toErrorResponse(error), 404), error);
+	});
 
 	app.get("/health", (c) =>
 		c.json({
@@ -1396,6 +1842,169 @@ export function createServerApp(
 		}),
 	);
 
+	app.post(STATEFUL_INTERNAL_OPERATIONS_ROUTE, async (c) => {
+		let rawBodyText = "";
+		let rawBody: unknown;
+		let operationId: string | undefined;
+		const operation = "stateful-internal";
+		const requestCost = startRequestCost();
+		try {
+			if (!options.internalOperationExecutor) {
+				throw new ProviderError("Stateful internal operation executor is not configured.", {
+					code: "STATEFUL_INTERNAL_EXECUTOR_NOT_CONFIGURED",
+				});
+			}
+			rawBodyText = await c.req.raw.clone().text();
+			verifyStatefulForwardingRequest({
+				options,
+				rawBody: rawBodyText,
+				headers: c.req.raw.headers,
+				method: c.req.raw.method,
+				path: STATEFUL_INTERNAL_OPERATIONS_ROUTE,
+				replayCache: statefulForwardingReplayCache,
+			});
+			try {
+				rawBody = JSON.parse(rawBodyText);
+			} catch {
+				throw new ProviderError("Stateful forwarding envelope is not valid JSON.", {
+					code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
+				});
+			}
+			const envelope = parseStatefulForwardingEnvelope(rawBody);
+			if (envelope.providerId !== provider.id) {
+				throw new ProviderError(
+					"Stateful forwarding envelope providerId does not match the served provider.",
+					{ code: "STATEFUL_FORWARDING_PROVIDER_MISMATCH" },
+				);
+			}
+			if (envelope.requestId !== envelope.operationRequest.requestId) {
+				throw new ProviderError("Stateful forwarding requestId values do not match.", {
+					code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
+				});
+			}
+			if (
+				envelope.sourcePodId !==
+				(c.req.raw.headers.get(STATEFUL_FORWARDING_SOURCE_POD_HEADER) ?? "")
+			) {
+				throw new ProviderError("Stateful forwarding source pod does not match its header.", {
+					code: "STATEFUL_FORWARDING_SOURCE_POD_MISMATCH",
+				});
+			}
+			if (
+				envelope.forwardedAt !== (c.req.raw.headers.get(STATEFUL_FORWARDING_TIMESTAMP_HEADER) ?? "")
+			) {
+				throw new ProviderError(
+					"Stateful forwarding forwardedAt does not match its signature timestamp.",
+					{ code: "STATEFUL_FORWARDING_ENVELOPE_INVALID" },
+				);
+			}
+			const deadlineAtMs = envelope.deadlineAt ? Date.parse(envelope.deadlineAt) : undefined;
+			if (deadlineAtMs !== undefined && deadlineAtMs <= Date.now()) {
+				throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt as string);
+			}
+			const remainingDeadlineMs =
+				deadlineAtMs === undefined ? undefined : deadlineAtMs - Date.now();
+			const deadlineSignal =
+				remainingDeadlineMs === undefined ? undefined : AbortSignal.timeout(remainingDeadlineMs);
+			const signal = deadlineSignal
+				? AbortSignal.any([c.req.raw.signal, deadlineSignal])
+				: c.req.raw.signal;
+			const ownerFenceValidation = Promise.resolve(
+				options.statefulForwarding?.validateOwnerFence(
+					{
+						providerId: envelope.providerId,
+						sessionKey: envelope.sessionKey,
+						ownerPodId: envelope.ownerPodId,
+						generation: envelope.generation,
+						sourcePodId: envelope.sourcePodId,
+						forwardedAt: envelope.forwardedAt,
+						requestId: envelope.requestId,
+						...(envelope.idempotencyKey ? { idempotencyKey: envelope.idempotencyKey } : {}),
+					},
+					signal,
+				),
+			);
+			let ownerFenceValid: boolean | undefined;
+			try {
+				ownerFenceValid = deadlineSignal
+					? await Promise.race([
+							ownerFenceValidation,
+							new Promise<never>((_resolve, reject) => {
+								deadlineSignal.addEventListener(
+									"abort",
+									() =>
+										reject(
+											new StatefulRoutingDeadlineError(
+												envelope.requestId,
+												envelope.deadlineAt as string,
+											),
+										),
+									{ once: true },
+								);
+							}),
+						])
+					: await ownerFenceValidation;
+			} catch (error) {
+				if (deadlineSignal?.aborted) {
+					throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt as string);
+				}
+				throw error;
+			}
+			if (ownerFenceValid !== true) {
+				throw new ProviderError("Stateful forwarding owner fence is no longer current.", {
+					code: "STATEFUL_FORWARDING_OWNER_FENCE_INVALID",
+				});
+			}
+			const request = operationRequestFromForwardingEnvelope(envelope);
+			operationId = envelope.operationId;
+			const ctx = createProviderContext(provider, request, operationId, options, state);
+			if (deadlineAtMs !== undefined && deadlineAtMs <= Date.now()) {
+				throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt as string);
+			}
+			const output = await options.internalOperationExecutor({
+				provider,
+				operationId,
+				ctx,
+				request,
+				internalStatefulForward: envelope,
+				signal,
+			});
+			logProviderSuccess(
+				logger,
+				provider,
+				"operation",
+				operationId || operation,
+				request.requestId,
+				200,
+				finishRequestCost(requestCost),
+			);
+			return c.json({ data: output });
+		} catch (error) {
+			const declaredErrorCode = declaredErrorCodeFor(error, operationId, operationErrorCodes);
+			const status = toStatusCode(error, declaredErrorCode);
+			if (isProviderError(error) && error.code === "STATEFUL_FORWARDING_REPLAY_CACHE_FULL") {
+				c.header("Retry-After", String(STATEFUL_FORWARDING_REPLAY_RETRY_AFTER_SECONDS));
+			}
+			const requestId = extractRequestId(rawBody);
+			logProviderError(
+				logger,
+				provider,
+				"operation",
+				operationId || operation,
+				requestId,
+				error,
+				status,
+				finishRequestCost(requestCost),
+				declaredErrorCode,
+			);
+			return responseWithErrorObservability(
+				c.json(toErrorResponse(error, requestId, declaredErrorCode), status),
+				error,
+				declaredErrorCode,
+			);
+		}
+	});
+
 	app.post("/v1/:operation", async (c) => {
 		let rawBody: unknown;
 		const operation = c.req.param("operation");
@@ -1442,7 +2051,8 @@ export function createServerApp(
 			);
 			return c.json(response);
 		} catch (error) {
-			const status = toStatusCode(error);
+			const declaredErrorCode = declaredErrorCodeFor(error, operation, operationErrorCodes);
+			const status = toStatusCode(error, declaredErrorCode);
 			const requestId = extractRequestId(rawBody);
 			logProviderError(
 				logger,
@@ -1453,10 +2063,15 @@ export function createServerApp(
 				error,
 				status,
 				finishRequestCost(requestCost),
+				declaredErrorCode,
 			);
 			const telemetryHeader = proxyTelemetry.toHeaderValue();
 			if (telemetryHeader) c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
-			return c.json(toErrorResponse(error, requestId), status);
+			return responseWithErrorObservability(
+				c.json(toErrorResponse(error, requestId, declaredErrorCode), status),
+				error,
+				declaredErrorCode,
+			);
 		}
 	});
 
@@ -1468,17 +2083,8 @@ export function createServerApp(
 				.clone()
 				.json()
 				.catch(() => undefined);
-			const body = withAuthRequestHeaders(
-				AuthFlowRequestSchema.parse(rawBody),
-				c.req.raw.headers,
-			);
-			const response = await handleAuthFlow(
-				provider,
-				body,
-				"start",
-				options,
-				c.req.raw.signal,
-			);
+			const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
+			const response = await handleAuthFlow(provider, body, "start", options, c.req.raw.signal);
 			logProviderSuccess(
 				logger,
 				provider,
@@ -1502,7 +2108,10 @@ export function createServerApp(
 				status,
 				finishRequestCost(requestCost),
 			);
-			return c.json(toErrorResponse(error, requestId), status);
+			return responseWithErrorObservability(
+				c.json(toErrorResponse(error, requestId), status),
+				error,
+			);
 		}
 	});
 
@@ -1514,17 +2123,8 @@ export function createServerApp(
 				.clone()
 				.json()
 				.catch(() => undefined);
-			const body = withAuthRequestHeaders(
-				AuthFlowRequestSchema.parse(rawBody),
-				c.req.raw.headers,
-			);
-			const response = await handleAuthFlow(
-				provider,
-				body,
-				"continue",
-				options,
-				c.req.raw.signal,
-			);
+			const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
+			const response = await handleAuthFlow(provider, body, "continue", options, c.req.raw.signal);
 			logProviderSuccess(
 				logger,
 				provider,
@@ -1548,7 +2148,10 @@ export function createServerApp(
 				status,
 				finishRequestCost(requestCost),
 			);
-			return c.json(toErrorResponse(error, requestId), status);
+			return responseWithErrorObservability(
+				c.json(toErrorResponse(error, requestId), status),
+				error,
+			);
 		}
 	});
 
@@ -1560,17 +2163,8 @@ export function createServerApp(
 				.clone()
 				.json()
 				.catch(() => undefined);
-			const body = withAuthRequestHeaders(
-				AuthFlowRequestSchema.parse(rawBody),
-				c.req.raw.headers,
-			);
-			const response = await handleAuthFlow(
-				provider,
-				body,
-				"poll",
-				options,
-				c.req.raw.signal,
-			);
+			const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
+			const response = await handleAuthFlow(provider, body, "poll", options, c.req.raw.signal);
 			logProviderSuccess(
 				logger,
 				provider,
@@ -1594,7 +2188,10 @@ export function createServerApp(
 				status,
 				finishRequestCost(requestCost),
 			);
-			return c.json(toErrorResponse(error, requestId), status);
+			return responseWithErrorObservability(
+				c.json(toErrorResponse(error, requestId), status),
+				error,
+			);
 		}
 	});
 
@@ -1606,17 +2203,8 @@ export function createServerApp(
 				.clone()
 				.json()
 				.catch(() => undefined);
-			const body = withAuthRequestHeaders(
-				AuthFlowRequestSchema.parse(rawBody),
-				c.req.raw.headers,
-			);
-			const response = await handleAuthFlow(
-				provider,
-				body,
-				"refresh",
-				options,
-				c.req.raw.signal,
-			);
+			const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
+			const response = await handleAuthFlow(provider, body, "refresh", options, c.req.raw.signal);
 			logProviderSuccess(
 				logger,
 				provider,
@@ -1640,7 +2228,10 @@ export function createServerApp(
 				status,
 				finishRequestCost(requestCost),
 			);
-			return c.json(toErrorResponse(error, requestId), status);
+			return responseWithErrorObservability(
+				c.json(toErrorResponse(error, requestId), status),
+				error,
+			);
 		}
 	});
 
@@ -1652,17 +2243,8 @@ export function createServerApp(
 				.clone()
 				.json()
 				.catch(() => undefined);
-			const body = withAuthRequestHeaders(
-				AuthFlowRequestSchema.parse(rawBody),
-				c.req.raw.headers,
-			);
-			const response = await handleAuthFlow(
-				provider,
-				body,
-				"abort",
-				options,
-				c.req.raw.signal,
-			);
+			const body = withAuthRequestHeaders(AuthFlowRequestSchema.parse(rawBody), c.req.raw.headers);
+			const response = await handleAuthFlow(provider, body, "abort", options, c.req.raw.signal);
 			logProviderSuccess(
 				logger,
 				provider,
@@ -1686,19 +2268,64 @@ export function createServerApp(
 				status,
 				finishRequestCost(requestCost),
 			);
-			return c.json(toErrorResponse(error, requestId), status);
+			return responseWithErrorObservability(
+				c.json(toErrorResponse(error, requestId), status),
+				error,
+			);
 		}
 	});
 
 	return app;
 }
 
+function validateStatefulServerConfig(options: ProviderServerOptions): void {
+	if (options.statefulForwarding && !options.internalOperationExecutor) {
+		throw new Error(
+			"Invalid provider server configuration: statefulForwarding requires internalOperationExecutor; missing option internalOperationExecutor.",
+		);
+	}
+	if (options.internalOperationExecutor && !options.statefulForwarding?.secret) {
+		throw new Error(
+			"Invalid provider server configuration: internalOperationExecutor requires statefulForwarding.secret; missing option statefulForwarding.secret.",
+		);
+	}
+	if (
+		options.statefulForwarding &&
+		typeof options.statefulForwarding.validateOwnerFence !== "function"
+	) {
+		throw new Error(
+			"Invalid provider server configuration: statefulForwarding requires validateOwnerFence.",
+		);
+	}
+	if (
+		options.statefulForwarding?.maxSkewMs !== undefined &&
+		(!Number.isFinite(options.statefulForwarding.maxSkewMs) ||
+			options.statefulForwarding.maxSkewMs <= 0)
+	) {
+		throw new Error("Invalid provider server configuration: maxSkewMs must be positive.");
+	}
+	if (
+		options.statefulForwarding?.replayCacheMaxEntries !== undefined &&
+		(!Number.isInteger(options.statefulForwarding.replayCacheMaxEntries) ||
+			options.statefulForwarding.replayCacheMaxEntries <= 0)
+	) {
+		throw new Error(
+			"Invalid provider server configuration: replayCacheMaxEntries must be a positive integer.",
+		);
+	}
+}
+
 type BunServeRuntime = {
 	serve: (options: {
 		port: number;
 		hostname: string;
 		fetch: (request: Request) => Response | Promise<Response>;
-	}) => unknown;
+	}) => BunServerHandle;
+};
+
+type BunServerHandle = {
+	readonly port: number;
+	stop(closeActiveConnections?: boolean): Promise<void>;
 };
 
 function getBunServeRuntime(): BunServeRuntime | undefined {
@@ -1714,11 +2341,20 @@ function getBunServeRuntime(): BunServeRuntime | undefined {
 
 	return {
 		serve(options) {
-			return serve(options);
+			return serve(options) as BunServerHandle;
 		},
 	};
 }
 
+export type ProviderServerCloseOptions = {
+	readonly timeoutMs?: number;
+};
+
+export type ProviderServerHandle = {
+	readonly port: number;
+	close(options?: ProviderServerCloseOptions): Promise<void>;
+};
+
 export interface ServeOptions extends ProviderServerOptions {
 	host?: string;
 	port?: number;
@@ -1730,46 +2366,233 @@ export interface ServeOptions extends ProviderServerOptions {
 	selfTestPort?: number;
 }
 
+const DEFAULT_SHUTDOWN_TIMEOUT_MS = 30_000;
+const DEFAULT_SHUTDOWN_SIGNALS: NodeJS.Signals[] = ["SIGTERM", "SIGINT"];
+
+type SignalServerRegistration = {
+	readonly signals: ReadonlySet<NodeJS.Signals>;
+	readonly close: () => Promise<void>;
+};
+
+type ProcessSignalCoordinator = {
+	readonly registrations: Set<SignalServerRegistration>;
+	readonly listener: () => void;
+	handling: boolean;
+};
+
+const processSignalCoordinators = new Map<NodeJS.Signals, ProcessSignalCoordinator>();
+
 export async function serve(
 	provider: ProviderDefinition,
 	options: ServeOptions = {},
-): Promise<void> {
+): Promise<ProviderServerHandle> {
 	const bunRuntime = getBunServeRuntime();
 
 	if (bunRuntime === undefined) {
-		throw new ProviderError(
-			"Bun runtime is required to start the provider server",
-			{
-				code: "RUNTIME_UNSUPPORTED",
-			},
-		);
+		throw new ProviderError("Bun runtime is required to start the provider server", {
+			code: "RUNTIME_UNSUPPORTED",
+		});
 	}
+	const logger = options.logger ?? defaultProviderServerLogger;
+	const configuredTimeoutMs = shutdownTimeout(
+		options.shutdown?.timeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS,
+	);
+	const configuredSignals = resolveShutdownSignals(options.shutdown?.signals ?? true);
 
 	const app = createServerApp(provider, {
 		logger: options.logger,
 		stt: options.stt,
+		state: options.state,
+		allowMemoryStateFallback: options.allowMemoryStateFallback,
+		operationExecutor: options.operationExecutor,
+		internalOperationExecutor: options.internalOperationExecutor,
+		statefulForwarding: options.statefulForwarding,
+	});
+
+	const servers: BunServerHandle[] = [];
+	try {
+		servers.push(
+			bunRuntime.serve({
+				port: options.port ?? DEFAULT_PORT,
+				hostname: options.host ?? DEFAULT_HOST,
+				fetch: app.fetch,
+			}),
+		);
+
+		// Internal self-test listener (health dependency inversion): a SEPARATE
+		// socket the tenant-facing gateway never dials. Off by default — it only
+		// starts when the shared self-test master secret env is present.
+		const selfTestSecrets = resolveSelfTestMasterSecrets();
+		if (selfTestSecrets) {
+			const selfTestApp = createSelfTestApp(provider, {
+				secrets: selfTestSecrets,
+				invoke: createSelfTestInvoke(app),
+				authFlow: createSelfTestAuthFlowInvoke(app),
+			});
+			servers.push(
+				bunRuntime.serve({
+					port: options.selfTestPort ?? resolveSelfTestPort(),
+					hostname: options.host ?? DEFAULT_HOST,
+					fetch: selfTestApp.fetch,
+				}),
+			);
+		}
+	} catch (error) {
+		await Promise.allSettled(servers.map((startedServer) => startedServer.stop(true)));
+		throw error;
+	}
+
+	const server = servers[0];
+	if (!server) throw new Error("Provider server failed to create its primary listener.");
+	let closePromise: Promise<void> | undefined;
+	let unregisterSignals = () => {};
+
+	const close = (closeOptions: ProviderServerCloseOptions = {}): Promise<void> => {
+		if (closePromise) return closePromise;
+		const timeoutMs = shutdownTimeout(closeOptions.timeoutMs ?? configuredTimeoutMs);
+		closePromise = closeProviderServers({
+			servers,
+			hooks: options.shutdown?.hooks ?? [],
+			timeoutMs,
+			logger,
+			providerId: provider.id,
+		}).finally(() => unregisterSignals());
+		return closePromise;
+	};
+
+	try {
+		unregisterSignals = registerForProcessSignals(configuredSignals, () =>
+			close({ timeoutMs: configuredTimeoutMs }),
+		);
+	} catch (error) {
+		unregisterSignals();
+		await Promise.allSettled(servers.map((startedServer) => startedServer.stop(true)));
+		throw error;
+	}
+
+	return { port: server.port, close };
+}
+
+function registerForProcessSignals(
+	signals: NodeJS.Signals[],
+	close: () => Promise<void>,
+): () => void {
+	if (signals.length === 0) return () => {};
+	const registration: SignalServerRegistration = {
+		signals: new Set(signals),
+		close,
+	};
+	let registered = true;
+	const unregister = () => {
+		if (!registered) return;
+		registered = false;
+		for (const signal of registration.signals) {
+			const coordinator = processSignalCoordinators.get(signal);
+			if (!coordinator) continue;
+			coordinator.registrations.delete(registration);
+			if (coordinator.registrations.size === 0 && !coordinator.handling) {
+				process.removeListener(signal, coordinator.listener);
+				processSignalCoordinators.delete(signal);
+			}
+		}
+	};
+	try {
+		for (const signal of signals) {
+			let coordinator = processSignalCoordinators.get(signal);
+			if (!coordinator) {
+				const created: ProcessSignalCoordinator = {
+					registrations: new Set(),
+					handling: false,
+					listener: () => handleCoordinatedSignal(signal, created),
+				};
+				coordinator = created;
+				processSignalCoordinators.set(signal, coordinator);
+				process.on(signal, coordinator.listener);
+			}
+			coordinator.registrations.add(registration);
+		}
+	} catch (error) {
+		unregister();
+		throw error;
+	}
+	return unregister;
+}
+
+function handleCoordinatedSignal(
+	signal: NodeJS.Signals,
+	coordinator: ProcessSignalCoordinator,
+): void {
+	if (coordinator.handling) return;
+	coordinator.handling = true;
+	const registrations = [...coordinator.registrations];
+	void Promise.allSettled(registrations.map((registration) => registration.close())).finally(() => {
+		if (processSignalCoordinators.get(signal) === coordinator) {
+			process.removeListener(signal, coordinator.listener);
+			processSignalCoordinators.delete(signal);
+		}
+		try {
+			process.kill(process.pid, signal);
+		} catch {
+			process.exitCode = 1;
+		}
 	});
+}
+
+async function closeProviderServers(input: {
+	readonly servers: BunServerHandle[];
+	readonly hooks: Array<() => Promise<void>>;
+	readonly timeoutMs: number;
+	readonly logger: ProviderServerLogger;
+	readonly providerId: string;
+}): Promise<void> {
+	const deadline = Date.now() + input.timeoutMs;
+	const gracefulStops = input.servers.map((server) => server.stop(false));
+	for (const gracefulStop of gracefulStops) gracefulStop.catch(() => undefined);
+	for (const [hookIndex, hook] of input.hooks.entries()) {
+		try {
+			await withinShutdownBudget(Promise.resolve().then(hook), deadline);
+		} catch (error) {
+			try {
+				input.logger({
+					level: "error",
+					event: "provider_shutdown_hook_failed",
+					providerId: input.providerId,
+					hookIndex,
+					errorClass: error instanceof Error ? error.name : "UnknownError",
+					message: error instanceof Error ? error.message : "Shutdown hook failed.",
+				});
+			} catch {}
+		}
+	}
+	const forcedStops = input.servers.map((server) => server.stop(true));
+	await withinShutdownBudget(
+		Promise.allSettled([...gracefulStops, ...forcedStops]).then(() => undefined),
+		deadline,
+	).catch(() => undefined);
+}
 
-	bunRuntime.serve({
-		port: options.port ?? DEFAULT_PORT,
-		hostname: options.host ?? DEFAULT_HOST,
-		fetch: app.fetch,
+async function withinShutdownBudget<T>(promise: Promise<T>, deadline: number): Promise<T> {
+	promise.catch(() => undefined);
+	const remainingMs = Math.max(0, deadline - Date.now());
+	let timer: ReturnType<typeof setTimeout> | undefined;
+	const timeout = new Promise<never>((_resolve, reject) => {
+		timer = setTimeout(() => reject(new Error("Provider server shutdown timed out.")), remainingMs);
 	});
+	try {
+		return await Promise.race([promise, timeout]);
+	} finally {
+		if (timer) clearTimeout(timer);
+	}
+}
 
-	// Internal self-test listener (health dependency inversion): a SEPARATE
-	// socket the tenant-facing gateway never dials. Off by default — it only
-	// starts when the shared self-test master secret env is present.
-	const selfTestSecrets = resolveSelfTestMasterSecrets();
-	if (selfTestSecrets) {
-		const selfTestApp = createSelfTestApp(provider, {
-			secrets: selfTestSecrets,
-			invoke: createSelfTestInvoke(app),
-		});
-		bunRuntime.serve({
-			port: options.selfTestPort ?? resolveSelfTestPort(),
-			hostname: options.host ?? DEFAULT_HOST,
-			fetch: selfTestApp.fetch,
-		});
+function resolveShutdownSignals(signals: boolean | NodeJS.Signals[]): NodeJS.Signals[] {
+	if (signals === false) return [];
+	return [...new Set(signals === true ? DEFAULT_SHUTDOWN_SIGNALS : signals)];
+}
+
+function shutdownTimeout(value: number): number {
+	if (!Number.isFinite(value) || value < 0) {
+		throw new Error("Provider server shutdown timeoutMs must be a non-negative finite number.");
 	}
-	await Promise.resolve();
+	return value;
 }