@apifuse/provider-sdk 2.2.0-beta.2 → 2.2.0-beta.21

package/dist/server/serve.js

@@ -2,36 +2,72 @@ import { existsSync } from "node:fs";
 import { join } from "node:path";
 import { Hono } from "hono";
 import { z } from "zod";
-import { AuthAbortError, createAuthFlowHelpers } from "../auth";
-import { AuthError, ProviderError, SessionExpiredError, TransportError, } from "../errors";
-import { loadProviderLocaleCatalogs, localizeAuthTurn, } from "../i18n/catalog";
-import { categoryForStatus, isRetryableCategory, PROVIDER_OBSERVABILITY_TAXONOMY_VERSION, } from "../observability";
-import { createScratchpad } from "../runtime/auth-flow";
-import { createBrowserClient } from "../runtime/browser";
-import { createProviderCache } from "../runtime/cache";
-import { createProviderChoiceContext, PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV, } from "../runtime/choice";
-import { createCredentialContext } from "../runtime/credential";
-import { createEnvContext } from "../runtime/env";
-import { executeOperation } from "../runtime/executor";
-import { createHttpClient } from "../runtime/http";
-import { wrapWithInstrumentation } from "../runtime/instrumentation";
-import { getProviderBaseUrl } from "../runtime/provider";
-import { PROXY_AUTH_IP_DENIED_CODE, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_EXHAUSTED_CODE, } from "../runtime/proxy-errors";
-import { PROVIDER_TELEMETRY_HEADER, ProxyTelemetryCollector, } from "../runtime/proxy-telemetry";
-import { createProviderRuntimeStateFromEnv, createUnsupportedProviderRuntimeState, } from "../runtime/state";
-import { createStealthClient } from "../runtime/stealth";
-import { createSttClientFromEnv } from "../runtime/stt";
-import { createTraceContext } from "../runtime/trace";
-import { parseSchema } from "../schema";
-import { getStealthProfile } from "../stealth/profiles";
-import { APIFUSE_STREAM_DONE_EVENT, APIFUSE_STREAM_ERROR_EVENT, encodeSseEvent, error as streamError, } from "../stream";
-import { createSelfTestApp, createSelfTestInvoke, resolveSelfTestPort, } from "./self-test";
-import { resolveSelfTestMasterSecrets } from "./self-test-token";
-import { AuthFlowRequestSchema, OperationRequestSchema, } from "./types";
+import { AuthAbortError, createAuthFlowHelpers } from "../auth.js";
+import { SDK_OWNED_PROVIDER_ERROR_CODES, SDK_RUNTIME_OWNED_ERROR_CODES, } from "../error-resolution.js";
+import { AuthError, isProviderError, isSessionExpiredError, isTransportError, isValidationError, ProviderError, } from "../errors.js";
+import { loadProviderLocaleCatalogs, localizeAuthTurn, } from "../i18n/catalog.js";
+import { categoryForStatus, sourceForCategory, isRetryableCategory, PROVIDER_OBSERVABILITY_TAXONOMY_VERSION, } from "../observability.js";
+import { createScratchpad } from "../runtime/auth-flow.js";
+import { createBrowserClient } from "../runtime/browser.js";
+import { createProviderCache } from "../runtime/cache.js";
+import { createProviderChoiceContext, PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV, } from "../runtime/choice.js";
+import { createCredentialContext } from "../runtime/credential.js";
+import { createEnvContext } from "../runtime/env.js";
+import { executeOperation } from "../runtime/executor.js";
+import { createHttpClient } from "../runtime/http.js";
+import { wrapWithInstrumentation } from "../runtime/instrumentation.js";
+import { createEnvVendorCredentialResolver, createNativeNetworkClient, } from "../runtime/native-network.js";
+import { getProviderBaseUrl } from "../runtime/provider.js";
+import { PROXY_AUTH_IP_DENIED_CODE, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_EXHAUSTED_CODE, } from "../runtime/proxy-errors.js";
+import { PROVIDER_TELEMETRY_HEADER, ProxyTelemetryCollector } from "../runtime/proxy-telemetry.js";
+import { assertRequiredSecretsPresent, listMissingRequiredSecrets, MISSING_SECRET_CODE, } from "../runtime/secrets.js";
+import { createProviderRuntimeStateFromEnv, createUnsupportedProviderRuntimeState, } from "../runtime/state.js";
+import { createStealthClient } from "../runtime/stealth.js";
+import { createSttClientFromEnv } from "../runtime/stt.js";
+import { createTraceContext } from "../runtime/trace.js";
+import { parseSchema } from "../schema.js";
+import { STATEFUL_NONCE_HEADER as STATEFUL_FORWARDING_NONCE_HEADER, STATEFUL_SIGNATURE_HEADER as STATEFUL_FORWARDING_SIGNATURE_HEADER, STATEFUL_TIMESTAMP_HEADER as STATEFUL_FORWARDING_TIMESTAMP_HEADER, verifyStatefulRequestSignature, } from "../stateful-signing.js";
+import { StatefulRoutingDeadlineError } from "../stateful/stateful-provider-session-routing.js";
+import { getStealthProfile } from "../stealth/profiles.js";
+import { APIFUSE_STREAM_DONE_EVENT, APIFUSE_STREAM_ERROR_EVENT, encodeSseEvent, error as streamError, } from "../stream.js";
+import { VALID_OPERATION_ERROR_STATUSES } from "../types.js";
+import { createSelfTestApp, createSelfTestAuthFlowInvoke, createSelfTestInvoke, resolveSelfTestPort, } from "./self-test.js";
+import { resolveSelfTestMasterSecrets } from "./self-test-token.js";
+import { AuthFlowRequestSchema, OperationConnectionSchema, OperationRequestSchema, } from "./types.js";
 const DEFAULT_HOST = "0.0.0.0";
 const DEFAULT_PORT = 3000;
+/** Compact SDK-owned error classification emitted separately from the public response body. */
+export const ERROR_OBSERVABILITY_HEADER = "X-ApiFuse-Error-Observability";
 const AUTH_FLOW_LOCALES = ["en", "ko", "ja"];
 const retryResponseMeta = new WeakMap();
+const STATEFUL_INTERNAL_OPERATIONS_ROUTE = "/__apifuse/stateful/operations";
+const STATEFUL_FORWARDING_SOURCE_POD_HEADER = "x-apifuse-stateful-source-pod";
+const DEFAULT_STATEFUL_FORWARDING_MAX_SKEW_MS = 5 * 60_000;
+const DEFAULT_STATEFUL_FORWARDING_REPLAY_CACHE_MAX_ENTRIES = 10_000;
+const STATEFUL_FORWARDING_REPLAY_BUCKET_MS = 10_000;
+const STATEFUL_FORWARDING_REPLAY_RETRY_AFTER_SECONDS = Math.ceil(STATEFUL_FORWARDING_REPLAY_BUCKET_MS / 1_000);
+export const ProviderServerStatefulForwardEnvelopeSchema = z
+    .object({
+    requestId: z.string().min(1),
+    providerId: z.string().min(1),
+    operationId: z.string().min(1),
+    sessionKey: z.string().min(1),
+    connectionId: z.string().min(1),
+    serviceAccountId: z.string().min(1),
+    ownerPodId: z.string().min(1),
+    generation: z.number().int().positive(),
+    sourcePodId: z.string().min(1),
+    forwardedAt: z.string().refine((value) => Number.isFinite(Date.parse(value))),
+    deadlineAt: z
+        .string()
+        .refine((value) => Number.isFinite(Date.parse(value)))
+        .optional(),
+    idempotencyKey: z.string().min(1).optional(),
+    operationRequest: OperationRequestSchema.extend({
+        connection: OperationConnectionSchema.strict().optional(),
+    }).strict(),
+})
+    .strict();
 function createAuthStub() {
     return {
         async requestField(name) {
@@ -93,9 +129,7 @@ function getProviderStealthBaseUrl(provider) {
     return firstHost ? `https://${firstHost}` : undefined;
 }
 function getProviderStealthProfile(provider) {
-    return provider.stealth?.profile
-        ? getStealthProfile(provider.stealth.profile)
-        : undefined;
+    return provider.stealth?.profile ? getStealthProfile(provider.stealth.profile) : undefined;
 }
 function isProductionProviderBrowserMode(provider, env = process.env) {
     if (provider.runtime !== "browser") {
@@ -104,13 +138,11 @@ function isProductionProviderBrowserMode(provider, env = process.env) {
     if (env.APIFUSE__PROVIDER__RUNTIME === "browser") {
         return true;
     }
-    return (env.NODE_ENV === "production" && env.APIFUSE__PROVIDER__ID === provider.id);
+    return env.NODE_ENV === "production" && env.APIFUSE__PROVIDER__ID === provider.id;
 }
 export function resolveProviderProxyAffinityKey(provider, request, operationId) {
     const connectionKey = resolveOperationConnectionId(request) ?? request.connection?.externalRef;
-    const affinity = typeof provider.proxy === "object"
-        ? provider.proxy.session?.affinity
-        : undefined;
+    const affinity = typeof provider.proxy === "object" ? provider.proxy.session?.affinity : undefined;
     if (affinity === "operation") {
         return `${provider.id}/${operationId}`;
     }
@@ -119,6 +151,15 @@ export function resolveProviderProxyAffinityKey(provider, request, operationId)
 function resolveOperationConnectionId(request) {
     return request.connection?.id ?? request.connectionId;
 }
+function resolveNativeProxyPolicy(provider) {
+    if (typeof provider.proxy === "object")
+        return provider.proxy;
+    if (provider.proxy === true)
+        return { mode: "optional" };
+    if (provider.proxy === false)
+        return { mode: "disabled" };
+    return undefined;
+}
 function createProviderContext(provider, request, operationId, options = {}, state = createUnsupportedProviderRuntimeState(), proxyTelemetry) {
     const baseUrl = getProviderBaseUrl(provider);
     const stealthBaseUrl = getProviderStealthBaseUrl(provider);
@@ -177,6 +218,18 @@ function createProviderContext(provider, request, operationId, options = {}, sta
                 engine: provider.browser?.engine,
             })
             : createBrowserStub(),
+        ...(provider.native
+            ? {
+                native: {
+                    network: createNativeNetworkClient({
+                        egress: provider.native.network,
+                        proxyPolicy: resolveNativeProxyPolicy(provider),
+                        affinityKey: proxyClientOptions.affinityKey,
+                        credentials: createEnvVendorCredentialResolver(env),
+                    }),
+                },
+            }
+            : {}),
         trace: createTraceContext(),
         auth: createAuthStub(),
         stt: options.stt ?? createSttClientFromEnv(provider.stt),
@@ -243,6 +296,7 @@ function createAuthFlowContext(provider, request, options = {}, signal) {
         : undefined;
     return {
         context: {
+            flowId: request.flowId,
             connectionId: request.connectionId,
             externalRef: request.externalRef,
             tenantId: request.tenantId ?? "",
@@ -253,7 +307,24 @@ function createAuthFlowContext(provider, request, options = {}, signal) {
                     ? createStealthClient(stealthBaseUrl, stealthProfile.name, stealthClientOptions)
                     : createStealthClient(stealthBaseUrl, stealthClientOptions)
                 : createStealthStub(),
-            env: createEnvContext(provider.secrets?.map((secret) => secret.name)),
+            ...(provider.native
+                ? {
+                    native: {
+                        network: createNativeNetworkClient({
+                            egress: provider.native.network,
+                            proxyPolicy: resolveNativeProxyPolicy(provider),
+                            affinityKey: proxyClientOptions.affinityKey,
+                            credentials: createEnvVendorCredentialResolver(createEnvContext(provider.secrets?.map((secret) => secret.name))),
+                        }),
+                    },
+                }
+                : {}),
+            env: createEnvContext([
+                ...(provider.secrets?.map((secret) => secret.name) ?? []),
+                ...(provider.auth?.mode === "oauth2_proxied"
+                    ? ["APIFUSE__AUTH_PROXY__URL"]
+                    : []),
+            ]),
             credential,
             context: flowContextStore.context,
             stt: options.stt ?? createSttClientFromEnv(provider.stt),
@@ -292,16 +363,50 @@ function zodDetails(error) {
         message: issue.message,
     }));
 }
-function toErrorResponse(error, requestId) {
-    if (error instanceof ProviderError) {
-        const details = publicProviderErrorDetails(error);
+// Category-level projection with code-aware honesty overrides: a missing
+// deployment secret is an APIFuse-side defect even though its category
+// (credential_unavailable) usually means a caller credential problem, an
+// internal stateful-routing deadline is APIFuse-owned despite its timeout
+// category, and the built-in upstream failure families keep their upstream
+// attribution even when the author left the category at the provider_error
+// default.
+function publicErrorSource(error, category) {
+    if (error instanceof StatefulRoutingDeadlineError)
+        return "apifuse";
+    if (isProviderError(error)) {
+        if (error.code === MISSING_SECRET_CODE)
+            return "apifuse";
+        if (error.code === "UPSTREAM_ERROR" || error.code === "BLOCKED") {
+            return "upstream_failure";
+        }
+    }
+    return sourceForCategory(category);
+}
+function toErrorResponse(error, requestId, declaredErrorCode) {
+    const observability = errorObservabilityDetails(error, declaredErrorCode);
+    const source = publicErrorSource(error, observability.category);
+    if (error instanceof StatefulRoutingDeadlineError) {
+        return {
+            error: {
+                code: "STATEFUL_FORWARDING_DEADLINE_EXPIRED",
+                message: "Stateful forwarding deadline expired.",
+                ...(requestId ? { requestId } : {}),
+                retryable: observability.retryable,
+                source,
+            },
+        };
+    }
+    if (isProviderError(error)) {
+        const details = error.details;
         return {
             error: {
                 code: error.code ?? "provider_error",
                 message: publicProviderErrorMessage(error),
                 ...(requestId ? { requestId } : {}),
+                retryable: observability.retryable,
+                source,
                 ...(error.fix ? { fix: error.fix } : {}),
-                ...(details ? { details } : {}),
+                ...(details !== undefined ? { details } : {}),
             },
         };
     }
@@ -311,52 +416,67 @@ function toErrorResponse(error, requestId) {
                 code: "invalid_request",
                 message: "Invalid request body",
                 ...(requestId ? { requestId } : {}),
+                retryable: observability.retryable,
+                source,
                 details: zodDetails(error),
             },
         };
     }
+    // A masked internal error MUST NOT be advertised as retryable: without an
+    // explicit retryable:false the hub (bori provider-backed engine) defaults 5xx
+    // to retryable:true, which turns a deterministic pre-upstream crash into an
+    // infinite START->CONTINUE->restart loop (2026-07-22 catchtable reserve RCA).
+    // We still refuse to leak message/stack — only the error class name (or the
+    // primitive type for non-Error throwables) is surfaced for ops triage.
     return {
         error: {
             code: "internal_error",
             message: "Internal error",
             ...(requestId ? { requestId } : {}),
+            retryable: observability.retryable,
+            source,
+            details: {
+                retryable: false,
+                category: "internal_error",
+                errorClass: error instanceof Error ? error.name : typeof error,
+            },
         },
     };
 }
-function publicProviderErrorDetails(error) {
-    const providerDetails = error.details;
-    const observabilityDetails = providerObservabilityDetails(error);
-    if (providerDetails === undefined) {
-        return observabilityDetails;
-    }
-    if (observabilityDetails === undefined) {
-        return providerDetails;
-    }
-    if (isPlainRecord(providerDetails) && isPlainRecord(observabilityDetails)) {
-        return { ...providerDetails, ...observabilityDetails };
-    }
-    return {
-        provider: providerDetails,
-        observability: observabilityDetails,
-    };
-}
-function isPlainRecord(value) {
-    return value !== null && typeof value === "object" && !Array.isArray(value);
-}
-function providerObservabilityDetails(error) {
+// Accepts `unknown` so the branded guards narrow cleanly from the top: the
+// subtype error classes are structurally compatible with ProviderError, so
+// narrowing from a ProviderError-typed value would collapse the negative branch
+// to `never`. Narrowing from unknown avoids that while still recognizing errors
+// from a duplicate SDK module instance.
+function providerObservabilityDetails(error, declaredErrorCode) {
+    const declaredRetryable = sdkOwnsErrorResolution(error)
+        ? undefined
+        : declaredErrorCode?.retryable;
     // Session-expiry surfaces the credential_expired category + the opt-in
     // retryable signal so Gateway/Credential Service can refresh and re-drive the
     // operation (see design.md §4.3 D3). Without this branch the auth error would
     // serialize as a bare 401 with no retryable/category, losing the refresh
     // signal for exactly the retryOnAuthRefresh operations it is meant to enable.
-    if (error instanceof SessionExpiredError) {
+    if (isSessionExpiredError(error)) {
         return {
             category: error.options?.category ?? "credential_expired",
             taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
-            retryable: error.options?.retryable ?? false,
+            retryable: error.options?.retryable ?? declaredRetryable ?? false,
+        };
+    }
+    // Missing-secret errors carry the canonical credential_unavailable category
+    // so Gateway/observability can attribute the failure to provisioning, not
+    // the upstream. Matched by code (not constructor) so both the SDK-owned
+    // runtime gate and any not-yet-migrated provider-thrown MISSING_SECRET
+    // serialize identically, including across duplicate SDK module instances.
+    if (isProviderError(error) && error.code === MISSING_SECRET_CODE) {
+        return {
+            category: error.options?.category ?? "credential_unavailable",
+            taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
+            retryable: error.options?.retryable ?? declaredRetryable ?? false,
         };
     }
-    if (!(error instanceof TransportError)) {
+    if (!isTransportError(error)) {
         return undefined;
     }
     const isProxyPoolCode = error.code === PROXY_POOL_EXHAUSTED_CODE ||
@@ -384,8 +504,69 @@ function providerObservabilityDetails(error) {
         ...(error.upstreamStatus ? { upstreamStatus: error.upstreamStatus } : {}),
     };
 }
+function errorObservabilityDetails(error, declaredErrorCode) {
+    const effectiveDeclaration = sdkOwnsErrorResolution(error) ? undefined : declaredErrorCode;
+    const providerDetails = providerObservabilityDetails(error, effectiveDeclaration);
+    if (providerDetails)
+        return providerDetails;
+    if (error instanceof z.ZodError || isValidationError(error)) {
+        const declaredStatus = effectiveDeclaration?.status;
+        return {
+            category: isProviderError(error) && error.options?.category
+                ? error.options.category
+                : isEmittableErrorStatus(declaredStatus) &&
+                    categoryForStatus(declaredStatus) === "upstream_rejected"
+                    ? "upstream_rejected"
+                    : isEmittableErrorStatus(declaredStatus) && declaredStatus >= 500
+                        ? "provider_error"
+                        : "input_validation",
+            taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
+            retryable: isProviderError(error)
+                ? (error.options?.retryable ?? effectiveDeclaration?.retryable ?? false)
+                : false,
+        };
+    }
+    if (error instanceof StatefulRoutingDeadlineError) {
+        return {
+            category: "timeout",
+            taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
+            retryable: false,
+        };
+    }
+    if (isProviderError(error)) {
+        // Deterministic upstream refusals default to the rejection category:
+        // the UPSTREAM_REJECTED family and any operation-declared rejection
+        // status (409/410/422) classify as upstream_rejected unless the
+        // author set an explicit category.
+        const declaredStatus = effectiveDeclaration?.status;
+        const rejectionDefault = error.code === "UPSTREAM_REJECTED" ||
+            (isEmittableErrorStatus(declaredStatus) &&
+                categoryForStatus(declaredStatus) === "upstream_rejected")
+            ? "upstream_rejected"
+            : "provider_error";
+        return {
+            category: error.options?.category ?? rejectionDefault,
+            taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
+            retryable: error.options?.retryable ?? effectiveDeclaration?.retryable ?? false,
+        };
+    }
+    return {
+        category: "internal_error",
+        taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
+        retryable: false,
+    };
+}
+function responseWithErrorObservability(response, error, declaredErrorCode) {
+    const headers = new Headers(response.headers);
+    headers.set(ERROR_OBSERVABILITY_HEADER, JSON.stringify(errorObservabilityDetails(error, declaredErrorCode)));
+    return new Response(response.body, {
+        status: response.status,
+        statusText: response.statusText,
+        headers,
+    });
+}
 function publicProviderErrorMessage(error) {
-    if (error instanceof TransportError) {
+    if (isTransportError(error)) {
         if (error.code === PROXY_AUTH_IP_DENIED_CODE) {
             return error.message;
         }
@@ -409,18 +590,30 @@ function publicProviderErrorMessage(error) {
     }
     return error.message;
 }
-function toStatusCode(error) {
+function isEmittableErrorStatus(value) {
+    return (typeof value === "number" &&
+        VALID_OPERATION_ERROR_STATUSES.some((status) => status === value));
+}
+function toStatusCode(error, declaredErrorCode) {
     if (error instanceof z.ZodError) {
         return 400;
     }
-    if (error instanceof TransportError) {
-        return error.code === "transport_timeout" ? 504 : 502;
+    if (error instanceof StatefulRoutingDeadlineError) {
+        return 504;
     }
-    if (error instanceof ProviderError) {
+    if (isProviderError(error)) {
+        if (!sdkOwnsErrorResolution(error) &&
+            isEmittableErrorStatus(declaredErrorCode?.status)) {
+            return declaredErrorCode.status;
+        }
         switch (error.code) {
             case "AUTH_REQUIRED":
             case "reauth_required":
                 return 401;
+            // Unprovisioned declared secret: a deployment/config defect, never an
+            // upstream failure — explicit 400 (was only reached via fallthrough).
+            case MISSING_SECRET_CODE:
+                return 400;
             case "NOT_FOUND":
             case "not_found":
             case "NO_DATA":
@@ -429,17 +622,55 @@ function toStatusCode(error) {
             case "UPSTREAM_RATE_LIMIT":
             case "LIMITED_NUMBER_OF_SERVICE_REQUESTS_EXCEEDS_ERROR":
                 return 429;
+            // Deterministic upstream business refusal (honest-provider-error-
+            // contract): the upstream evaluated the request and said no under
+            // its own rules — a conflict with upstream state, never a 5xx.
+            case "UPSTREAM_REJECTED":
+                return 409;
             case "UPSTREAM_ERROR":
             case "BLOCKED":
                 return 502;
             case "STT_UNAVAILABLE":
             case "UNSUPPORTED_STT_BACKEND":
+            case "STATEFUL_FORWARDING_REPLAY_CACHE_FULL":
                 return 503;
         }
-        return 400;
+        if (isTransportError(error)) {
+            return error.code === "transport_timeout" ? 504 : 502;
+        }
+        if (isValidationError(error)) {
+            return error.options?.category === "output_validation" ? 500 : 400;
+        }
+        return 500;
     }
     return 500;
 }
+function sdkOwnsErrorResolution(error) {
+    if (isSessionExpiredError(error))
+        return true;
+    if (isTransportError(error))
+        return true;
+    if (error instanceof z.ZodError)
+        return true;
+    if (error instanceof StatefulRoutingDeadlineError)
+        return true;
+    return (isProviderError(error) &&
+        typeof error.code === "string" &&
+        SDK_RUNTIME_OWNED_ERROR_CODES.has(error.code));
+}
+function buildOperationErrorCodeLookup(provider) {
+    return new Map(Object.entries(provider.operations).flatMap(([operationId, operation]) => {
+        const errorCodes = operation.docs?.errorCodes;
+        return errorCodes?.length
+            ? [[operationId, new Map(errorCodes.map((entry) => [entry.code, entry]))]]
+            : [];
+    }));
+}
+function declaredErrorCodeFor(error, operationId, lookup) {
+    if (!operationId || !isProviderError(error) || typeof error.code !== "string")
+        return undefined;
+    return lookup.get(operationId)?.get(error.code);
+}
 function extractRequestId(raw) {
     if (!raw || typeof raw !== "object") {
         return undefined;
@@ -447,17 +678,23 @@ function extractRequestId(raw) {
     const value = Object.getOwnPropertyDescriptor(raw, "requestId")?.value;
     return typeof value === "string" ? value : undefined;
 }
-function logProviderError(logger, provider, kind, route, requestId, error, status, cost) {
-    const code = error instanceof ProviderError
+function logProviderError(logger, provider, kind, route, requestId, error, status, cost, declaredErrorCode) {
+    const code = isProviderError(error)
         ? (error.code ?? "provider_error")
         : error instanceof z.ZodError
             ? "invalid_request"
-            : "internal_error";
+            : error instanceof StatefulRoutingDeadlineError
+                ? "STATEFUL_FORWARDING_DEADLINE_EXPIRED"
+                : "internal_error";
     const errorClass = error instanceof Error ? error.name : typeof error;
     const message = error instanceof Error ? error.message : String(error);
-    const details = error instanceof ProviderError
-        ? providerObservabilityDetails(error)
-        : undefined;
+    const details = errorObservabilityDetails(error, declaredErrorCode);
+    const isUnregisteredProviderErrorCode = status === 500 &&
+        isProviderError(error) &&
+        !isValidationError(error) &&
+        typeof error.code === "string" &&
+        !SDK_OWNED_PROVIDER_ERROR_CODES.has(error.code) &&
+        declaredErrorCode === undefined;
     const emit = typeof logger === "function" ? logger : defaultProviderServerLogger;
     emit({
         level: status >= 500 ? "error" : "warn",
@@ -471,14 +708,14 @@ function logProviderError(logger, provider, kind, route, requestId, error, statu
         code,
         errorClass,
         message,
-        ...(error instanceof TransportError && error.upstreamStatus
-            ? { upstreamStatus: error.upstreamStatus }
-            : {}),
-        ...(details
+        ...(details.upstreamStatus ? { upstreamStatus: details.upstreamStatus } : {}),
+        errorCategory: details.category,
+        taxonomyVersion: details.taxonomyVersion,
+        retryable: details.retryable,
+        ...(isUnregisteredProviderErrorCode
             ? {
-                errorCategory: details.category,
-                taxonomyVersion: details.taxonomyVersion,
-                retryable: details.retryable,
+                signal: "unregistered_provider_error_code",
+                signalFix: "Declare this code (with status and retryable) in the operation's docs.errorCodes so it serves its intended status instead of 500.",
             }
             : {}),
         ...(error instanceof z.ZodError ? { issues: zodDetails(error) } : {}),
@@ -594,8 +831,7 @@ async function validateSseEvent(operation, event) {
     const transport = getSseTransport(operation);
     const schema = transport?.events?.[event.event];
     if (!schema) {
-        if (event.event === APIFUSE_STREAM_ERROR_EVENT ||
-            event.event === APIFUSE_STREAM_DONE_EVENT) {
+        if (event.event === APIFUSE_STREAM_ERROR_EVENT || event.event === APIFUSE_STREAM_DONE_EVENT) {
             return event;
         }
         throw new ProviderError(`SSE event "${event.event}" is not declared in operation transport.events.`, {
@@ -714,8 +950,7 @@ function enforceStreamChunkLimit(body, maxChunkBytes) {
 }
 function toStreamingResponse(operation, result, cleanup, requestId) {
     const transport = operation.transport?.kind ?? "json";
-    if (transport === "sse" &&
-        (result instanceof Response || result instanceof ReadableStream)) {
+    if (transport === "sse" && (result instanceof Response || result instanceof ReadableStream)) {
         void cleanup();
         throw new ProviderError("SSE operations must return an AsyncIterable of typed stream.event(...) values.", {
             code: "SSE_RESULT_UNSUPPORTED",
@@ -726,9 +961,7 @@ function toStreamingResponse(operation, result, cleanup, requestId) {
     }
     if (result instanceof Response) {
         const httpTransport = getHttpStreamTransport(operation);
-        if (httpTransport &&
-            result.body &&
-            httpTransport?.maxChunkBytes !== undefined) {
+        if (httpTransport && result.body && httpTransport?.maxChunkBytes !== undefined) {
             return responseWithCleanup(new Response(enforceStreamChunkLimit(result.body, httpTransport.maxChunkBytes), {
                 headers: result.headers,
                 status: result.status,
@@ -747,8 +980,7 @@ function toStreamingResponse(operation, result, cleanup, requestId) {
                 ? { "Content-Type": "text/event-stream; charset=utf-8" }
                 : {
                     "Content-Type": operation.transport?.kind === "http-stream"
-                        ? (operation.transport.contentType ??
-                            "application/octet-stream")
+                        ? (operation.transport.contentType ?? "application/octet-stream")
                         : "application/octet-stream",
                 },
         }), cleanup);
@@ -766,9 +998,7 @@ function getSseTransport(operation) {
     return operation.transport?.kind === "sse" ? operation.transport : undefined;
 }
 function getHttpStreamTransport(operation) {
-    return operation.transport?.kind === "http-stream"
-        ? operation.transport
-        : undefined;
+    return operation.transport?.kind === "http-stream" ? operation.transport : undefined;
 }
 function toAuthFlowResponse(result, contextPatch) {
     if (result instanceof Response) {
@@ -796,7 +1026,7 @@ function isAuthFlowLocale(value) {
     return value === "en" || value === "ko" || value === "ja";
 }
 function isAuthTurn(value) {
-    return (!!value && typeof value === "object" && "kind" in value && "turnId" in value);
+    return !!value && typeof value === "object" && "kind" in value && "turnId" in value;
 }
 function loadAuthFlowLocaleCatalogs(provider) {
     for (const providerDir of [
@@ -859,7 +1089,14 @@ async function handleOperation(provider, request, operationId, options = {}, sta
         }
     };
     try {
-        const result = await executeOperation(provider, operationId, ctx, request.input);
+        const result = options.operationExecutor
+            ? await options.operationExecutor({
+                provider,
+                operationId,
+                ctx,
+                request,
+            })
+            : await executeOperation(provider, operationId, ctx, request.input);
         if (streaming && operation) {
             return toStreamingResponse(operation, result, cleanup, request.requestId);
         }
@@ -893,8 +1130,16 @@ async function handleAuthFlow(provider, request, route, options = {}, signal) {
             code: "AUTH_FLOW_NOT_CONFIGURED",
         });
     }
+    // Same SDK-owned gate as executeOperation: OAuth/credentials ceremonies
+    // depend on declared secrets (client ids/secrets), so fail structured before
+    // any flow code runs instead of at whatever point the ceremony first reads
+    // the env. `abort` stays exempt: a user must always be able to cancel a
+    // stranded flow even when provisioning is broken.
     const { context, getPatch } = createAuthFlowContext(provider, request, options, signal);
     try {
+        if (route !== "abort") {
+            assertRequiredSecretsPresent(provider, context.env);
+        }
         const result = route === "start"
             ? await flow.start(context)
             : route === "continue"
@@ -933,25 +1178,268 @@ async function handleAuthFlow(provider, request, route, options = {}, signal) {
         context.stealth.close?.();
     }
 }
+class StatefulForwardingReplayCache {
+    maxEntries;
+    #nonces = new Map();
+    #expiryBuckets = new Map();
+    #nextExpiryBucket;
+    #latestExpiryBucket;
+    constructor(maxEntries) {
+        this.maxEntries = maxEntries;
+    }
+    claim(nonce, expiresAtMs, nowMs) {
+        this.dropExpiredBuckets(nowMs);
+        if (this.#nonces.has(nonce))
+            return "replayed";
+        if (this.#nonces.size >= this.maxEntries)
+            return "full";
+        const expiryBucket = Math.ceil(expiresAtMs / STATEFUL_FORWARDING_REPLAY_BUCKET_MS) *
+            STATEFUL_FORWARDING_REPLAY_BUCKET_MS;
+        this.#nonces.set(nonce, expiryBucket);
+        const bucket = this.#expiryBuckets.get(expiryBucket) ?? new Set();
+        bucket.add(nonce);
+        this.#expiryBuckets.set(expiryBucket, bucket);
+        this.#nextExpiryBucket = Math.min(this.#nextExpiryBucket ?? expiryBucket, expiryBucket);
+        this.#latestExpiryBucket = Math.max(this.#latestExpiryBucket ?? expiryBucket, expiryBucket);
+        return "accepted";
+    }
+    dropExpiredBuckets(nowMs) {
+        if (this.#nextExpiryBucket === undefined || this.#latestExpiryBucket === undefined)
+            return;
+        if (nowMs >= this.#latestExpiryBucket) {
+            this.#nonces.clear();
+            this.#expiryBuckets.clear();
+            this.#nextExpiryBucket = undefined;
+            this.#latestExpiryBucket = undefined;
+            return;
+        }
+        while (this.#nextExpiryBucket <= nowMs) {
+            const bucket = this.#expiryBuckets.get(this.#nextExpiryBucket);
+            if (bucket) {
+                for (const cachedNonce of bucket)
+                    this.#nonces.delete(cachedNonce);
+                this.#expiryBuckets.delete(this.#nextExpiryBucket);
+            }
+            this.#nextExpiryBucket += STATEFUL_FORWARDING_REPLAY_BUCKET_MS;
+        }
+    }
+}
+function verifyStatefulForwardingRequest(input) {
+    const config = input.options.statefulForwarding;
+    if (!config?.secret) {
+        throw new ProviderError("Stateful forwarding is not configured.", {
+            code: "STATEFUL_FORWARDING_NOT_CONFIGURED",
+        });
+    }
+    const timestamp = input.headers.get(STATEFUL_FORWARDING_TIMESTAMP_HEADER) ?? "";
+    const signature = input.headers.get(STATEFUL_FORWARDING_SIGNATURE_HEADER) ?? "";
+    const nonce = input.headers.get(STATEFUL_FORWARDING_NONCE_HEADER) ?? "";
+    if (!timestamp || !signature || !nonce) {
+        throw new ProviderError("Stateful forwarding signature headers are missing.", {
+            code: "STATEFUL_FORWARDING_SIGNATURE_MISSING",
+        });
+    }
+    if (nonce.length > 256) {
+        throw new ProviderError("Stateful forwarding nonce is invalid.", {
+            code: "STATEFUL_FORWARDING_NONCE_INVALID",
+        });
+    }
+    const timestampMs = Date.parse(timestamp);
+    const maxSkewMs = config.maxSkewMs ?? DEFAULT_STATEFUL_FORWARDING_MAX_SKEW_MS;
+    if (!Number.isFinite(timestampMs) || Math.abs(Date.now() - timestampMs) > maxSkewMs) {
+        throw new ProviderError("Stateful forwarding signature timestamp is outside the allowed skew.", { code: "STATEFUL_FORWARDING_TIMESTAMP_INVALID" });
+    }
+    if (!verifyStatefulRequestSignature({
+        secret: config.secret,
+        timestamp,
+        rawBody: input.rawBody,
+        method: input.method,
+        path: input.path,
+        nonce,
+        signature,
+    })) {
+        throw new ProviderError("Stateful forwarding signature is invalid.", {
+            code: "STATEFUL_FORWARDING_SIGNATURE_INVALID",
+        });
+    }
+    const replayResult = input.replayCache.claim(nonce, timestampMs + maxSkewMs, Date.now());
+    if (replayResult === "replayed") {
+        throw new ProviderError("Stateful forwarding nonce has already been used.", {
+            code: "STATEFUL_FORWARDING_REPLAY_DETECTED",
+        });
+    }
+    if (replayResult === "full") {
+        throw new ProviderError("Stateful forwarding replay cache is at capacity.", {
+            code: "STATEFUL_FORWARDING_REPLAY_CACHE_FULL",
+        });
+    }
+}
+function operationRequestFromForwardingEnvelope(envelope) {
+    return {
+        ...envelope.operationRequest,
+        ...(envelope.deadlineAt !== undefined ? { deadlineAt: envelope.deadlineAt } : {}),
+    };
+}
+function parseStatefulForwardingEnvelope(rawBody) {
+    const parsed = ProviderServerStatefulForwardEnvelopeSchema.safeParse(rawBody);
+    if (parsed.success)
+        return parsed.data;
+    throw new ProviderError("Stateful forwarding envelope is invalid.", {
+        code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
+        details: zodDetails(parsed.error),
+    });
+}
 export function createServerApp(provider, options = {}) {
+    validateStatefulServerConfig(options);
     const app = new Hono();
     const logger = options.logger ?? defaultProviderServerLogger;
+    const operationErrorCodes = buildOperationErrorCodeLookup(provider);
+    const statefulForwardingReplayCache = new StatefulForwardingReplayCache(options.statefulForwarding?.replayCacheMaxEntries ??
+        DEFAULT_STATEFUL_FORWARDING_REPLAY_CACHE_MAX_ENTRIES);
     const state = options.state ??
         createProviderRuntimeStateFromEnv({
             providerId: provider.id,
             allowMemoryFallback: options.allowMemoryStateFallback === true,
         });
-    app.notFound((c) => c.json({
-        error: {
-            code: "not_found",
-            message: "Not found",
-        },
-    }, 404));
+    // Boot-time visibility for unprovisioned declared secrets: emit a structured
+    // warn so deploy tooling/alerting sees the gap the moment the pod boots,
+    // instead of discovering it request-by-request. Deliberately log-only — a
+    // boot crash would trade a structured MISSING_SECRET signal for
+    // CrashLoopBackOff. Requests still fail closed via the executeOperation gate.
+    const missingSecretsAtBoot = listMissingRequiredSecrets(provider, createEnvContext(provider.secrets?.map((secret) => secret.name)));
+    if (missingSecretsAtBoot.length > 0) {
+        logger({
+            level: "warn",
+            event: "provider_secrets_missing",
+            providerId: provider.id,
+            missingSecrets: missingSecretsAtBoot,
+        });
+    }
+    app.notFound((c) => {
+        const error = new ProviderError("Not found", { code: "not_found", retryable: false });
+        return responseWithErrorObservability(c.json(toErrorResponse(error), 404), error);
+    });
     app.get("/health", (c) => c.json({
         status: "ok",
         provider: provider.id,
         version: provider.version,
     }));
+    app.post(STATEFUL_INTERNAL_OPERATIONS_ROUTE, async (c) => {
+        let rawBodyText = "";
+        let rawBody;
+        let operationId;
+        const operation = "stateful-internal";
+        const requestCost = startRequestCost();
+        try {
+            if (!options.internalOperationExecutor) {
+                throw new ProviderError("Stateful internal operation executor is not configured.", {
+                    code: "STATEFUL_INTERNAL_EXECUTOR_NOT_CONFIGURED",
+                });
+            }
+            rawBodyText = await c.req.raw.clone().text();
+            verifyStatefulForwardingRequest({
+                options,
+                rawBody: rawBodyText,
+                headers: c.req.raw.headers,
+                method: c.req.raw.method,
+                path: STATEFUL_INTERNAL_OPERATIONS_ROUTE,
+                replayCache: statefulForwardingReplayCache,
+            });
+            try {
+                rawBody = JSON.parse(rawBodyText);
+            }
+            catch {
+                throw new ProviderError("Stateful forwarding envelope is not valid JSON.", {
+                    code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
+                });
+            }
+            const envelope = parseStatefulForwardingEnvelope(rawBody);
+            if (envelope.providerId !== provider.id) {
+                throw new ProviderError("Stateful forwarding envelope providerId does not match the served provider.", { code: "STATEFUL_FORWARDING_PROVIDER_MISMATCH" });
+            }
+            if (envelope.requestId !== envelope.operationRequest.requestId) {
+                throw new ProviderError("Stateful forwarding requestId values do not match.", {
+                    code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
+                });
+            }
+            if (envelope.sourcePodId !==
+                (c.req.raw.headers.get(STATEFUL_FORWARDING_SOURCE_POD_HEADER) ?? "")) {
+                throw new ProviderError("Stateful forwarding source pod does not match its header.", {
+                    code: "STATEFUL_FORWARDING_SOURCE_POD_MISMATCH",
+                });
+            }
+            if (envelope.forwardedAt !== (c.req.raw.headers.get(STATEFUL_FORWARDING_TIMESTAMP_HEADER) ?? "")) {
+                throw new ProviderError("Stateful forwarding forwardedAt does not match its signature timestamp.", { code: "STATEFUL_FORWARDING_ENVELOPE_INVALID" });
+            }
+            const deadlineAtMs = envelope.deadlineAt ? Date.parse(envelope.deadlineAt) : undefined;
+            if (deadlineAtMs !== undefined && deadlineAtMs <= Date.now()) {
+                throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt);
+            }
+            const remainingDeadlineMs = deadlineAtMs === undefined ? undefined : deadlineAtMs - Date.now();
+            const deadlineSignal = remainingDeadlineMs === undefined ? undefined : AbortSignal.timeout(remainingDeadlineMs);
+            const signal = deadlineSignal
+                ? AbortSignal.any([c.req.raw.signal, deadlineSignal])
+                : c.req.raw.signal;
+            const ownerFenceValidation = Promise.resolve(options.statefulForwarding?.validateOwnerFence({
+                providerId: envelope.providerId,
+                sessionKey: envelope.sessionKey,
+                ownerPodId: envelope.ownerPodId,
+                generation: envelope.generation,
+                sourcePodId: envelope.sourcePodId,
+                forwardedAt: envelope.forwardedAt,
+                requestId: envelope.requestId,
+                ...(envelope.idempotencyKey ? { idempotencyKey: envelope.idempotencyKey } : {}),
+            }, signal));
+            let ownerFenceValid;
+            try {
+                ownerFenceValid = deadlineSignal
+                    ? await Promise.race([
+                        ownerFenceValidation,
+                        new Promise((_resolve, reject) => {
+                            deadlineSignal.addEventListener("abort", () => reject(new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt)), { once: true });
+                        }),
+                    ])
+                    : await ownerFenceValidation;
+            }
+            catch (error) {
+                if (deadlineSignal?.aborted) {
+                    throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt);
+                }
+                throw error;
+            }
+            if (ownerFenceValid !== true) {
+                throw new ProviderError("Stateful forwarding owner fence is no longer current.", {
+                    code: "STATEFUL_FORWARDING_OWNER_FENCE_INVALID",
+                });
+            }
+            const request = operationRequestFromForwardingEnvelope(envelope);
+            operationId = envelope.operationId;
+            const ctx = createProviderContext(provider, request, operationId, options, state);
+            if (deadlineAtMs !== undefined && deadlineAtMs <= Date.now()) {
+                throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt);
+            }
+            const output = await options.internalOperationExecutor({
+                provider,
+                operationId,
+                ctx,
+                request,
+                internalStatefulForward: envelope,
+                signal,
+            });
+            logProviderSuccess(logger, provider, "operation", operationId || operation, request.requestId, 200, finishRequestCost(requestCost));
+            return c.json({ data: output });
+        }
+        catch (error) {
+            const declaredErrorCode = declaredErrorCodeFor(error, operationId, operationErrorCodes);
+            const status = toStatusCode(error, declaredErrorCode);
+            if (isProviderError(error) && error.code === "STATEFUL_FORWARDING_REPLAY_CACHE_FULL") {
+                c.header("Retry-After", String(STATEFUL_FORWARDING_REPLAY_RETRY_AFTER_SECONDS));
+            }
+            const requestId = extractRequestId(rawBody);
+            logProviderError(logger, provider, "operation", operationId || operation, requestId, error, status, finishRequestCost(requestCost), declaredErrorCode);
+            return responseWithErrorObservability(c.json(toErrorResponse(error, requestId, declaredErrorCode), status), error, declaredErrorCode);
+        }
+    });
     app.post("/v1/:operation", async (c) => {
         let rawBody;
         const operation = c.req.param("operation");
@@ -977,13 +1465,14 @@ export function createServerApp(provider, options = {}) {
             return c.json(response);
         }
         catch (error) {
-            const status = toStatusCode(error);
+            const declaredErrorCode = declaredErrorCodeFor(error, operation, operationErrorCodes);
+            const status = toStatusCode(error, declaredErrorCode);
             const requestId = extractRequestId(rawBody);
-            logProviderError(logger, provider, "operation", operation, requestId, error, status, finishRequestCost(requestCost));
+            logProviderError(logger, provider, "operation", operation, requestId, error, status, finishRequestCost(requestCost), declaredErrorCode);
             const telemetryHeader = proxyTelemetry.toHeaderValue();
             if (telemetryHeader)
                 c.header(PROVIDER_TELEMETRY_HEADER, telemetryHeader);
-            return c.json(toErrorResponse(error, requestId), status);
+            return responseWithErrorObservability(c.json(toErrorResponse(error, requestId, declaredErrorCode), status), error, declaredErrorCode);
         }
     });
     app.post("/auth/start", async (c) => {
@@ -1003,7 +1492,7 @@ export function createServerApp(provider, options = {}) {
             const status = toStatusCode(error);
             const requestId = extractRequestId(rawBody);
             logProviderError(logger, provider, "auth", "start", requestId, error, status, finishRequestCost(requestCost));
-            return c.json(toErrorResponse(error, requestId), status);
+            return responseWithErrorObservability(c.json(toErrorResponse(error, requestId), status), error);
         }
     });
     app.post("/auth/continue", async (c) => {
@@ -1023,7 +1512,7 @@ export function createServerApp(provider, options = {}) {
             const status = toStatusCode(error);
             const requestId = extractRequestId(rawBody);
             logProviderError(logger, provider, "auth", "continue", requestId, error, status, finishRequestCost(requestCost));
-            return c.json(toErrorResponse(error, requestId), status);
+            return responseWithErrorObservability(c.json(toErrorResponse(error, requestId), status), error);
         }
     });
     app.post("/auth/poll", async (c) => {
@@ -1043,7 +1532,7 @@ export function createServerApp(provider, options = {}) {
             const status = toStatusCode(error);
             const requestId = extractRequestId(rawBody);
             logProviderError(logger, provider, "auth", "poll", requestId, error, status, finishRequestCost(requestCost));
-            return c.json(toErrorResponse(error, requestId), status);
+            return responseWithErrorObservability(c.json(toErrorResponse(error, requestId), status), error);
         }
     });
     app.post("/auth/refresh", async (c) => {
@@ -1063,7 +1552,7 @@ export function createServerApp(provider, options = {}) {
             const status = toStatusCode(error);
             const requestId = extractRequestId(rawBody);
             logProviderError(logger, provider, "auth", "refresh", requestId, error, status, finishRequestCost(requestCost));
-            return c.json(toErrorResponse(error, requestId), status);
+            return responseWithErrorObservability(c.json(toErrorResponse(error, requestId), status), error);
         }
     });
     app.post("/auth/disconnect", async (c) => {
@@ -1083,11 +1572,33 @@ export function createServerApp(provider, options = {}) {
             const status = toStatusCode(error);
             const requestId = extractRequestId(rawBody);
             logProviderError(logger, provider, "auth", "disconnect", requestId, error, status, finishRequestCost(requestCost));
-            return c.json(toErrorResponse(error, requestId), status);
+            return responseWithErrorObservability(c.json(toErrorResponse(error, requestId), status), error);
         }
     });
     return app;
 }
+function validateStatefulServerConfig(options) {
+    if (options.statefulForwarding && !options.internalOperationExecutor) {
+        throw new Error("Invalid provider server configuration: statefulForwarding requires internalOperationExecutor; missing option internalOperationExecutor.");
+    }
+    if (options.internalOperationExecutor && !options.statefulForwarding?.secret) {
+        throw new Error("Invalid provider server configuration: internalOperationExecutor requires statefulForwarding.secret; missing option statefulForwarding.secret.");
+    }
+    if (options.statefulForwarding &&
+        typeof options.statefulForwarding.validateOwnerFence !== "function") {
+        throw new Error("Invalid provider server configuration: statefulForwarding requires validateOwnerFence.");
+    }
+    if (options.statefulForwarding?.maxSkewMs !== undefined &&
+        (!Number.isFinite(options.statefulForwarding.maxSkewMs) ||
+            options.statefulForwarding.maxSkewMs <= 0)) {
+        throw new Error("Invalid provider server configuration: maxSkewMs must be positive.");
+    }
+    if (options.statefulForwarding?.replayCacheMaxEntries !== undefined &&
+        (!Number.isInteger(options.statefulForwarding.replayCacheMaxEntries) ||
+            options.statefulForwarding.replayCacheMaxEntries <= 0)) {
+        throw new Error("Invalid provider server configuration: replayCacheMaxEntries must be a positive integer.");
+    }
+}
 function getBunServeRuntime() {
     const bunValue = Object.getOwnPropertyDescriptor(globalThis, "Bun")?.value;
     if (!bunValue || typeof bunValue !== "object") {
@@ -1103,6 +1614,9 @@ function getBunServeRuntime() {
         },
     };
 }
+const DEFAULT_SHUTDOWN_TIMEOUT_MS = 30_000;
+const DEFAULT_SHUTDOWN_SIGNALS = ["SIGTERM", "SIGINT"];
+const processSignalCoordinators = new Map();
 export async function serve(provider, options = {}) {
     const bunRuntime = getBunServeRuntime();
     if (bunRuntime === undefined) {
@@ -1110,29 +1624,186 @@ export async function serve(provider, options = {}) {
             code: "RUNTIME_UNSUPPORTED",
         });
     }
+    const logger = options.logger ?? defaultProviderServerLogger;
+    const configuredTimeoutMs = shutdownTimeout(options.shutdown?.timeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS);
+    const configuredSignals = resolveShutdownSignals(options.shutdown?.signals ?? true);
     const app = createServerApp(provider, {
         logger: options.logger,
         stt: options.stt,
+        state: options.state,
+        allowMemoryStateFallback: options.allowMemoryStateFallback,
+        operationExecutor: options.operationExecutor,
+        internalOperationExecutor: options.internalOperationExecutor,
+        statefulForwarding: options.statefulForwarding,
     });
-    bunRuntime.serve({
-        port: options.port ?? DEFAULT_PORT,
-        hostname: options.host ?? DEFAULT_HOST,
-        fetch: app.fetch,
-    });
-    // Internal self-test listener (health dependency inversion): a SEPARATE
-    // socket the tenant-facing gateway never dials. Off by default — it only
-    // starts when the shared self-test master secret env is present.
-    const selfTestSecrets = resolveSelfTestMasterSecrets();
-    if (selfTestSecrets) {
-        const selfTestApp = createSelfTestApp(provider, {
-            secrets: selfTestSecrets,
-            invoke: createSelfTestInvoke(app),
-        });
-        bunRuntime.serve({
-            port: options.selfTestPort ?? resolveSelfTestPort(),
+    const servers = [];
+    try {
+        servers.push(bunRuntime.serve({
+            port: options.port ?? DEFAULT_PORT,
             hostname: options.host ?? DEFAULT_HOST,
-            fetch: selfTestApp.fetch,
-        });
+            fetch: app.fetch,
+        }));
+        // Internal self-test listener (health dependency inversion): a SEPARATE
+        // socket the tenant-facing gateway never dials. Off by default — it only
+        // starts when the shared self-test master secret env is present.
+        const selfTestSecrets = resolveSelfTestMasterSecrets();
+        if (selfTestSecrets) {
+            const selfTestApp = createSelfTestApp(provider, {
+                secrets: selfTestSecrets,
+                invoke: createSelfTestInvoke(app),
+                authFlow: createSelfTestAuthFlowInvoke(app),
+            });
+            servers.push(bunRuntime.serve({
+                port: options.selfTestPort ?? resolveSelfTestPort(),
+                hostname: options.host ?? DEFAULT_HOST,
+                fetch: selfTestApp.fetch,
+            }));
+        }
+    }
+    catch (error) {
+        await Promise.allSettled(servers.map((startedServer) => startedServer.stop(true)));
+        throw error;
+    }
+    const server = servers[0];
+    if (!server)
+        throw new Error("Provider server failed to create its primary listener.");
+    let closePromise;
+    let unregisterSignals = () => { };
+    const close = (closeOptions = {}) => {
+        if (closePromise)
+            return closePromise;
+        const timeoutMs = shutdownTimeout(closeOptions.timeoutMs ?? configuredTimeoutMs);
+        closePromise = closeProviderServers({
+            servers,
+            hooks: options.shutdown?.hooks ?? [],
+            timeoutMs,
+            logger,
+            providerId: provider.id,
+        }).finally(() => unregisterSignals());
+        return closePromise;
+    };
+    try {
+        unregisterSignals = registerForProcessSignals(configuredSignals, () => close({ timeoutMs: configuredTimeoutMs }));
+    }
+    catch (error) {
+        unregisterSignals();
+        await Promise.allSettled(servers.map((startedServer) => startedServer.stop(true)));
+        throw error;
+    }
+    return { port: server.port, close };
+}
+function registerForProcessSignals(signals, close) {
+    if (signals.length === 0)
+        return () => { };
+    const registration = {
+        signals: new Set(signals),
+        close,
+    };
+    let registered = true;
+    const unregister = () => {
+        if (!registered)
+            return;
+        registered = false;
+        for (const signal of registration.signals) {
+            const coordinator = processSignalCoordinators.get(signal);
+            if (!coordinator)
+                continue;
+            coordinator.registrations.delete(registration);
+            if (coordinator.registrations.size === 0 && !coordinator.handling) {
+                process.removeListener(signal, coordinator.listener);
+                processSignalCoordinators.delete(signal);
+            }
+        }
+    };
+    try {
+        for (const signal of signals) {
+            let coordinator = processSignalCoordinators.get(signal);
+            if (!coordinator) {
+                const created = {
+                    registrations: new Set(),
+                    handling: false,
+                    listener: () => handleCoordinatedSignal(signal, created),
+                };
+                coordinator = created;
+                processSignalCoordinators.set(signal, coordinator);
+                process.on(signal, coordinator.listener);
+            }
+            coordinator.registrations.add(registration);
+        }
+    }
+    catch (error) {
+        unregister();
+        throw error;
+    }
+    return unregister;
+}
+function handleCoordinatedSignal(signal, coordinator) {
+    if (coordinator.handling)
+        return;
+    coordinator.handling = true;
+    const registrations = [...coordinator.registrations];
+    void Promise.allSettled(registrations.map((registration) => registration.close())).finally(() => {
+        if (processSignalCoordinators.get(signal) === coordinator) {
+            process.removeListener(signal, coordinator.listener);
+            processSignalCoordinators.delete(signal);
+        }
+        try {
+            process.kill(process.pid, signal);
+        }
+        catch {
+            process.exitCode = 1;
+        }
+    });
+}
+async function closeProviderServers(input) {
+    const deadline = Date.now() + input.timeoutMs;
+    const gracefulStops = input.servers.map((server) => server.stop(false));
+    for (const gracefulStop of gracefulStops)
+        gracefulStop.catch(() => undefined);
+    for (const [hookIndex, hook] of input.hooks.entries()) {
+        try {
+            await withinShutdownBudget(Promise.resolve().then(hook), deadline);
+        }
+        catch (error) {
+            try {
+                input.logger({
+                    level: "error",
+                    event: "provider_shutdown_hook_failed",
+                    providerId: input.providerId,
+                    hookIndex,
+                    errorClass: error instanceof Error ? error.name : "UnknownError",
+                    message: error instanceof Error ? error.message : "Shutdown hook failed.",
+                });
+            }
+            catch { }
+        }
+    }
+    const forcedStops = input.servers.map((server) => server.stop(true));
+    await withinShutdownBudget(Promise.allSettled([...gracefulStops, ...forcedStops]).then(() => undefined), deadline).catch(() => undefined);
+}
+async function withinShutdownBudget(promise, deadline) {
+    promise.catch(() => undefined);
+    const remainingMs = Math.max(0, deadline - Date.now());
+    let timer;
+    const timeout = new Promise((_resolve, reject) => {
+        timer = setTimeout(() => reject(new Error("Provider server shutdown timed out.")), remainingMs);
+    });
+    try {
+        return await Promise.race([promise, timeout]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+function resolveShutdownSignals(signals) {
+    if (signals === false)
+        return [];
+    return [...new Set(signals === true ? DEFAULT_SHUTDOWN_SIGNALS : signals)];
+}
+function shutdownTimeout(value) {
+    if (!Number.isFinite(value) || value < 0) {
+        throw new Error("Provider server shutdown timeoutMs must be a non-negative finite number.");
     }
-    await Promise.resolve();
+    return value;
 }