@apifuse/provider-sdk 2.2.0-beta.1 → 2.2.0-beta.11

package/dist/server/serve.js

@@ -2,36 +2,67 @@ import { existsSync } from "node:fs";
 import { join } from "node:path";
 import { Hono } from "hono";
 import { z } from "zod";
-import { AuthAbortError, createAuthFlowHelpers } from "../auth";
-import { AuthError, ProviderError, SessionExpiredError, TransportError, } from "../errors";
-import { loadProviderLocaleCatalogs, localizeAuthTurn, } from "../i18n/catalog";
-import { categoryForStatus, isRetryableCategory, PROVIDER_OBSERVABILITY_TAXONOMY_VERSION, } from "../observability";
-import { createScratchpad } from "../runtime/auth-flow";
-import { createBrowserClient } from "../runtime/browser";
-import { createProviderCache } from "../runtime/cache";
-import { createProviderChoiceContext, PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV, } from "../runtime/choice";
-import { createCredentialContext } from "../runtime/credential";
-import { createEnvContext } from "../runtime/env";
-import { executeOperation } from "../runtime/executor";
-import { createHttpClient } from "../runtime/http";
-import { wrapWithInstrumentation } from "../runtime/instrumentation";
-import { getProviderBaseUrl } from "../runtime/provider";
-import { PROXY_AUTH_IP_DENIED_CODE, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_EXHAUSTED_CODE, } from "../runtime/proxy-errors";
-import { PROVIDER_TELEMETRY_HEADER, ProxyTelemetryCollector, } from "../runtime/proxy-telemetry";
-import { createProviderRuntimeStateFromEnv, createUnsupportedProviderRuntimeState, } from "../runtime/state";
-import { createStealthClient } from "../runtime/stealth";
-import { createSttClientFromEnv } from "../runtime/stt";
-import { createTraceContext } from "../runtime/trace";
-import { parseSchema } from "../schema";
-import { getStealthProfile } from "../stealth/profiles";
-import { APIFUSE_STREAM_DONE_EVENT, APIFUSE_STREAM_ERROR_EVENT, encodeSseEvent, error as streamError, } from "../stream";
-import { createSelfTestApp, createSelfTestInvoke, resolveSelfTestPort, } from "./self-test";
-import { resolveSelfTestMasterSecrets } from "./self-test-token";
-import { AuthFlowRequestSchema, OperationRequestSchema, } from "./types";
+import { AuthAbortError, createAuthFlowHelpers } from "../auth.js";
+import { AuthError, isProviderError, isSessionExpiredError, isTransportError, ProviderError, } from "../errors.js";
+import { loadProviderLocaleCatalogs, localizeAuthTurn, } from "../i18n/catalog.js";
+import { categoryForStatus, isRetryableCategory, PROVIDER_OBSERVABILITY_TAXONOMY_VERSION, } from "../observability.js";
+import { createScratchpad } from "../runtime/auth-flow.js";
+import { createBrowserClient } from "../runtime/browser.js";
+import { createProviderCache } from "../runtime/cache.js";
+import { createProviderChoiceContext, PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV, } from "../runtime/choice.js";
+import { createCredentialContext } from "../runtime/credential.js";
+import { createEnvContext } from "../runtime/env.js";
+import { executeOperation } from "../runtime/executor.js";
+import { createHttpClient } from "../runtime/http.js";
+import { wrapWithInstrumentation } from "../runtime/instrumentation.js";
+import { getProviderBaseUrl } from "../runtime/provider.js";
+import { PROXY_AUTH_IP_DENIED_CODE, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_EXHAUSTED_CODE, } from "../runtime/proxy-errors.js";
+import { PROVIDER_TELEMETRY_HEADER, ProxyTelemetryCollector } from "../runtime/proxy-telemetry.js";
+import { assertRequiredSecretsPresent, listMissingRequiredSecrets, MISSING_SECRET_CODE, } from "../runtime/secrets.js";
+import { createProviderRuntimeStateFromEnv, createUnsupportedProviderRuntimeState, } from "../runtime/state.js";
+import { createStealthClient } from "../runtime/stealth.js";
+import { createSttClientFromEnv } from "../runtime/stt.js";
+import { createTraceContext } from "../runtime/trace.js";
+import { parseSchema } from "../schema.js";
+import { STATEFUL_NONCE_HEADER as STATEFUL_FORWARDING_NONCE_HEADER, STATEFUL_SIGNATURE_HEADER as STATEFUL_FORWARDING_SIGNATURE_HEADER, STATEFUL_TIMESTAMP_HEADER as STATEFUL_FORWARDING_TIMESTAMP_HEADER, verifyStatefulRequestSignature, } from "../stateful-signing.js";
+import { StatefulRoutingDeadlineError } from "../stateful/stateful-provider-session-routing.js";
+import { getStealthProfile } from "../stealth/profiles.js";
+import { APIFUSE_STREAM_DONE_EVENT, APIFUSE_STREAM_ERROR_EVENT, encodeSseEvent, error as streamError, } from "../stream.js";
+import { createSelfTestApp, createSelfTestAuthFlowInvoke, createSelfTestInvoke, resolveSelfTestPort, } from "./self-test.js";
+import { resolveSelfTestMasterSecrets } from "./self-test-token.js";
+import { AuthFlowRequestSchema, OperationConnectionSchema, OperationRequestSchema, } from "./types.js";
 const DEFAULT_HOST = "0.0.0.0";
 const DEFAULT_PORT = 3000;
 const AUTH_FLOW_LOCALES = ["en", "ko", "ja"];
 const retryResponseMeta = new WeakMap();
+const STATEFUL_INTERNAL_OPERATIONS_ROUTE = "/__apifuse/stateful/operations";
+const STATEFUL_FORWARDING_SOURCE_POD_HEADER = "x-apifuse-stateful-source-pod";
+const DEFAULT_STATEFUL_FORWARDING_MAX_SKEW_MS = 5 * 60_000;
+const DEFAULT_STATEFUL_FORWARDING_REPLAY_CACHE_MAX_ENTRIES = 10_000;
+const STATEFUL_FORWARDING_REPLAY_BUCKET_MS = 10_000;
+const STATEFUL_FORWARDING_REPLAY_RETRY_AFTER_SECONDS = Math.ceil(STATEFUL_FORWARDING_REPLAY_BUCKET_MS / 1_000);
+export const ProviderServerStatefulForwardEnvelopeSchema = z
+    .object({
+    requestId: z.string().min(1),
+    providerId: z.string().min(1),
+    operationId: z.string().min(1),
+    sessionKey: z.string().min(1),
+    connectionId: z.string().min(1),
+    serviceAccountId: z.string().min(1),
+    ownerPodId: z.string().min(1),
+    generation: z.number().int().positive(),
+    sourcePodId: z.string().min(1),
+    forwardedAt: z.string().refine((value) => Number.isFinite(Date.parse(value))),
+    deadlineAt: z
+        .string()
+        .refine((value) => Number.isFinite(Date.parse(value)))
+        .optional(),
+    idempotencyKey: z.string().min(1).optional(),
+    operationRequest: OperationRequestSchema.extend({
+        connection: OperationConnectionSchema.strict().optional(),
+    }).strict(),
+})
+    .strict();
 function createAuthStub() {
     return {
         async requestField(name) {
@@ -93,9 +124,7 @@ function getProviderStealthBaseUrl(provider) {
     return firstHost ? `https://${firstHost}` : undefined;
 }
 function getProviderStealthProfile(provider) {
-    return provider.stealth?.profile
-        ? getStealthProfile(provider.stealth.profile)
-        : undefined;
+    return provider.stealth?.profile ? getStealthProfile(provider.stealth.profile) : undefined;
 }
 function isProductionProviderBrowserMode(provider, env = process.env) {
     if (provider.runtime !== "browser") {
@@ -104,18 +133,19 @@ function isProductionProviderBrowserMode(provider, env = process.env) {
     if (env.APIFUSE__PROVIDER__RUNTIME === "browser") {
         return true;
     }
-    return (env.NODE_ENV === "production" && env.APIFUSE__PROVIDER__ID === provider.id);
+    return env.NODE_ENV === "production" && env.APIFUSE__PROVIDER__ID === provider.id;
 }
 export function resolveProviderProxyAffinityKey(provider, request, operationId) {
-    const connectionKey = request.connection?.id ?? request.connection?.externalRef;
-    const affinity = typeof provider.proxy === "object"
-        ? provider.proxy.session?.affinity
-        : undefined;
+    const connectionKey = resolveOperationConnectionId(request) ?? request.connection?.externalRef;
+    const affinity = typeof provider.proxy === "object" ? provider.proxy.session?.affinity : undefined;
     if (affinity === "operation") {
         return `${provider.id}/${operationId}`;
     }
     return connectionKey ?? provider.id;
 }
+function resolveOperationConnectionId(request) {
+    return request.connection?.id ?? request.connectionId;
+}
 function createProviderContext(provider, request, operationId, options = {}, state = createUnsupportedProviderRuntimeState(), proxyTelemetry) {
     const baseUrl = getProviderBaseUrl(provider);
     const stealthBaseUrl = getProviderStealthBaseUrl(provider);
@@ -142,7 +172,7 @@ function createProviderContext(provider, request, operationId, options = {}, sta
         values: request.connection?.secrets,
     });
     const requestContext = {
-        connectionId: request.connection?.id,
+        connectionId: resolveOperationConnectionId(request),
         headers: request.headers ?? {},
     };
     const context = wrapWithInstrumentation({
@@ -290,7 +320,17 @@ function zodDetails(error) {
     }));
 }
 function toErrorResponse(error, requestId) {
-    if (error instanceof ProviderError) {
+    if (error instanceof StatefulRoutingDeadlineError) {
+        return {
+            error: {
+                code: "STATEFUL_FORWARDING_DEADLINE_EXPIRED",
+                message: "Stateful forwarding deadline expired.",
+                ...(requestId ? { requestId } : {}),
+                details: { retryable: false },
+            },
+        };
+    }
+    if (isProviderError(error)) {
         const details = publicProviderErrorDetails(error);
         return {
             error: {
@@ -312,11 +352,22 @@ function toErrorResponse(error, requestId) {
             },
         };
     }
+    // A masked internal error MUST NOT be advertised as retryable: without an
+    // explicit retryable:false the hub (bori provider-backed engine) defaults 5xx
+    // to retryable:true, which turns a deterministic pre-upstream crash into an
+    // infinite START->CONTINUE->restart loop (2026-07-22 catchtable reserve RCA).
+    // We still refuse to leak message/stack — only the error class name (or the
+    // primitive type for non-Error throwables) is surfaced for ops triage.
     return {
         error: {
             code: "internal_error",
             message: "Internal error",
             ...(requestId ? { requestId } : {}),
+            details: {
+                retryable: false,
+                category: "internal_error",
+                errorClass: error instanceof Error ? error.name : typeof error,
+            },
         },
     };
 }
@@ -340,20 +391,37 @@ function publicProviderErrorDetails(error) {
 function isPlainRecord(value) {
     return value !== null && typeof value === "object" && !Array.isArray(value);
 }
+// Accepts `unknown` so the branded guards narrow cleanly from the top: the
+// subtype error classes are structurally compatible with ProviderError, so
+// narrowing from a ProviderError-typed value would collapse the negative branch
+// to `never`. Narrowing from unknown avoids that while still recognizing errors
+// from a duplicate SDK module instance.
 function providerObservabilityDetails(error) {
     // Session-expiry surfaces the credential_expired category + the opt-in
     // retryable signal so Gateway/Credential Service can refresh and re-drive the
     // operation (see design.md §4.3 D3). Without this branch the auth error would
     // serialize as a bare 401 with no retryable/category, losing the refresh
     // signal for exactly the retryOnAuthRefresh operations it is meant to enable.
-    if (error instanceof SessionExpiredError) {
+    if (isSessionExpiredError(error)) {
         return {
             category: error.options?.category ?? "credential_expired",
             taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
             retryable: error.options?.retryable ?? false,
         };
     }
-    if (!(error instanceof TransportError)) {
+    // Missing-secret errors carry the canonical credential_unavailable category
+    // so Gateway/observability can attribute the failure to provisioning, not
+    // the upstream. Matched by code (not constructor) so both the SDK-owned
+    // runtime gate and any not-yet-migrated provider-thrown MISSING_SECRET
+    // serialize identically, including across duplicate SDK module instances.
+    if (isProviderError(error) && error.code === MISSING_SECRET_CODE) {
+        return {
+            category: error.options?.category ?? "credential_unavailable",
+            taxonomyVersion: PROVIDER_OBSERVABILITY_TAXONOMY_VERSION,
+            retryable: error.options?.retryable ?? false,
+        };
+    }
+    if (!isTransportError(error)) {
         return undefined;
     }
     const isProxyPoolCode = error.code === PROXY_POOL_EXHAUSTED_CODE ||
@@ -382,7 +450,7 @@ function providerObservabilityDetails(error) {
     };
 }
 function publicProviderErrorMessage(error) {
-    if (error instanceof TransportError) {
+    if (isTransportError(error)) {
         if (error.code === PROXY_AUTH_IP_DENIED_CODE) {
             return error.message;
         }
@@ -410,14 +478,21 @@ function toStatusCode(error) {
     if (error instanceof z.ZodError) {
         return 400;
     }
-    if (error instanceof TransportError) {
+    if (error instanceof StatefulRoutingDeadlineError) {
+        return 504;
+    }
+    if (isTransportError(error)) {
         return error.code === "transport_timeout" ? 504 : 502;
     }
-    if (error instanceof ProviderError) {
+    if (isProviderError(error)) {
         switch (error.code) {
             case "AUTH_REQUIRED":
             case "reauth_required":
                 return 401;
+            // Unprovisioned declared secret: a deployment/config defect, never an
+            // upstream failure — explicit 400 (was only reached via fallthrough).
+            case MISSING_SECRET_CODE:
+                return 400;
             case "NOT_FOUND":
             case "not_found":
             case "NO_DATA":
@@ -431,6 +506,7 @@ function toStatusCode(error) {
                 return 502;
             case "STT_UNAVAILABLE":
             case "UNSUPPORTED_STT_BACKEND":
+            case "STATEFUL_FORWARDING_REPLAY_CACHE_FULL":
                 return 503;
         }
         return 400;
@@ -445,16 +521,16 @@ function extractRequestId(raw) {
     return typeof value === "string" ? value : undefined;
 }
 function logProviderError(logger, provider, kind, route, requestId, error, status, cost) {
-    const code = error instanceof ProviderError
+    const code = isProviderError(error)
         ? (error.code ?? "provider_error")
         : error instanceof z.ZodError
             ? "invalid_request"
-            : "internal_error";
+            : error instanceof StatefulRoutingDeadlineError
+                ? "STATEFUL_FORWARDING_DEADLINE_EXPIRED"
+                : "internal_error";
     const errorClass = error instanceof Error ? error.name : typeof error;
     const message = error instanceof Error ? error.message : String(error);
-    const details = error instanceof ProviderError
-        ? providerObservabilityDetails(error)
-        : undefined;
+    const details = isProviderError(error) ? providerObservabilityDetails(error) : undefined;
     const emit = typeof logger === "function" ? logger : defaultProviderServerLogger;
     emit({
         level: status >= 500 ? "error" : "warn",
@@ -468,7 +544,7 @@ function logProviderError(logger, provider, kind, route, requestId, error, statu
         code,
         errorClass,
         message,
-        ...(error instanceof TransportError && error.upstreamStatus
+        ...(isTransportError(error) && error.upstreamStatus
             ? { upstreamStatus: error.upstreamStatus }
             : {}),
         ...(details
@@ -591,8 +667,7 @@ async function validateSseEvent(operation, event) {
     const transport = getSseTransport(operation);
     const schema = transport?.events?.[event.event];
     if (!schema) {
-        if (event.event === APIFUSE_STREAM_ERROR_EVENT ||
-            event.event === APIFUSE_STREAM_DONE_EVENT) {
+        if (event.event === APIFUSE_STREAM_ERROR_EVENT || event.event === APIFUSE_STREAM_DONE_EVENT) {
             return event;
         }
         throw new ProviderError(`SSE event "${event.event}" is not declared in operation transport.events.`, {
@@ -711,8 +786,7 @@ function enforceStreamChunkLimit(body, maxChunkBytes) {
 }
 function toStreamingResponse(operation, result, cleanup, requestId) {
     const transport = operation.transport?.kind ?? "json";
-    if (transport === "sse" &&
-        (result instanceof Response || result instanceof ReadableStream)) {
+    if (transport === "sse" && (result instanceof Response || result instanceof ReadableStream)) {
         void cleanup();
         throw new ProviderError("SSE operations must return an AsyncIterable of typed stream.event(...) values.", {
             code: "SSE_RESULT_UNSUPPORTED",
@@ -723,9 +797,7 @@ function toStreamingResponse(operation, result, cleanup, requestId) {
     }
     if (result instanceof Response) {
         const httpTransport = getHttpStreamTransport(operation);
-        if (httpTransport &&
-            result.body &&
-            httpTransport?.maxChunkBytes !== undefined) {
+        if (httpTransport && result.body && httpTransport?.maxChunkBytes !== undefined) {
             return responseWithCleanup(new Response(enforceStreamChunkLimit(result.body, httpTransport.maxChunkBytes), {
                 headers: result.headers,
                 status: result.status,
@@ -744,8 +816,7 @@ function toStreamingResponse(operation, result, cleanup, requestId) {
                 ? { "Content-Type": "text/event-stream; charset=utf-8" }
                 : {
                     "Content-Type": operation.transport?.kind === "http-stream"
-                        ? (operation.transport.contentType ??
-                            "application/octet-stream")
+                        ? (operation.transport.contentType ?? "application/octet-stream")
                         : "application/octet-stream",
                 },
         }), cleanup);
@@ -763,9 +834,7 @@ function getSseTransport(operation) {
     return operation.transport?.kind === "sse" ? operation.transport : undefined;
 }
 function getHttpStreamTransport(operation) {
-    return operation.transport?.kind === "http-stream"
-        ? operation.transport
-        : undefined;
+    return operation.transport?.kind === "http-stream" ? operation.transport : undefined;
 }
 function toAuthFlowResponse(result, contextPatch) {
     if (result instanceof Response) {
@@ -793,7 +862,7 @@ function isAuthFlowLocale(value) {
     return value === "en" || value === "ko" || value === "ja";
 }
 function isAuthTurn(value) {
-    return (!!value && typeof value === "object" && "kind" in value && "turnId" in value);
+    return !!value && typeof value === "object" && "kind" in value && "turnId" in value;
 }
 function loadAuthFlowLocaleCatalogs(provider) {
     for (const providerDir of [
@@ -856,7 +925,14 @@ async function handleOperation(provider, request, operationId, options = {}, sta
         }
     };
     try {
-        const result = await executeOperation(provider, operationId, ctx, request.input);
+        const result = options.operationExecutor
+            ? await options.operationExecutor({
+                provider,
+                operationId,
+                ctx,
+                request,
+            })
+            : await executeOperation(provider, operationId, ctx, request.input);
         if (streaming && operation) {
             return toStreamingResponse(operation, result, cleanup, request.requestId);
         }
@@ -890,8 +966,16 @@ async function handleAuthFlow(provider, request, route, options = {}, signal) {
             code: "AUTH_FLOW_NOT_CONFIGURED",
         });
     }
+    // Same SDK-owned gate as executeOperation: OAuth/credentials ceremonies
+    // depend on declared secrets (client ids/secrets), so fail structured before
+    // any flow code runs instead of at whatever point the ceremony first reads
+    // the env. `abort` stays exempt: a user must always be able to cancel a
+    // stranded flow even when provisioning is broken.
     const { context, getPatch } = createAuthFlowContext(provider, request, options, signal);
     try {
+        if (route !== "abort") {
+            assertRequiredSecretsPresent(provider, context.env);
+        }
         const result = route === "start"
             ? await flow.start(context)
             : route === "continue"
@@ -930,14 +1014,142 @@ async function handleAuthFlow(provider, request, route, options = {}, signal) {
         context.stealth.close?.();
     }
 }
+class StatefulForwardingReplayCache {
+    maxEntries;
+    #nonces = new Map();
+    #expiryBuckets = new Map();
+    #nextExpiryBucket;
+    #latestExpiryBucket;
+    constructor(maxEntries) {
+        this.maxEntries = maxEntries;
+    }
+    claim(nonce, expiresAtMs, nowMs) {
+        this.dropExpiredBuckets(nowMs);
+        if (this.#nonces.has(nonce))
+            return "replayed";
+        if (this.#nonces.size >= this.maxEntries)
+            return "full";
+        const expiryBucket = Math.ceil(expiresAtMs / STATEFUL_FORWARDING_REPLAY_BUCKET_MS) *
+            STATEFUL_FORWARDING_REPLAY_BUCKET_MS;
+        this.#nonces.set(nonce, expiryBucket);
+        const bucket = this.#expiryBuckets.get(expiryBucket) ?? new Set();
+        bucket.add(nonce);
+        this.#expiryBuckets.set(expiryBucket, bucket);
+        this.#nextExpiryBucket = Math.min(this.#nextExpiryBucket ?? expiryBucket, expiryBucket);
+        this.#latestExpiryBucket = Math.max(this.#latestExpiryBucket ?? expiryBucket, expiryBucket);
+        return "accepted";
+    }
+    dropExpiredBuckets(nowMs) {
+        if (this.#nextExpiryBucket === undefined || this.#latestExpiryBucket === undefined)
+            return;
+        if (nowMs >= this.#latestExpiryBucket) {
+            this.#nonces.clear();
+            this.#expiryBuckets.clear();
+            this.#nextExpiryBucket = undefined;
+            this.#latestExpiryBucket = undefined;
+            return;
+        }
+        while (this.#nextExpiryBucket <= nowMs) {
+            const bucket = this.#expiryBuckets.get(this.#nextExpiryBucket);
+            if (bucket) {
+                for (const cachedNonce of bucket)
+                    this.#nonces.delete(cachedNonce);
+                this.#expiryBuckets.delete(this.#nextExpiryBucket);
+            }
+            this.#nextExpiryBucket += STATEFUL_FORWARDING_REPLAY_BUCKET_MS;
+        }
+    }
+}
+function verifyStatefulForwardingRequest(input) {
+    const config = input.options.statefulForwarding;
+    if (!config?.secret) {
+        throw new ProviderError("Stateful forwarding is not configured.", {
+            code: "STATEFUL_FORWARDING_NOT_CONFIGURED",
+        });
+    }
+    const timestamp = input.headers.get(STATEFUL_FORWARDING_TIMESTAMP_HEADER) ?? "";
+    const signature = input.headers.get(STATEFUL_FORWARDING_SIGNATURE_HEADER) ?? "";
+    const nonce = input.headers.get(STATEFUL_FORWARDING_NONCE_HEADER) ?? "";
+    if (!timestamp || !signature || !nonce) {
+        throw new ProviderError("Stateful forwarding signature headers are missing.", {
+            code: "STATEFUL_FORWARDING_SIGNATURE_MISSING",
+        });
+    }
+    if (nonce.length > 256) {
+        throw new ProviderError("Stateful forwarding nonce is invalid.", {
+            code: "STATEFUL_FORWARDING_NONCE_INVALID",
+        });
+    }
+    const timestampMs = Date.parse(timestamp);
+    const maxSkewMs = config.maxSkewMs ?? DEFAULT_STATEFUL_FORWARDING_MAX_SKEW_MS;
+    if (!Number.isFinite(timestampMs) || Math.abs(Date.now() - timestampMs) > maxSkewMs) {
+        throw new ProviderError("Stateful forwarding signature timestamp is outside the allowed skew.", { code: "STATEFUL_FORWARDING_TIMESTAMP_INVALID" });
+    }
+    if (!verifyStatefulRequestSignature({
+        secret: config.secret,
+        timestamp,
+        rawBody: input.rawBody,
+        method: input.method,
+        path: input.path,
+        nonce,
+        signature,
+    })) {
+        throw new ProviderError("Stateful forwarding signature is invalid.", {
+            code: "STATEFUL_FORWARDING_SIGNATURE_INVALID",
+        });
+    }
+    const replayResult = input.replayCache.claim(nonce, timestampMs + maxSkewMs, Date.now());
+    if (replayResult === "replayed") {
+        throw new ProviderError("Stateful forwarding nonce has already been used.", {
+            code: "STATEFUL_FORWARDING_REPLAY_DETECTED",
+        });
+    }
+    if (replayResult === "full") {
+        throw new ProviderError("Stateful forwarding replay cache is at capacity.", {
+            code: "STATEFUL_FORWARDING_REPLAY_CACHE_FULL",
+        });
+    }
+}
+function operationRequestFromForwardingEnvelope(envelope) {
+    return {
+        ...envelope.operationRequest,
+        ...(envelope.deadlineAt !== undefined ? { deadlineAt: envelope.deadlineAt } : {}),
+    };
+}
+function parseStatefulForwardingEnvelope(rawBody) {
+    const parsed = ProviderServerStatefulForwardEnvelopeSchema.safeParse(rawBody);
+    if (parsed.success)
+        return parsed.data;
+    throw new ProviderError("Stateful forwarding envelope is invalid.", {
+        code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
+        details: zodDetails(parsed.error),
+    });
+}
 export function createServerApp(provider, options = {}) {
+    validateStatefulServerConfig(options);
     const app = new Hono();
     const logger = options.logger ?? defaultProviderServerLogger;
+    const statefulForwardingReplayCache = new StatefulForwardingReplayCache(options.statefulForwarding?.replayCacheMaxEntries ??
+        DEFAULT_STATEFUL_FORWARDING_REPLAY_CACHE_MAX_ENTRIES);
     const state = options.state ??
         createProviderRuntimeStateFromEnv({
             providerId: provider.id,
             allowMemoryFallback: options.allowMemoryStateFallback === true,
         });
+    // Boot-time visibility for unprovisioned declared secrets: emit a structured
+    // warn so deploy tooling/alerting sees the gap the moment the pod boots,
+    // instead of discovering it request-by-request. Deliberately log-only — a
+    // boot crash would trade a structured MISSING_SECRET signal for
+    // CrashLoopBackOff. Requests still fail closed via the executeOperation gate.
+    const missingSecretsAtBoot = listMissingRequiredSecrets(provider, createEnvContext(provider.secrets?.map((secret) => secret.name)));
+    if (missingSecretsAtBoot.length > 0) {
+        logger({
+            level: "warn",
+            event: "provider_secrets_missing",
+            providerId: provider.id,
+            missingSecrets: missingSecretsAtBoot,
+        });
+    }
     app.notFound((c) => c.json({
         error: {
             code: "not_found",
@@ -949,6 +1161,120 @@ export function createServerApp(provider, options = {}) {
         provider: provider.id,
         version: provider.version,
     }));
+    app.post(STATEFUL_INTERNAL_OPERATIONS_ROUTE, async (c) => {
+        let rawBodyText = "";
+        let rawBody;
+        const operation = "stateful-internal";
+        const requestCost = startRequestCost();
+        try {
+            if (!options.internalOperationExecutor) {
+                throw new ProviderError("Stateful internal operation executor is not configured.", {
+                    code: "STATEFUL_INTERNAL_EXECUTOR_NOT_CONFIGURED",
+                });
+            }
+            rawBodyText = await c.req.raw.clone().text();
+            verifyStatefulForwardingRequest({
+                options,
+                rawBody: rawBodyText,
+                headers: c.req.raw.headers,
+                method: c.req.raw.method,
+                path: STATEFUL_INTERNAL_OPERATIONS_ROUTE,
+                replayCache: statefulForwardingReplayCache,
+            });
+            try {
+                rawBody = JSON.parse(rawBodyText);
+            }
+            catch {
+                throw new ProviderError("Stateful forwarding envelope is not valid JSON.", {
+                    code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
+                });
+            }
+            const envelope = parseStatefulForwardingEnvelope(rawBody);
+            if (envelope.providerId !== provider.id) {
+                throw new ProviderError("Stateful forwarding envelope providerId does not match the served provider.", { code: "STATEFUL_FORWARDING_PROVIDER_MISMATCH" });
+            }
+            if (envelope.requestId !== envelope.operationRequest.requestId) {
+                throw new ProviderError("Stateful forwarding requestId values do not match.", {
+                    code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
+                });
+            }
+            if (envelope.sourcePodId !==
+                (c.req.raw.headers.get(STATEFUL_FORWARDING_SOURCE_POD_HEADER) ?? "")) {
+                throw new ProviderError("Stateful forwarding source pod does not match its header.", {
+                    code: "STATEFUL_FORWARDING_SOURCE_POD_MISMATCH",
+                });
+            }
+            if (envelope.forwardedAt !== (c.req.raw.headers.get(STATEFUL_FORWARDING_TIMESTAMP_HEADER) ?? "")) {
+                throw new ProviderError("Stateful forwarding forwardedAt does not match its signature timestamp.", { code: "STATEFUL_FORWARDING_ENVELOPE_INVALID" });
+            }
+            const deadlineAtMs = envelope.deadlineAt ? Date.parse(envelope.deadlineAt) : undefined;
+            if (deadlineAtMs !== undefined && deadlineAtMs <= Date.now()) {
+                throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt);
+            }
+            const remainingDeadlineMs = deadlineAtMs === undefined ? undefined : deadlineAtMs - Date.now();
+            const deadlineSignal = remainingDeadlineMs === undefined ? undefined : AbortSignal.timeout(remainingDeadlineMs);
+            const signal = deadlineSignal
+                ? AbortSignal.any([c.req.raw.signal, deadlineSignal])
+                : c.req.raw.signal;
+            const ownerFenceValidation = Promise.resolve(options.statefulForwarding?.validateOwnerFence({
+                providerId: envelope.providerId,
+                sessionKey: envelope.sessionKey,
+                ownerPodId: envelope.ownerPodId,
+                generation: envelope.generation,
+                sourcePodId: envelope.sourcePodId,
+                forwardedAt: envelope.forwardedAt,
+                requestId: envelope.requestId,
+                ...(envelope.idempotencyKey ? { idempotencyKey: envelope.idempotencyKey } : {}),
+            }, signal));
+            let ownerFenceValid;
+            try {
+                ownerFenceValid = deadlineSignal
+                    ? await Promise.race([
+                        ownerFenceValidation,
+                        new Promise((_resolve, reject) => {
+                            deadlineSignal.addEventListener("abort", () => reject(new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt)), { once: true });
+                        }),
+                    ])
+                    : await ownerFenceValidation;
+            }
+            catch (error) {
+                if (deadlineSignal?.aborted) {
+                    throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt);
+                }
+                throw error;
+            }
+            if (ownerFenceValid !== true) {
+                throw new ProviderError("Stateful forwarding owner fence is no longer current.", {
+                    code: "STATEFUL_FORWARDING_OWNER_FENCE_INVALID",
+                });
+            }
+            const request = operationRequestFromForwardingEnvelope(envelope);
+            const operationId = envelope.operationId;
+            const ctx = createProviderContext(provider, request, operationId, options, state);
+            if (deadlineAtMs !== undefined && deadlineAtMs <= Date.now()) {
+                throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt);
+            }
+            const output = await options.internalOperationExecutor({
+                provider,
+                operationId,
+                ctx,
+                request,
+                internalStatefulForward: envelope,
+                signal,
+            });
+            logProviderSuccess(logger, provider, "operation", operationId || operation, request.requestId, 200, finishRequestCost(requestCost));
+            return c.json({ data: output });
+        }
+        catch (error) {
+            const status = toStatusCode(error);
+            if (isProviderError(error) && error.code === "STATEFUL_FORWARDING_REPLAY_CACHE_FULL") {
+                c.header("Retry-After", String(STATEFUL_FORWARDING_REPLAY_RETRY_AFTER_SECONDS));
+            }
+            const requestId = extractRequestId(rawBody);
+            logProviderError(logger, provider, "operation", operation, requestId, error, status, finishRequestCost(requestCost));
+            return c.json(toErrorResponse(error, requestId), status);
+        }
+    });
     app.post("/v1/:operation", async (c) => {
         let rawBody;
         const operation = c.req.param("operation");
@@ -1085,6 +1411,28 @@ export function createServerApp(provider, options = {}) {
     });
     return app;
 }
+function validateStatefulServerConfig(options) {
+    if (options.statefulForwarding && !options.internalOperationExecutor) {
+        throw new Error("Invalid provider server configuration: statefulForwarding requires internalOperationExecutor; missing option internalOperationExecutor.");
+    }
+    if (options.internalOperationExecutor && !options.statefulForwarding?.secret) {
+        throw new Error("Invalid provider server configuration: internalOperationExecutor requires statefulForwarding.secret; missing option statefulForwarding.secret.");
+    }
+    if (options.statefulForwarding &&
+        typeof options.statefulForwarding.validateOwnerFence !== "function") {
+        throw new Error("Invalid provider server configuration: statefulForwarding requires validateOwnerFence.");
+    }
+    if (options.statefulForwarding?.maxSkewMs !== undefined &&
+        (!Number.isFinite(options.statefulForwarding.maxSkewMs) ||
+            options.statefulForwarding.maxSkewMs <= 0)) {
+        throw new Error("Invalid provider server configuration: maxSkewMs must be positive.");
+    }
+    if (options.statefulForwarding?.replayCacheMaxEntries !== undefined &&
+        (!Number.isInteger(options.statefulForwarding.replayCacheMaxEntries) ||
+            options.statefulForwarding.replayCacheMaxEntries <= 0)) {
+        throw new Error("Invalid provider server configuration: replayCacheMaxEntries must be a positive integer.");
+    }
+}
 function getBunServeRuntime() {
     const bunValue = Object.getOwnPropertyDescriptor(globalThis, "Bun")?.value;
     if (!bunValue || typeof bunValue !== "object") {
@@ -1100,6 +1448,9 @@ function getBunServeRuntime() {
         },
     };
 }
+const DEFAULT_SHUTDOWN_TIMEOUT_MS = 30_000;
+const DEFAULT_SHUTDOWN_SIGNALS = ["SIGTERM", "SIGINT"];
+const processSignalCoordinators = new Map();
 export async function serve(provider, options = {}) {
     const bunRuntime = getBunServeRuntime();
     if (bunRuntime === undefined) {
@@ -1107,29 +1458,186 @@ export async function serve(provider, options = {}) {
             code: "RUNTIME_UNSUPPORTED",
         });
     }
+    const logger = options.logger ?? defaultProviderServerLogger;
+    const configuredTimeoutMs = shutdownTimeout(options.shutdown?.timeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS);
+    const configuredSignals = resolveShutdownSignals(options.shutdown?.signals ?? true);
     const app = createServerApp(provider, {
         logger: options.logger,
         stt: options.stt,
+        state: options.state,
+        allowMemoryStateFallback: options.allowMemoryStateFallback,
+        operationExecutor: options.operationExecutor,
+        internalOperationExecutor: options.internalOperationExecutor,
+        statefulForwarding: options.statefulForwarding,
     });
-    bunRuntime.serve({
-        port: options.port ?? DEFAULT_PORT,
-        hostname: options.host ?? DEFAULT_HOST,
-        fetch: app.fetch,
-    });
-    // Internal self-test listener (health dependency inversion): a SEPARATE
-    // socket the tenant-facing gateway never dials. Off by default — it only
-    // starts when the shared self-test master secret env is present.
-    const selfTestSecrets = resolveSelfTestMasterSecrets();
-    if (selfTestSecrets) {
-        const selfTestApp = createSelfTestApp(provider, {
-            secrets: selfTestSecrets,
-            invoke: createSelfTestInvoke(app),
-        });
-        bunRuntime.serve({
-            port: options.selfTestPort ?? resolveSelfTestPort(),
+    const servers = [];
+    try {
+        servers.push(bunRuntime.serve({
+            port: options.port ?? DEFAULT_PORT,
             hostname: options.host ?? DEFAULT_HOST,
-            fetch: selfTestApp.fetch,
-        });
+            fetch: app.fetch,
+        }));
+        // Internal self-test listener (health dependency inversion): a SEPARATE
+        // socket the tenant-facing gateway never dials. Off by default — it only
+        // starts when the shared self-test master secret env is present.
+        const selfTestSecrets = resolveSelfTestMasterSecrets();
+        if (selfTestSecrets) {
+            const selfTestApp = createSelfTestApp(provider, {
+                secrets: selfTestSecrets,
+                invoke: createSelfTestInvoke(app),
+                authFlow: createSelfTestAuthFlowInvoke(app),
+            });
+            servers.push(bunRuntime.serve({
+                port: options.selfTestPort ?? resolveSelfTestPort(),
+                hostname: options.host ?? DEFAULT_HOST,
+                fetch: selfTestApp.fetch,
+            }));
+        }
+    }
+    catch (error) {
+        await Promise.allSettled(servers.map((startedServer) => startedServer.stop(true)));
+        throw error;
+    }
+    const server = servers[0];
+    if (!server)
+        throw new Error("Provider server failed to create its primary listener.");
+    let closePromise;
+    let unregisterSignals = () => { };
+    const close = (closeOptions = {}) => {
+        if (closePromise)
+            return closePromise;
+        const timeoutMs = shutdownTimeout(closeOptions.timeoutMs ?? configuredTimeoutMs);
+        closePromise = closeProviderServers({
+            servers,
+            hooks: options.shutdown?.hooks ?? [],
+            timeoutMs,
+            logger,
+            providerId: provider.id,
+        }).finally(() => unregisterSignals());
+        return closePromise;
+    };
+    try {
+        unregisterSignals = registerForProcessSignals(configuredSignals, () => close({ timeoutMs: configuredTimeoutMs }));
+    }
+    catch (error) {
+        unregisterSignals();
+        await Promise.allSettled(servers.map((startedServer) => startedServer.stop(true)));
+        throw error;
+    }
+    return { port: server.port, close };
+}
+function registerForProcessSignals(signals, close) {
+    if (signals.length === 0)
+        return () => { };
+    const registration = {
+        signals: new Set(signals),
+        close,
+    };
+    let registered = true;
+    const unregister = () => {
+        if (!registered)
+            return;
+        registered = false;
+        for (const signal of registration.signals) {
+            const coordinator = processSignalCoordinators.get(signal);
+            if (!coordinator)
+                continue;
+            coordinator.registrations.delete(registration);
+            if (coordinator.registrations.size === 0 && !coordinator.handling) {
+                process.removeListener(signal, coordinator.listener);
+                processSignalCoordinators.delete(signal);
+            }
+        }
+    };
+    try {
+        for (const signal of signals) {
+            let coordinator = processSignalCoordinators.get(signal);
+            if (!coordinator) {
+                const created = {
+                    registrations: new Set(),
+                    handling: false,
+                    listener: () => handleCoordinatedSignal(signal, created),
+                };
+                coordinator = created;
+                processSignalCoordinators.set(signal, coordinator);
+                process.on(signal, coordinator.listener);
+            }
+            coordinator.registrations.add(registration);
+        }
+    }
+    catch (error) {
+        unregister();
+        throw error;
+    }
+    return unregister;
+}
+function handleCoordinatedSignal(signal, coordinator) {
+    if (coordinator.handling)
+        return;
+    coordinator.handling = true;
+    const registrations = [...coordinator.registrations];
+    void Promise.allSettled(registrations.map((registration) => registration.close())).finally(() => {
+        if (processSignalCoordinators.get(signal) === coordinator) {
+            process.removeListener(signal, coordinator.listener);
+            processSignalCoordinators.delete(signal);
+        }
+        try {
+            process.kill(process.pid, signal);
+        }
+        catch {
+            process.exitCode = 1;
+        }
+    });
+}
+async function closeProviderServers(input) {
+    const deadline = Date.now() + input.timeoutMs;
+    const gracefulStops = input.servers.map((server) => server.stop(false));
+    for (const gracefulStop of gracefulStops)
+        gracefulStop.catch(() => undefined);
+    for (const [hookIndex, hook] of input.hooks.entries()) {
+        try {
+            await withinShutdownBudget(Promise.resolve().then(hook), deadline);
+        }
+        catch (error) {
+            try {
+                input.logger({
+                    level: "error",
+                    event: "provider_shutdown_hook_failed",
+                    providerId: input.providerId,
+                    hookIndex,
+                    errorClass: error instanceof Error ? error.name : "UnknownError",
+                    message: error instanceof Error ? error.message : "Shutdown hook failed.",
+                });
+            }
+            catch { }
+        }
+    }
+    const forcedStops = input.servers.map((server) => server.stop(true));
+    await withinShutdownBudget(Promise.allSettled([...gracefulStops, ...forcedStops]).then(() => undefined), deadline).catch(() => undefined);
+}
+async function withinShutdownBudget(promise, deadline) {
+    promise.catch(() => undefined);
+    const remainingMs = Math.max(0, deadline - Date.now());
+    let timer;
+    const timeout = new Promise((_resolve, reject) => {
+        timer = setTimeout(() => reject(new Error("Provider server shutdown timed out.")), remainingMs);
+    });
+    try {
+        return await Promise.race([promise, timeout]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+function resolveShutdownSignals(signals) {
+    if (signals === false)
+        return [];
+    return [...new Set(signals === true ? DEFAULT_SHUTDOWN_SIGNALS : signals)];
+}
+function shutdownTimeout(value) {
+    if (!Number.isFinite(value) || value < 0) {
+        throw new Error("Provider server shutdown timeoutMs must be a non-negative finite number.");
     }
-    await Promise.resolve();
+    return value;
 }