@apifuse/provider-sdk 2.2.0-beta.10 → 2.2.0-beta.11

package/dist/server/serve.js

@@ -24,15 +24,45 @@ import { createStealthClient } from "../runtime/stealth.js";
 import { createSttClientFromEnv } from "../runtime/stt.js";
 import { createTraceContext } from "../runtime/trace.js";
 import { parseSchema } from "../schema.js";
+import { STATEFUL_NONCE_HEADER as STATEFUL_FORWARDING_NONCE_HEADER, STATEFUL_SIGNATURE_HEADER as STATEFUL_FORWARDING_SIGNATURE_HEADER, STATEFUL_TIMESTAMP_HEADER as STATEFUL_FORWARDING_TIMESTAMP_HEADER, verifyStatefulRequestSignature, } from "../stateful-signing.js";
+import { StatefulRoutingDeadlineError } from "../stateful/stateful-provider-session-routing.js";
 import { getStealthProfile } from "../stealth/profiles.js";
 import { APIFUSE_STREAM_DONE_EVENT, APIFUSE_STREAM_ERROR_EVENT, encodeSseEvent, error as streamError, } from "../stream.js";
 import { createSelfTestApp, createSelfTestAuthFlowInvoke, createSelfTestInvoke, resolveSelfTestPort, } from "./self-test.js";
 import { resolveSelfTestMasterSecrets } from "./self-test-token.js";
-import { AuthFlowRequestSchema, OperationRequestSchema, } from "./types.js";
+import { AuthFlowRequestSchema, OperationConnectionSchema, OperationRequestSchema, } from "./types.js";
 const DEFAULT_HOST = "0.0.0.0";
 const DEFAULT_PORT = 3000;
 const AUTH_FLOW_LOCALES = ["en", "ko", "ja"];
 const retryResponseMeta = new WeakMap();
+const STATEFUL_INTERNAL_OPERATIONS_ROUTE = "/__apifuse/stateful/operations";
+const STATEFUL_FORWARDING_SOURCE_POD_HEADER = "x-apifuse-stateful-source-pod";
+const DEFAULT_STATEFUL_FORWARDING_MAX_SKEW_MS = 5 * 60_000;
+const DEFAULT_STATEFUL_FORWARDING_REPLAY_CACHE_MAX_ENTRIES = 10_000;
+const STATEFUL_FORWARDING_REPLAY_BUCKET_MS = 10_000;
+const STATEFUL_FORWARDING_REPLAY_RETRY_AFTER_SECONDS = Math.ceil(STATEFUL_FORWARDING_REPLAY_BUCKET_MS / 1_000);
+export const ProviderServerStatefulForwardEnvelopeSchema = z
+    .object({
+    requestId: z.string().min(1),
+    providerId: z.string().min(1),
+    operationId: z.string().min(1),
+    sessionKey: z.string().min(1),
+    connectionId: z.string().min(1),
+    serviceAccountId: z.string().min(1),
+    ownerPodId: z.string().min(1),
+    generation: z.number().int().positive(),
+    sourcePodId: z.string().min(1),
+    forwardedAt: z.string().refine((value) => Number.isFinite(Date.parse(value))),
+    deadlineAt: z
+        .string()
+        .refine((value) => Number.isFinite(Date.parse(value)))
+        .optional(),
+    idempotencyKey: z.string().min(1).optional(),
+    operationRequest: OperationRequestSchema.extend({
+        connection: OperationConnectionSchema.strict().optional(),
+    }).strict(),
+})
+    .strict();
 function createAuthStub() {
     return {
         async requestField(name) {
@@ -290,6 +320,16 @@ function zodDetails(error) {
     }));
 }
 function toErrorResponse(error, requestId) {
+    if (error instanceof StatefulRoutingDeadlineError) {
+        return {
+            error: {
+                code: "STATEFUL_FORWARDING_DEADLINE_EXPIRED",
+                message: "Stateful forwarding deadline expired.",
+                ...(requestId ? { requestId } : {}),
+                details: { retryable: false },
+            },
+        };
+    }
     if (isProviderError(error)) {
         const details = publicProviderErrorDetails(error);
         return {
@@ -438,6 +478,9 @@ function toStatusCode(error) {
     if (error instanceof z.ZodError) {
         return 400;
     }
+    if (error instanceof StatefulRoutingDeadlineError) {
+        return 504;
+    }
     if (isTransportError(error)) {
         return error.code === "transport_timeout" ? 504 : 502;
     }
@@ -463,6 +506,7 @@ function toStatusCode(error) {
                 return 502;
             case "STT_UNAVAILABLE":
             case "UNSUPPORTED_STT_BACKEND":
+            case "STATEFUL_FORWARDING_REPLAY_CACHE_FULL":
                 return 503;
         }
         return 400;
@@ -481,7 +525,9 @@ function logProviderError(logger, provider, kind, route, requestId, error, statu
         ? (error.code ?? "provider_error")
         : error instanceof z.ZodError
             ? "invalid_request"
-            : "internal_error";
+            : error instanceof StatefulRoutingDeadlineError
+                ? "STATEFUL_FORWARDING_DEADLINE_EXPIRED"
+                : "internal_error";
     const errorClass = error instanceof Error ? error.name : typeof error;
     const message = error instanceof Error ? error.message : String(error);
     const details = isProviderError(error) ? providerObservabilityDetails(error) : undefined;
@@ -879,7 +925,14 @@ async function handleOperation(provider, request, operationId, options = {}, sta
         }
     };
     try {
-        const result = await executeOperation(provider, operationId, ctx, request.input);
+        const result = options.operationExecutor
+            ? await options.operationExecutor({
+                provider,
+                operationId,
+                ctx,
+                request,
+            })
+            : await executeOperation(provider, operationId, ctx, request.input);
         if (streaming && operation) {
             return toStreamingResponse(operation, result, cleanup, request.requestId);
         }
@@ -961,9 +1014,123 @@ async function handleAuthFlow(provider, request, route, options = {}, signal) {
         context.stealth.close?.();
     }
 }
+class StatefulForwardingReplayCache {
+    maxEntries;
+    #nonces = new Map();
+    #expiryBuckets = new Map();
+    #nextExpiryBucket;
+    #latestExpiryBucket;
+    constructor(maxEntries) {
+        this.maxEntries = maxEntries;
+    }
+    claim(nonce, expiresAtMs, nowMs) {
+        this.dropExpiredBuckets(nowMs);
+        if (this.#nonces.has(nonce))
+            return "replayed";
+        if (this.#nonces.size >= this.maxEntries)
+            return "full";
+        const expiryBucket = Math.ceil(expiresAtMs / STATEFUL_FORWARDING_REPLAY_BUCKET_MS) *
+            STATEFUL_FORWARDING_REPLAY_BUCKET_MS;
+        this.#nonces.set(nonce, expiryBucket);
+        const bucket = this.#expiryBuckets.get(expiryBucket) ?? new Set();
+        bucket.add(nonce);
+        this.#expiryBuckets.set(expiryBucket, bucket);
+        this.#nextExpiryBucket = Math.min(this.#nextExpiryBucket ?? expiryBucket, expiryBucket);
+        this.#latestExpiryBucket = Math.max(this.#latestExpiryBucket ?? expiryBucket, expiryBucket);
+        return "accepted";
+    }
+    dropExpiredBuckets(nowMs) {
+        if (this.#nextExpiryBucket === undefined || this.#latestExpiryBucket === undefined)
+            return;
+        if (nowMs >= this.#latestExpiryBucket) {
+            this.#nonces.clear();
+            this.#expiryBuckets.clear();
+            this.#nextExpiryBucket = undefined;
+            this.#latestExpiryBucket = undefined;
+            return;
+        }
+        while (this.#nextExpiryBucket <= nowMs) {
+            const bucket = this.#expiryBuckets.get(this.#nextExpiryBucket);
+            if (bucket) {
+                for (const cachedNonce of bucket)
+                    this.#nonces.delete(cachedNonce);
+                this.#expiryBuckets.delete(this.#nextExpiryBucket);
+            }
+            this.#nextExpiryBucket += STATEFUL_FORWARDING_REPLAY_BUCKET_MS;
+        }
+    }
+}
+function verifyStatefulForwardingRequest(input) {
+    const config = input.options.statefulForwarding;
+    if (!config?.secret) {
+        throw new ProviderError("Stateful forwarding is not configured.", {
+            code: "STATEFUL_FORWARDING_NOT_CONFIGURED",
+        });
+    }
+    const timestamp = input.headers.get(STATEFUL_FORWARDING_TIMESTAMP_HEADER) ?? "";
+    const signature = input.headers.get(STATEFUL_FORWARDING_SIGNATURE_HEADER) ?? "";
+    const nonce = input.headers.get(STATEFUL_FORWARDING_NONCE_HEADER) ?? "";
+    if (!timestamp || !signature || !nonce) {
+        throw new ProviderError("Stateful forwarding signature headers are missing.", {
+            code: "STATEFUL_FORWARDING_SIGNATURE_MISSING",
+        });
+    }
+    if (nonce.length > 256) {
+        throw new ProviderError("Stateful forwarding nonce is invalid.", {
+            code: "STATEFUL_FORWARDING_NONCE_INVALID",
+        });
+    }
+    const timestampMs = Date.parse(timestamp);
+    const maxSkewMs = config.maxSkewMs ?? DEFAULT_STATEFUL_FORWARDING_MAX_SKEW_MS;
+    if (!Number.isFinite(timestampMs) || Math.abs(Date.now() - timestampMs) > maxSkewMs) {
+        throw new ProviderError("Stateful forwarding signature timestamp is outside the allowed skew.", { code: "STATEFUL_FORWARDING_TIMESTAMP_INVALID" });
+    }
+    if (!verifyStatefulRequestSignature({
+        secret: config.secret,
+        timestamp,
+        rawBody: input.rawBody,
+        method: input.method,
+        path: input.path,
+        nonce,
+        signature,
+    })) {
+        throw new ProviderError("Stateful forwarding signature is invalid.", {
+            code: "STATEFUL_FORWARDING_SIGNATURE_INVALID",
+        });
+    }
+    const replayResult = input.replayCache.claim(nonce, timestampMs + maxSkewMs, Date.now());
+    if (replayResult === "replayed") {
+        throw new ProviderError("Stateful forwarding nonce has already been used.", {
+            code: "STATEFUL_FORWARDING_REPLAY_DETECTED",
+        });
+    }
+    if (replayResult === "full") {
+        throw new ProviderError("Stateful forwarding replay cache is at capacity.", {
+            code: "STATEFUL_FORWARDING_REPLAY_CACHE_FULL",
+        });
+    }
+}
+function operationRequestFromForwardingEnvelope(envelope) {
+    return {
+        ...envelope.operationRequest,
+        ...(envelope.deadlineAt !== undefined ? { deadlineAt: envelope.deadlineAt } : {}),
+    };
+}
+function parseStatefulForwardingEnvelope(rawBody) {
+    const parsed = ProviderServerStatefulForwardEnvelopeSchema.safeParse(rawBody);
+    if (parsed.success)
+        return parsed.data;
+    throw new ProviderError("Stateful forwarding envelope is invalid.", {
+        code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
+        details: zodDetails(parsed.error),
+    });
+}
 export function createServerApp(provider, options = {}) {
+    validateStatefulServerConfig(options);
     const app = new Hono();
     const logger = options.logger ?? defaultProviderServerLogger;
+    const statefulForwardingReplayCache = new StatefulForwardingReplayCache(options.statefulForwarding?.replayCacheMaxEntries ??
+        DEFAULT_STATEFUL_FORWARDING_REPLAY_CACHE_MAX_ENTRIES);
     const state = options.state ??
         createProviderRuntimeStateFromEnv({
             providerId: provider.id,
@@ -994,6 +1161,120 @@ export function createServerApp(provider, options = {}) {
         provider: provider.id,
         version: provider.version,
     }));
+    app.post(STATEFUL_INTERNAL_OPERATIONS_ROUTE, async (c) => {
+        let rawBodyText = "";
+        let rawBody;
+        const operation = "stateful-internal";
+        const requestCost = startRequestCost();
+        try {
+            if (!options.internalOperationExecutor) {
+                throw new ProviderError("Stateful internal operation executor is not configured.", {
+                    code: "STATEFUL_INTERNAL_EXECUTOR_NOT_CONFIGURED",
+                });
+            }
+            rawBodyText = await c.req.raw.clone().text();
+            verifyStatefulForwardingRequest({
+                options,
+                rawBody: rawBodyText,
+                headers: c.req.raw.headers,
+                method: c.req.raw.method,
+                path: STATEFUL_INTERNAL_OPERATIONS_ROUTE,
+                replayCache: statefulForwardingReplayCache,
+            });
+            try {
+                rawBody = JSON.parse(rawBodyText);
+            }
+            catch {
+                throw new ProviderError("Stateful forwarding envelope is not valid JSON.", {
+                    code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
+                });
+            }
+            const envelope = parseStatefulForwardingEnvelope(rawBody);
+            if (envelope.providerId !== provider.id) {
+                throw new ProviderError("Stateful forwarding envelope providerId does not match the served provider.", { code: "STATEFUL_FORWARDING_PROVIDER_MISMATCH" });
+            }
+            if (envelope.requestId !== envelope.operationRequest.requestId) {
+                throw new ProviderError("Stateful forwarding requestId values do not match.", {
+                    code: "STATEFUL_FORWARDING_ENVELOPE_INVALID",
+                });
+            }
+            if (envelope.sourcePodId !==
+                (c.req.raw.headers.get(STATEFUL_FORWARDING_SOURCE_POD_HEADER) ?? "")) {
+                throw new ProviderError("Stateful forwarding source pod does not match its header.", {
+                    code: "STATEFUL_FORWARDING_SOURCE_POD_MISMATCH",
+                });
+            }
+            if (envelope.forwardedAt !== (c.req.raw.headers.get(STATEFUL_FORWARDING_TIMESTAMP_HEADER) ?? "")) {
+                throw new ProviderError("Stateful forwarding forwardedAt does not match its signature timestamp.", { code: "STATEFUL_FORWARDING_ENVELOPE_INVALID" });
+            }
+            const deadlineAtMs = envelope.deadlineAt ? Date.parse(envelope.deadlineAt) : undefined;
+            if (deadlineAtMs !== undefined && deadlineAtMs <= Date.now()) {
+                throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt);
+            }
+            const remainingDeadlineMs = deadlineAtMs === undefined ? undefined : deadlineAtMs - Date.now();
+            const deadlineSignal = remainingDeadlineMs === undefined ? undefined : AbortSignal.timeout(remainingDeadlineMs);
+            const signal = deadlineSignal
+                ? AbortSignal.any([c.req.raw.signal, deadlineSignal])
+                : c.req.raw.signal;
+            const ownerFenceValidation = Promise.resolve(options.statefulForwarding?.validateOwnerFence({
+                providerId: envelope.providerId,
+                sessionKey: envelope.sessionKey,
+                ownerPodId: envelope.ownerPodId,
+                generation: envelope.generation,
+                sourcePodId: envelope.sourcePodId,
+                forwardedAt: envelope.forwardedAt,
+                requestId: envelope.requestId,
+                ...(envelope.idempotencyKey ? { idempotencyKey: envelope.idempotencyKey } : {}),
+            }, signal));
+            let ownerFenceValid;
+            try {
+                ownerFenceValid = deadlineSignal
+                    ? await Promise.race([
+                        ownerFenceValidation,
+                        new Promise((_resolve, reject) => {
+                            deadlineSignal.addEventListener("abort", () => reject(new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt)), { once: true });
+                        }),
+                    ])
+                    : await ownerFenceValidation;
+            }
+            catch (error) {
+                if (deadlineSignal?.aborted) {
+                    throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt);
+                }
+                throw error;
+            }
+            if (ownerFenceValid !== true) {
+                throw new ProviderError("Stateful forwarding owner fence is no longer current.", {
+                    code: "STATEFUL_FORWARDING_OWNER_FENCE_INVALID",
+                });
+            }
+            const request = operationRequestFromForwardingEnvelope(envelope);
+            const operationId = envelope.operationId;
+            const ctx = createProviderContext(provider, request, operationId, options, state);
+            if (deadlineAtMs !== undefined && deadlineAtMs <= Date.now()) {
+                throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt);
+            }
+            const output = await options.internalOperationExecutor({
+                provider,
+                operationId,
+                ctx,
+                request,
+                internalStatefulForward: envelope,
+                signal,
+            });
+            logProviderSuccess(logger, provider, "operation", operationId || operation, request.requestId, 200, finishRequestCost(requestCost));
+            return c.json({ data: output });
+        }
+        catch (error) {
+            const status = toStatusCode(error);
+            if (isProviderError(error) && error.code === "STATEFUL_FORWARDING_REPLAY_CACHE_FULL") {
+                c.header("Retry-After", String(STATEFUL_FORWARDING_REPLAY_RETRY_AFTER_SECONDS));
+            }
+            const requestId = extractRequestId(rawBody);
+            logProviderError(logger, provider, "operation", operation, requestId, error, status, finishRequestCost(requestCost));
+            return c.json(toErrorResponse(error, requestId), status);
+        }
+    });
     app.post("/v1/:operation", async (c) => {
         let rawBody;
         const operation = c.req.param("operation");
@@ -1130,6 +1411,28 @@ export function createServerApp(provider, options = {}) {
     });
     return app;
 }
+function validateStatefulServerConfig(options) {
+    if (options.statefulForwarding && !options.internalOperationExecutor) {
+        throw new Error("Invalid provider server configuration: statefulForwarding requires internalOperationExecutor; missing option internalOperationExecutor.");
+    }
+    if (options.internalOperationExecutor && !options.statefulForwarding?.secret) {
+        throw new Error("Invalid provider server configuration: internalOperationExecutor requires statefulForwarding.secret; missing option statefulForwarding.secret.");
+    }
+    if (options.statefulForwarding &&
+        typeof options.statefulForwarding.validateOwnerFence !== "function") {
+        throw new Error("Invalid provider server configuration: statefulForwarding requires validateOwnerFence.");
+    }
+    if (options.statefulForwarding?.maxSkewMs !== undefined &&
+        (!Number.isFinite(options.statefulForwarding.maxSkewMs) ||
+            options.statefulForwarding.maxSkewMs <= 0)) {
+        throw new Error("Invalid provider server configuration: maxSkewMs must be positive.");
+    }
+    if (options.statefulForwarding?.replayCacheMaxEntries !== undefined &&
+        (!Number.isInteger(options.statefulForwarding.replayCacheMaxEntries) ||
+            options.statefulForwarding.replayCacheMaxEntries <= 0)) {
+        throw new Error("Invalid provider server configuration: replayCacheMaxEntries must be a positive integer.");
+    }
+}
 function getBunServeRuntime() {
     const bunValue = Object.getOwnPropertyDescriptor(globalThis, "Bun")?.value;
     if (!bunValue || typeof bunValue !== "object") {
@@ -1145,6 +1448,9 @@ function getBunServeRuntime() {
         },
     };
 }
+const DEFAULT_SHUTDOWN_TIMEOUT_MS = 30_000;
+const DEFAULT_SHUTDOWN_SIGNALS = ["SIGTERM", "SIGINT"];
+const processSignalCoordinators = new Map();
 export async function serve(provider, options = {}) {
     const bunRuntime = getBunServeRuntime();
     if (bunRuntime === undefined) {
@@ -1152,30 +1458,186 @@ export async function serve(provider, options = {}) {
             code: "RUNTIME_UNSUPPORTED",
         });
     }
+    const logger = options.logger ?? defaultProviderServerLogger;
+    const configuredTimeoutMs = shutdownTimeout(options.shutdown?.timeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS);
+    const configuredSignals = resolveShutdownSignals(options.shutdown?.signals ?? true);
     const app = createServerApp(provider, {
         logger: options.logger,
         stt: options.stt,
+        state: options.state,
+        allowMemoryStateFallback: options.allowMemoryStateFallback,
+        operationExecutor: options.operationExecutor,
+        internalOperationExecutor: options.internalOperationExecutor,
+        statefulForwarding: options.statefulForwarding,
     });
-    bunRuntime.serve({
-        port: options.port ?? DEFAULT_PORT,
-        hostname: options.host ?? DEFAULT_HOST,
-        fetch: app.fetch,
-    });
-    // Internal self-test listener (health dependency inversion): a SEPARATE
-    // socket the tenant-facing gateway never dials. Off by default — it only
-    // starts when the shared self-test master secret env is present.
-    const selfTestSecrets = resolveSelfTestMasterSecrets();
-    if (selfTestSecrets) {
-        const selfTestApp = createSelfTestApp(provider, {
-            secrets: selfTestSecrets,
-            invoke: createSelfTestInvoke(app),
-            authFlow: createSelfTestAuthFlowInvoke(app),
-        });
-        bunRuntime.serve({
-            port: options.selfTestPort ?? resolveSelfTestPort(),
+    const servers = [];
+    try {
+        servers.push(bunRuntime.serve({
+            port: options.port ?? DEFAULT_PORT,
             hostname: options.host ?? DEFAULT_HOST,
-            fetch: selfTestApp.fetch,
-        });
+            fetch: app.fetch,
+        }));
+        // Internal self-test listener (health dependency inversion): a SEPARATE
+        // socket the tenant-facing gateway never dials. Off by default — it only
+        // starts when the shared self-test master secret env is present.
+        const selfTestSecrets = resolveSelfTestMasterSecrets();
+        if (selfTestSecrets) {
+            const selfTestApp = createSelfTestApp(provider, {
+                secrets: selfTestSecrets,
+                invoke: createSelfTestInvoke(app),
+                authFlow: createSelfTestAuthFlowInvoke(app),
+            });
+            servers.push(bunRuntime.serve({
+                port: options.selfTestPort ?? resolveSelfTestPort(),
+                hostname: options.host ?? DEFAULT_HOST,
+                fetch: selfTestApp.fetch,
+            }));
+        }
+    }
+    catch (error) {
+        await Promise.allSettled(servers.map((startedServer) => startedServer.stop(true)));
+        throw error;
+    }
+    const server = servers[0];
+    if (!server)
+        throw new Error("Provider server failed to create its primary listener.");
+    let closePromise;
+    let unregisterSignals = () => { };
+    const close = (closeOptions = {}) => {
+        if (closePromise)
+            return closePromise;
+        const timeoutMs = shutdownTimeout(closeOptions.timeoutMs ?? configuredTimeoutMs);
+        closePromise = closeProviderServers({
+            servers,
+            hooks: options.shutdown?.hooks ?? [],
+            timeoutMs,
+            logger,
+            providerId: provider.id,
+        }).finally(() => unregisterSignals());
+        return closePromise;
+    };
+    try {
+        unregisterSignals = registerForProcessSignals(configuredSignals, () => close({ timeoutMs: configuredTimeoutMs }));
+    }
+    catch (error) {
+        unregisterSignals();
+        await Promise.allSettled(servers.map((startedServer) => startedServer.stop(true)));
+        throw error;
+    }
+    return { port: server.port, close };
+}
+function registerForProcessSignals(signals, close) {
+    if (signals.length === 0)
+        return () => { };
+    const registration = {
+        signals: new Set(signals),
+        close,
+    };
+    let registered = true;
+    const unregister = () => {
+        if (!registered)
+            return;
+        registered = false;
+        for (const signal of registration.signals) {
+            const coordinator = processSignalCoordinators.get(signal);
+            if (!coordinator)
+                continue;
+            coordinator.registrations.delete(registration);
+            if (coordinator.registrations.size === 0 && !coordinator.handling) {
+                process.removeListener(signal, coordinator.listener);
+                processSignalCoordinators.delete(signal);
+            }
+        }
+    };
+    try {
+        for (const signal of signals) {
+            let coordinator = processSignalCoordinators.get(signal);
+            if (!coordinator) {
+                const created = {
+                    registrations: new Set(),
+                    handling: false,
+                    listener: () => handleCoordinatedSignal(signal, created),
+                };
+                coordinator = created;
+                processSignalCoordinators.set(signal, coordinator);
+                process.on(signal, coordinator.listener);
+            }
+            coordinator.registrations.add(registration);
+        }
+    }
+    catch (error) {
+        unregister();
+        throw error;
+    }
+    return unregister;
+}
+function handleCoordinatedSignal(signal, coordinator) {
+    if (coordinator.handling)
+        return;
+    coordinator.handling = true;
+    const registrations = [...coordinator.registrations];
+    void Promise.allSettled(registrations.map((registration) => registration.close())).finally(() => {
+        if (processSignalCoordinators.get(signal) === coordinator) {
+            process.removeListener(signal, coordinator.listener);
+            processSignalCoordinators.delete(signal);
+        }
+        try {
+            process.kill(process.pid, signal);
+        }
+        catch {
+            process.exitCode = 1;
+        }
+    });
+}
+async function closeProviderServers(input) {
+    const deadline = Date.now() + input.timeoutMs;
+    const gracefulStops = input.servers.map((server) => server.stop(false));
+    for (const gracefulStop of gracefulStops)
+        gracefulStop.catch(() => undefined);
+    for (const [hookIndex, hook] of input.hooks.entries()) {
+        try {
+            await withinShutdownBudget(Promise.resolve().then(hook), deadline);
+        }
+        catch (error) {
+            try {
+                input.logger({
+                    level: "error",
+                    event: "provider_shutdown_hook_failed",
+                    providerId: input.providerId,
+                    hookIndex,
+                    errorClass: error instanceof Error ? error.name : "UnknownError",
+                    message: error instanceof Error ? error.message : "Shutdown hook failed.",
+                });
+            }
+            catch { }
+        }
+    }
+    const forcedStops = input.servers.map((server) => server.stop(true));
+    await withinShutdownBudget(Promise.allSettled([...gracefulStops, ...forcedStops]).then(() => undefined), deadline).catch(() => undefined);
+}
+async function withinShutdownBudget(promise, deadline) {
+    promise.catch(() => undefined);
+    const remainingMs = Math.max(0, deadline - Date.now());
+    let timer;
+    const timeout = new Promise((_resolve, reject) => {
+        timer = setTimeout(() => reject(new Error("Provider server shutdown timed out.")), remainingMs);
+    });
+    try {
+        return await Promise.race([promise, timeout]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+function resolveShutdownSignals(signals) {
+    if (signals === false)
+        return [];
+    return [...new Set(signals === true ? DEFAULT_SHUTDOWN_SIGNALS : signals)];
+}
+function shutdownTimeout(value) {
+    if (!Number.isFinite(value) || value < 0) {
+        throw new Error("Provider server shutdown timeoutMs must be a non-negative finite number.");
     }
-    await Promise.resolve();
+    return value;
 }

package/dist/stateful/errors.d.ts

@@ -0,0 +1,14 @@
+export type StatefulControlPlaneOperation = "resolve" | "acquire" | "renew" | "release";
+export declare class StatefulControlPlaneError extends Error {
+    readonly code: string;
+    readonly operation: StatefulControlPlaneOperation;
+    readonly status?: number;
+    readonly cause?: unknown;
+    constructor(input: {
+        readonly code: string;
+        readonly message: string;
+        readonly operation: StatefulControlPlaneOperation;
+        readonly status?: number;
+        readonly cause?: unknown;
+    });
+}

package/dist/stateful/errors.js

@@ -0,0 +1,14 @@
+export class StatefulControlPlaneError extends Error {
+    code;
+    operation;
+    status;
+    cause;
+    constructor(input) {
+        super(input.message);
+        this.name = "StatefulControlPlaneError";
+        this.code = input.code;
+        this.operation = input.operation;
+        this.status = input.status;
+        this.cause = input.cause;
+    }
+}

package/dist/stateful/http-provider-event-emitter.d.ts

@@ -0,0 +1,40 @@
+import type { ProviderEventDeliveryFailureRecorder } from "./provider-event-delivery-failures.js";
+import type { ProviderEventPublisher, ProviderEventPublishOptions, PublishAck } from "./provider-event-pipeline.js";
+import { type ProviderEventMetricEmitter } from "./provider-event-pipeline-metrics.js";
+import type { ProviderEvent } from "./provider-events.js";
+type FetchTransport = (url: string | URL | Request, init?: RequestInit) => Promise<Response>;
+export type HttpProviderEventEmitterOptions = {
+    readonly baseUrl: string;
+    readonly secret: string;
+    readonly fetch?: FetchTransport;
+    readonly clock?: () => Date;
+    readonly random?: () => number;
+    readonly metricEmitter?: ProviderEventMetricEmitter;
+    readonly failureRecorder?: ProviderEventDeliveryFailureRecorder;
+    readonly maxBufferedEvents?: number;
+    readonly retryBaseMs?: number;
+    readonly retryMaxMs?: number;
+    readonly maxAttempts?: number;
+    readonly jitterRatio?: number;
+};
+export type ProviderEventFlushReport = {
+    /** Events delivered during this emitter's lifetime. */
+    readonly delivered: number;
+    /** Events permanently dropped during this emitter's lifetime. */
+    readonly failed: number;
+    /** Events still buffered when flush returned. */
+    readonly pending: number;
+};
+/**
+ * Provider-pod HTTP publication with a bounded best-effort delivery guarantee and at-least-once
+ * transport retries. This is not durable at-least-once delivery: buffered events can be lost if
+ * the process is killed. Graceful shutdown should drain the in-memory buffer with flush().
+ */
+export declare class HttpProviderEventEmitter implements ProviderEventPublisher {
+    #private;
+    constructor(options: HttpProviderEventEmitterOptions);
+    publish(event: ProviderEvent, options: ProviderEventPublishOptions): PublishAck;
+    pendingCount(): number;
+    flush(timeoutMs?: number): Promise<ProviderEventFlushReport>;
+}
+export {};